<commit_before>package cron

import (
	"log"
	"math"
	"strconv"
	"strings"
	"time"
)

// Parse returns a new crontab schedule representing the given spec.
// It panics with a descriptive error if the spec is not valid.
//
// It accepts
//   - Full crontab specs, e.g. "* * * * * ?"
//   - Descriptors, e.g. "@midnight", "@every 1h30m"
func Parse(spec string) Schedule {
	if spec[0] == '@' {
		return parseDescriptor(spec)
	}

	// Split on whitespace.  We require 4 or 5 fields.
	// (minute) (hour) (day of month) (month) (day of week, optional)
	fields := strings.Fields(spec)
	if len(fields) != 5 && len(fields) != 6 {
		log.Panicf("Expected 5 or 6 fields, found %d: %s", len(fields), spec)
	}

	// If a fifth field is not provided (DayOfWeek), then it is equivalent to star.
	if len(fields) == 5 {
		fields = append(fields, "*")
	}

	schedule := &SpecSchedule{
		Second: getField(fields[0], seconds),
		Minute: getField(fields[1], minutes),
		Hour:   getField(fields[2], hours),
		Dom:    getField(fields[3], dom),
		Month:  getField(fields[4], months),
		Dow:    getField(fields[5], dow),
	}

	return schedule
}

// getField returns an Int with the bits set representing all of the times that
// the field represents.  A "field" is a comma-separated list of "ranges".
func getField(field string, r bounds) uint64 {
	// list = range {"," range}
	var bits uint64
	ranges := strings.FieldsFunc(field, func(r rune) bool { return r == ',' })
	for _, expr := range ranges {
		bits |= getRange(expr, r)
	}
	return bits
}

// getRange returns the bits indicated by the given expression:
//   number | number "-" number [ "/" number ]
func getRange(expr string, r bounds) uint64 {

	var (
		start, end, step uint
		rangeAndStep     = strings.Split(expr, "/")
		lowAndHigh       = strings.Split(rangeAndStep[0], "-")
		singleDigit      = len(lowAndHigh) == 1
	)

	var extra_star uint64
	if lowAndHigh[0] == "*" || lowAndHigh[0] == "?" {
		start = r.min
		end = r.max
		extra_star = starBit
	} else {
		start = parseIntOrName(lowAndHigh[0], r.names)
		switch len(lowAndHigh) {
		case 1:
			end = start
		case 2:
			end = parseIntOrName(lowAndHigh[1], r.names)
		default:
			log.Panicf("Too many hyphens: %s", expr)
		}
	}

	switch len(rangeAndStep) {
	case 1:
		step = 1
	case 2:
		step = mustParseInt(rangeAndStep[1])

		// Special handling: "N/step" means "N-max/step".
		if singleDigit {
			end = r.max
		}
	default:
		log.Panicf("Too many slashes: %s", expr)
	}

	if start < r.min {
		log.Panicf("Beginning of range (%d) below minimum (%d): %s", start, r.min, expr)
	}
	if end > r.max {
		log.Panicf("End of range (%d) above maximum (%d): %s", end, r.max, expr)
	}
	if start > end {
		log.Panicf("Beginning of range (%d) beyond end of range (%d): %s", start, end, expr)
	}

	return getBits(start, end, step) | extra_star
}

// parseIntOrName returns the (possibly-named) integer contained in expr.
func parseIntOrName(expr string, names map[string]uint) uint {
	if names != nil {
		if namedInt, ok := names[strings.ToLower(expr)]; ok {
			return namedInt
		}
	}
	return mustParseInt(expr)
}

// mustParseInt parses the given expression as an int or panics.
func mustParseInt(expr string) uint {
	num, err := strconv.Atoi(expr)
	if err != nil {
		log.Panicf("Failed to parse int from %s: %s", expr, err)
	}
	if num < 0 {
		log.Panicf("Negative number (%d) not allowed: %s", num, expr)
	}

	return uint(num)
}

// getBits sets all bits in the range [min, max], modulo the given step size.
func getBits(min, max, step uint) uint64 {
	var bits uint64

	// If step is 1, use shifts.
	if step == 1 {
		return ^(math.MaxUint64 << (max + 1)) & (math.MaxUint64 << min)
	}

	// Else, use a simple loop.
	for i := min; i <= max; i += step {
		bits |= 1 << i
	}
	return bits
}

// all returns all bits within the given bounds.  (plus the star bit)
func all(r bounds) uint64 {
	return getBits(r.min, r.max, 1) | starBit
}

// first returns bits with only the first (minimum) value set.
func first(r bounds) uint64 {
	return getBits(r.min, r.min, 1)
}

// parseDescriptor returns a pre-defined schedule for the expression, or panics
// if none matches.
func parseDescriptor(spec string) Schedule {
	switch spec {
	case "@yearly", "@annually":
		return &SpecSchedule{
			Second: 1 << seconds.min,
			Minute: 1 << minutes.min,
			Hour:   1 << hours.min,
			Dom:    1 << dom.min,
			Month:  1 << months.min,
			Dow:    all(dow),
		}

	case "@monthly":
		return &SpecSchedule{
			Second: 1 << seconds.min,
			Minute: 1 << minutes.min,
			Hour:   1 << hours.min,
			Dom:    1 << dom.min,
			Month:  all(months),
			Dow:    all(dow),
		}

	case "@weekly":
		return &SpecSchedule{
			Second: 1 << seconds.min,
			Minute: 1 << minutes.min,
			Hour:   1 << hours.min,
			Dom:    all(dom),
			Month:  all(months),
			Dow:    1 << dow.min,
		}

	case "@daily", "@midnight":
		return &SpecSchedule{
			Second: 1 << seconds.min,
			Minute: 1 << minutes.min,
			Hour:   1 << hours.min,
			Dom:    all(dom),
			Month:  all(months),
			Dow:    all(dow),
		}

	case "@hourly":
		return &SpecSchedule{
			Second: 1 << seconds.min,
			Minute: 1 << minutes.min,
			Hour:   all(hours),
			Dom:    all(dom),
			Month:  all(months),
			Dow:    all(dow),
		}
	}

	const every = "@every "
	if strings.HasPrefix(spec, every) {
		duration, err := time.ParseDuration(spec[len(every):])
		if err != nil {
			log.Panicf("Failed to parse duration %s: %s", spec, err)
		}
		return Every(duration)
	}

	log.Panicf("Unrecognized descriptor: %s", spec)
	return nil
}
<commit_msg>Fix parser comment about the required fields<commit_after>package cron

import (
	"log"
	"math"
	"strconv"
	"strings"
	"time"
)

// Parse returns a new crontab schedule representing the given spec.
// It panics with a descriptive error if the spec is not valid.
//
// It accepts
//   - Full crontab specs, e.g. "* * * * * ?"
//   - Descriptors, e.g. "@midnight", "@every 1h30m"
func Parse(spec string) Schedule {
	if spec[0] == '@' {
		return parseDescriptor(spec)
	}

	// Split on whitespace.  We require 5 or 6 fields.
	// (second) (minute) (hour) (day of month) (month) (day of week, optional)
	fields := strings.Fields(spec)
	if len(fields) != 5 && len(fields) != 6 {
		log.Panicf("Expected 5 or 6 fields, found %d: %s", len(fields), spec)
	}

	// If a sixth field is not provided (DayOfWeek), then it is equivalent to star.
	if len(fields) == 5 {
		fields = append(fields, "*")
	}

	schedule := &SpecSchedule{
		Second: getField(fields[0], seconds),
		Minute: getField(fields[1], minutes),
		Hour:   getField(fields[2], hours),
		Dom:    getField(fields[3], dom),
		Month:  getField(fields[4], months),
		Dow:    getField(fields[5], dow),
	}

	return schedule
}

// getField returns an Int with the bits set representing all of the times that
// the field represents.  A "field" is a comma-separated list of "ranges".
func getField(field string, r bounds) uint64 {
	// list = range {"," range}
	var bits uint64
	ranges := strings.FieldsFunc(field, func(r rune) bool { return r == ',' })
	for _, expr := range ranges {
		bits |= getRange(expr, r)
	}
	return bits
}

// getRange returns the bits indicated by the given expression:
//   number | number "-" number [ "/" number ]
func getRange(expr string, r bounds) uint64 {

	var (
		start, end, step uint
		rangeAndStep     = strings.Split(expr, "/")
		lowAndHigh       = strings.Split(rangeAndStep[0], "-")
		singleDigit      = len(lowAndHigh) == 1
	)

	var extra_star uint64
	if lowAndHigh[0] == "*" || lowAndHigh[0] == "?" {
		start = r.min
		end = r.max
		extra_star = starBit
	} else {
		start = parseIntOrName(lowAndHigh[0], r.names)
		switch len(lowAndHigh) {
		case 1:
			end = start
		case 2:
			end = parseIntOrName(lowAndHigh[1], r.names)
		default:
			log.Panicf("Too many hyphens: %s", expr)
		}
	}

	switch len(rangeAndStep) {
	case 1:
		step = 1
	case 2:
		step = mustParseInt(rangeAndStep[1])

		// Special handling: "N/step" means "N-max/step".
		if singleDigit {
			end = r.max
		}
	default:
		log.Panicf("Too many slashes: %s", expr)
	}

	if start < r.min {
		log.Panicf("Beginning of range (%d) below minimum (%d): %s", start, r.min, expr)
	}
	if end > r.max {
		log.Panicf("End of range (%d) above maximum (%d): %s", end, r.max, expr)
	}
	if start > end {
		log.Panicf("Beginning of range (%d) beyond end of range (%d): %s", start, end, expr)
	}

	return getBits(start, end, step) | extra_star
}

// parseIntOrName returns the (possibly-named) integer contained in expr.
func parseIntOrName(expr string, names map[string]uint) uint {
	if names != nil {
		if namedInt, ok := names[strings.ToLower(expr)]; ok {
			return namedInt
		}
	}
	return mustParseInt(expr)
}

// mustParseInt parses the given expression as an int or panics.
func mustParseInt(expr string) uint {
	num, err := strconv.Atoi(expr)
	if err != nil {
		log.Panicf("Failed to parse int from %s: %s", expr, err)
	}
	if num < 0 {
		log.Panicf("Negative number (%d) not allowed: %s", num, expr)
	}

	return uint(num)
}

// getBits sets all bits in the range [min, max], modulo the given step size.
func getBits(min, max, step uint) uint64 {
	var bits uint64

	// If step is 1, use shifts.
	if step == 1 {
		return ^(math.MaxUint64 << (max + 1)) & (math.MaxUint64 << min)
	}

	// Else, use a simple loop.
	for i := min; i <= max; i += step {
		bits |= 1 << i
	}
	return bits
}

// all returns all bits within the given bounds.  (plus the star bit)
func all(r bounds) uint64 {
	return getBits(r.min, r.max, 1) | starBit
}

// first returns bits with only the first (minimum) value set.
func first(r bounds) uint64 {
	return getBits(r.min, r.min, 1)
}

// parseDescriptor returns a pre-defined schedule for the expression, or panics
// if none matches.
func parseDescriptor(spec string) Schedule {
	switch spec {
	case "@yearly", "@annually":
		return &SpecSchedule{
			Second: 1 << seconds.min,
			Minute: 1 << minutes.min,
			Hour:   1 << hours.min,
			Dom:    1 << dom.min,
			Month:  1 << months.min,
			Dow:    all(dow),
		}

	case "@monthly":
		return &SpecSchedule{
			Second: 1 << seconds.min,
			Minute: 1 << minutes.min,
			Hour:   1 << hours.min,
			Dom:    1 << dom.min,
			Month:  all(months),
			Dow:    all(dow),
		}

	case "@weekly":
		return &SpecSchedule{
			Second: 1 << seconds.min,
			Minute: 1 << minutes.min,
			Hour:   1 << hours.min,
			Dom:    all(dom),
			Month:  all(months),
			Dow:    1 << dow.min,
		}

	case "@daily", "@midnight":
		return &SpecSchedule{
			Second: 1 << seconds.min,
			Minute: 1 << minutes.min,
			Hour:   1 << hours.min,
			Dom:    all(dom),
			Month:  all(months),
			Dow:    all(dow),
		}

	case "@hourly":
		return &SpecSchedule{
			Second: 1 << seconds.min,
			Minute: 1 << minutes.min,
			Hour:   all(hours),
			Dom:    all(dom),
			Month:  all(months),
			Dow:    all(dow),
		}
	}

	const every = "@every "
	if strings.HasPrefix(spec, every) {
		duration, err := time.ParseDuration(spec[len(every):])
		if err != nil {
			log.Panicf("Failed to parse duration %s: %s", spec, err)
		}
		return Every(duration)
	}

	log.Panicf("Unrecognized descriptor: %s", spec)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"regexp"
	"strconv"
)

type state struct {
	Modules []moduleState `json:"modules"`
}

// keyNames contains the names of the keys to check for in each resource in the
// state file. This allows us to support multiple types of resource without too
// much fuss.
var keyNames []string
var nameParser *regexp.Regexp

func init() {
	keyNames = []string{
		"ipv4_address", // DO
		"public_ip",    // AWS
		"private_ip",   // AWS
		"ipaddress",    // CS
		"ip_address",   // VMware
		"access_ip_v4", // OpenStack
		"floating_ip",  // OpenStack
	}

	// type.name.0
	nameParser = regexp.MustCompile(`^(\w+)\.([\w\-]+)(?:\.(\d+))?$`)
}

// read populates the state object from a statefile.
func (s *state) read(stateFile io.Reader) error {

	// read statefile contents
	b, err := ioutil.ReadAll(stateFile)
	if err != nil {
		return err
	}

	// parse into struct
	err = json.Unmarshal(b, s)
	if err != nil {
		return err
	}

	return nil
}

// resources returns a map of name to resourceState, for any supported resources
// found in the statefile.
func (s *state) resources() map[string]resourceState {
	inst := make(map[string]resourceState)

	for _, m := range s.Modules {
		for k, r := range m.Resources {
			if r.isSupported() {

				_, name, counter := parseName(k)
				//fmt.Println(resType, name, counter)
				r.Name = name
				r.Counter = counter
				inst[k] = r
			}
		}
	}

	return inst
}

func parseName(name string) (string, string, int) {
	m := nameParser.FindStringSubmatch(name)

	// This should not happen unless our regex changes.
	// TODO: Warn instead of silently ignore error?
	if len(m) != 4 {
		//fmt.Printf("len=%d\n", len(m))
		return "", "", 0
	}

	var c int
	var err error
	if m[3] != "" {
		c, err = strconv.Atoi(m[3])
		if err != nil {
			fmt.Printf("err: %s\n", err)
			// ???
		}
	}

	return m[1], m[2], c
}

type moduleState struct {
	Resources map[string]resourceState `json:"resources"`
}

type resourceState struct {

	// Populated from statefile
	Type    string        `json:"type"`
	Primary instanceState `json:"primary"`

	// Extracted from key name, and injected by resources method
	Name    string
	Counter int
}

// isSupported returns true if terraform-inventory supports this resource.
func (s resourceState) isSupported() bool {
	return s.Address() != ""
}

// NameWithCounter returns the resource name with its counter. For resources
// created without a 'count=' attribute, this will always be zero.
func (s resourceState) NameWithCounter() string {
	return fmt.Sprintf("%s.%d", s.Name, s.Counter)
}

// Address returns the IP address of this resource.
func (s resourceState) Address() string {
	for _, key := range keyNames {
		if ip := s.Primary.Attributes[key]; ip != "" {
			return ip
		}
	}

	return ""
}

// Attributes returns a map containing everything we know about this resource.
func (s resourceState) Attributes() map[string]string {
	return s.Primary.Attributes
}

type instanceState struct {
	ID         string            `json:"id"`
	Attributes map[string]string `json:"attributes,omitempty"`
}
<commit_msg>Remove debugging printfs<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"regexp"
	"strconv"
)

type state struct {
	Modules []moduleState `json:"modules"`
}

// keyNames contains the names of the keys to check for in each resource in the
// state file. This allows us to support multiple types of resource without too
// much fuss.
var keyNames []string
var nameParser *regexp.Regexp

func init() {
	keyNames = []string{
		"ipv4_address", // DO
		"public_ip",    // AWS
		"private_ip",   // AWS
		"ipaddress",    // CS
		"ip_address",   // VMware
		"access_ip_v4", // OpenStack
		"floating_ip",  // OpenStack
	}

	// type.name.0
	nameParser = regexp.MustCompile(`^(\w+)\.([\w\-]+)(?:\.(\d+))?$`)
}

// read populates the state object from a statefile.
func (s *state) read(stateFile io.Reader) error {

	// read statefile contents
	b, err := ioutil.ReadAll(stateFile)
	if err != nil {
		return err
	}

	// parse into struct
	err = json.Unmarshal(b, s)
	if err != nil {
		return err
	}

	return nil
}

// resources returns a map of name to resourceState, for any supported resources
// found in the statefile.
func (s *state) resources() map[string]resourceState {
	inst := make(map[string]resourceState)

	for _, m := range s.Modules {
		for k, r := range m.Resources {
			if r.isSupported() {

				_, name, counter := parseName(k)
				r.Name = name
				r.Counter = counter
				inst[k] = r
			}
		}
	}

	return inst
}

func parseName(name string) (string, string, int) {
	m := nameParser.FindStringSubmatch(name)

	// This should not happen unless our regex changes.
	// TODO: Warn instead of silently ignore error?
	if len(m) != 4 {
		return "", "", 0
	}

	var c int
	var err error
	if m[3] != "" {

		// The third section should be the index, if it's present. Not sure what
		// else we can do other than panic (which seems highly undesirable) if that
		// isn't the case.
		c, err = strconv.Atoi(m[3])
		if err != nil {
			c = 0
		}
	}

	return m[1], m[2], c
}

type moduleState struct {
	Resources map[string]resourceState `json:"resources"`
}

type resourceState struct {

	// Populated from statefile
	Type    string        `json:"type"`
	Primary instanceState `json:"primary"`

	// Extracted from key name, and injected by resources method
	Name    string
	Counter int
}

// isSupported returns true if terraform-inventory supports this resource.
func (s resourceState) isSupported() bool {
	return s.Address() != ""
}

// NameWithCounter returns the resource name with its counter. For resources
// created without a 'count=' attribute, this will always be zero.
func (s resourceState) NameWithCounter() string {
	return fmt.Sprintf("%s.%d", s.Name, s.Counter)
}

// Address returns the IP address of this resource.
func (s resourceState) Address() string {
	for _, key := range keyNames {
		if ip := s.Primary.Attributes[key]; ip != "" {
			return ip
		}
	}

	return ""
}

// Attributes returns a map containing everything we know about this resource.
func (s resourceState) Attributes() map[string]string {
	return s.Primary.Attributes
}

type instanceState struct {
	ID         string            `json:"id"`
	Attributes map[string]string `json:"attributes,omitempty"`
}
<|endoftext|>


<commit_before>package meta

// StreamInfo contains information about the FLAC audio stream. It must be
// present as the first metadata block of a FLAC stream.
//
// ref: https://www.xiph.org/flac/format.html#metadata_block_streaminfo
type StreamInfo struct{}
<commit_msg>meta: Add StreamInfo definition.<commit_after>package meta

import "crypto/md5"

// StreamInfo contains information about the FLAC audio stream. It must be
// present as the first metadata block of a FLAC stream.
//
// ref: https://www.xiph.org/flac/format.html#metadata_block_streaminfo
type StreamInfo struct {
	// Minimum block size (in samples) used in the stream.
	BlockSizeMin uint16
	// Maximum block size (in samples) used in the stream.
	BlockSizeMax uint16
	// Minimum frame size in bytes; a 0 value implies unknown.
	FrameSizeMin uint32
	// Maximum frame size in bytes; a 0 value implies unknown.
	FrameSizeMax uint32
	// Sample rate in Hz; between 1 and 655350 Hz.
	SampleRate uint32
	// Number of channels; between 1 and 8 channels.
	NChannels uint8
	// Sample size in bits-per-sample; between 4 and 32 bits.
	BitsPerSample uint8
	// Total number of inter-channel samples in the stream. One second of 44.1
	// KHz audio will have 44100 samples regardless of the number of channels. A
	// 0 value implies unknown.
	NSamples uint64
	// MD5 checksum of the unencoded audio data.
	MD5sum [md5.BlockSize]uint8
}
<|endoftext|>


<commit_before>package main

import (
  "bufio"
  "fmt"
  "launchpad.net/xmlpath"
  "log"
  "os"
  "regexp"
  "sort"
  "strings"
)

type Command struct {
  Cmd            string
  Arguments      []string
  NamedArguments map[string]string
}

func (cmd *Command) ArgOrEmpty(key string) string {
  return cmd.ArgOrElse(key, "")
}

func (cmd *Command) ArgOrElse(key string, def string) string {
  ret, has := cmd.NamedArguments[key]
  if !has {
    return def
  }
  return ret
}

type Book struct {
  Title  string
  Author string
  Editor string
  Year   string
  Page   string
  Isbn   string
}

type Quote struct {
  Text   string
  Author string
  Book   Book
}

type BookQuote struct {
  Book  Book
  Quote Quote
}

func (bq *BookQuote) ToString() string {
  var out string
  out = fmt.Sprintf("Quote: \"%s\", by %s, from %s, page, %d published in %d", bq.Quote.Text, bq.Quote.Author, bq.Book.Title, bq.Book.Page, bq.Book.Year)
  return out
}

// Thank you andrew https://groups.google.com/d/msg/golang-nuts/FT7cjmcL7gw/Gj4_aEsE_IsJ
// A data structure to hold a key/value pair.
type Pair struct {
  Key   string
  Value int
}

// A slice of Pairs that implements sort.Interface to sort by Value.
type PairList []Pair

func (p PairList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p PairList) Len() int           { return len(p) }
func (p PairList) Less(i, j int) bool { return p[i].Value < p[j].Value }

// A function to turn a map into a PairList, then sort and return it.
func sortMapByValue(m map[string]int) PairList {
  p := make(PairList, len(m))
  i := 0
  for k, v := range m {
    p[i] = Pair{k, v}
    i += 1
  }
  sort.Sort(sort.Reverse(p))
  return p
}

func markupExtractor(body string) []Command {
  markup := regexp.MustCompile("(?s){{[^}]+}}")
  param := regexp.MustCompile("([^=]+)=(.*)")

  strCommands := markup.FindAllString(body, -1)

  commands := make([]Command, len(strCommands))
  pos := 0

  for _, cmd := range strCommands {
    cmd = cmd[2 : len(cmd)-2]

    arguments := make([]string, 10000)
    argumentsIndex := 0
    kvArguments := make(map[string]string, 1000)

    for _, arg := range strings.Split(cmd, "|") {
      kv := param.FindStringSubmatch(arg)
      if len(kv) == 3 {
        key := strings.TrimSpace(strings.ToLower(kv[1]))
        val, exists := kvArguments[key]
        if exists && val != kv[0] {
          // FIXME: handle the issue
          // panic(fmt.Sprintf("Parameter %s already exists with value \"%s\", here wants : \"%s\"", key, val, kv[0]))
        } else {
          kvArguments[key] = kv[2]
        }
      } else {
        arguments[argumentsIndex] = arg
        argumentsIndex += 1
      }
    }

    cmd := strings.TrimSpace(strings.ToLower(arguments[0]))

    // Parse special "defaultsort:", "if:", "msg:" commands
    if strings.Index(cmd, ":") != -1 {
      cmd = cmd[0:strings.Index(cmd, ":")]
      // FIXME inject arguments in Command anyway
    }

    // Ignore the empty command
    if cmd != "" {
      commands[pos] = Command{Cmd: cmd, Arguments: arguments[1:], NamedArguments: kvArguments}
      pos += 1
    }
  }
  return commands
}

func BuildQuote(qCommand Command, reference Command) {
  quote, hasQuote := qCommand.NamedArguments["citation"]
  if !hasQuote {
    quote = qCommand.Arguments[0]
    hasQuote = true
  }

  author := reference.ArgOrEmpty("auteur")
  title := reference.ArgOrEmpty("titre")
  page := reference.ArgOrEmpty("page")
  editor := reference.ArgOrEmpty("éditeur")
  year := reference.ArgOrEmpty("année")
  isbn := reference.ArgOrEmpty("isbn")

  if quote == "" || author == "" || title == "" || page == "" || editor == "" || year == "" || isbn == "" {
    fmt.Println("Some fields are missing")
    fmt.Println("", reference)
  }

}

func ExtractQuotes(commands []Command) {
  var buffer *Command = nil
  for ix, cmd := range commands {
    if len(cmd.Cmd) > 2 && cmd.Cmd[0:4] == "réf" {
      if buffer != nil {
        BuildQuote(*buffer, cmd)
      } else {
        fmt.Println("Found ref without quote")
        fmt.Println(cmd)
      }
      buffer = nil
    } else if strings.Index(cmd.Cmd, "citation") == 0 {
      buffer = &commands[ix]
    } else {
      fmt.Println("Unknown command:", cmd.Cmd)
      buffer = nil
    }
  }
}

func ExtractStats(commands []Command) {
  commandPopularity := make(map[string]int, 1000)
  argsPopularity := make(map[string]map[string]int, 100)

  var count int
  var hasCommand bool
  var aCount int
  var hasArg bool

  for _, cmd := range commands {
    count, hasCommand = commandPopularity[cmd.Cmd]
    if hasCommand {
      commandPopularity[cmd.Cmd] = count + 1
    } else {
      commandPopularity[cmd.Cmd] = 1
      argsPopularity[cmd.Cmd] = make(map[string]int, 100)
    }

    for k := range cmd.NamedArguments {
      aCount, hasArg = argsPopularity[cmd.Cmd][k]
      if hasArg {
        argsPopularity[cmd.Cmd][k] = aCount + 1
      } else {
        argsPopularity[cmd.Cmd][k] = 1
      }
    }
  }

  var maxArgsOcc int
  for _, pl := range sortMapByValue(commandPopularity) {
    fmt.Printf("## %s (%d occ.)\n", pl.Key, pl.Value)
    maxArgsOcc = 0
    for _, ar := range sortMapByValue(argsPopularity[pl.Key]) {
      if ar.Value > maxArgsOcc {
        maxArgsOcc = ar.Value
      }
      if ar.Value > maxArgsOcc/10 {
        fmt.Printf("- %s (%d occ.)\n", ar.Key, ar.Value)
      }
    }
  }
}

func main() {
  pageXPath := xmlpath.MustCompile("/mediawiki/page")
  textXPath := xmlpath.MustCompile(("revision/text"))
  fi, err := os.Open("frwikiquote-20140622-pages-articles-multistream.xml")
  // fi, err := os.Open("sample.xml")

  if err != nil {

    panic(err)
  }
  // close fi on exit and check for its returned error
  defer func() {
    if err := fi.Close(); err != nil {
      panic(err)
    }
  }()
  // make a read buffer
  r := bufio.NewReader(fi)

  root, err := xmlpath.Parse(r)
  if err != nil {
    log.Fatal(err)
  }
  commands := make([]Command, 0)
  iter := pageXPath.Iter(root)
  for iter.Next() {
    page := iter.Node()
    content, _ := textXPath.String(page)
    commands = append(commands, markupExtractor(content)...)
  }
  ExtractStats(commands)
}
<commit_msg>Dump % of parameters usage<commit_after>package main

import (
  "bufio"
  "fmt"
  "launchpad.net/xmlpath"
  "log"
  "os"
  "regexp"
  "sort"
  "strings"
)

type Command struct {
  Cmd            string
  Arguments      []string
  NamedArguments map[string]string
}

func (cmd *Command) ArgOrEmpty(key string) string {
  return cmd.ArgOrElse(key, "")
}

func (cmd *Command) ArgOrElse(key string, def string) string {
  ret, has := cmd.NamedArguments[key]
  if !has {
    return def
  }
  return ret
}

type Book struct {
  Title  string
  Author string
  Editor string
  Year   string
  Page   string
  Isbn   string
}

type Quote struct {
  Text   string
  Author string
  Book   Book
}

type BookQuote struct {
  Book  Book
  Quote Quote
}

func (bq *BookQuote) ToString() string {
  var out string
  out = fmt.Sprintf("Quote: \"%s\", by %s, from %s, page, %d published in %d", bq.Quote.Text, bq.Quote.Author, bq.Book.Title, bq.Book.Page, bq.Book.Year)
  return out
}

// Thank you andrew https://groups.google.com/d/msg/golang-nuts/FT7cjmcL7gw/Gj4_aEsE_IsJ
// A data structure to hold a key/value pair.
type Pair struct {
  Key   string
  Value int
}

// A slice of Pairs that implements sort.Interface to sort by Value.
type PairList []Pair

func (p PairList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p PairList) Len() int           { return len(p) }
func (p PairList) Less(i, j int) bool { return p[i].Value < p[j].Value }

// A function to turn a map into a PairList, then sort and return it.
func sortMapByValue(m map[string]int) PairList {
  p := make(PairList, len(m))
  i := 0
  for k, v := range m {
    p[i] = Pair{k, v}
    i += 1
  }
  sort.Sort(sort.Reverse(p))
  return p
}

func markupExtractor(body string) []Command {
  markup := regexp.MustCompile("(?s){{[^}]+}}")
  param := regexp.MustCompile("([^=]+)=(.*)")

  strCommands := markup.FindAllString(body, -1)

  commands := make([]Command, len(strCommands))
  pos := 0

  for _, cmd := range strCommands {
    cmd = cmd[2 : len(cmd)-2]

    arguments := make([]string, 10000)
    argumentsIndex := 0
    kvArguments := make(map[string]string, 1000)

    for _, arg := range strings.Split(cmd, "|") {
      kv := param.FindStringSubmatch(arg)
      if len(kv) == 3 {
        key := strings.TrimSpace(strings.ToLower(kv[1]))
        val, exists := kvArguments[key]
        if exists && val != kv[0] {
          // FIXME: handle the issue
          // panic(fmt.Sprintf("Parameter %s already exists with value \"%s\", here wants : \"%s\"", key, val, kv[0]))
        } else {
          kvArguments[key] = kv[2]
        }
      } else {
        arguments[argumentsIndex] = arg
        argumentsIndex += 1
      }
    }

    cmd := strings.TrimSpace(strings.ToLower(arguments[0]))

    // Parse special "defaultsort:", "if:", "msg:" commands
    if strings.Index(cmd, ":") != -1 {
      cmd = cmd[0:strings.Index(cmd, ":")]
      // FIXME inject arguments in Command anyway
    }

    // Ignore the empty command
    if cmd != "" {
      commands[pos] = Command{Cmd: cmd, Arguments: arguments[1:], NamedArguments: kvArguments}
      pos += 1
    }
  }
  return commands
}

func BuildQuote(qCommand Command, reference Command) {
  quote, hasQuote := qCommand.NamedArguments["citation"]
  if !hasQuote {
    quote = qCommand.Arguments[0]
    hasQuote = true
  }

  author := reference.ArgOrEmpty("auteur")
  title := reference.ArgOrEmpty("titre")
  page := reference.ArgOrEmpty("page")
  editor := reference.ArgOrEmpty("éditeur")
  year := reference.ArgOrEmpty("année")
  isbn := reference.ArgOrEmpty("isbn")

  if quote == "" || author == "" || title == "" || page == "" || editor == "" || year == "" || isbn == "" {
    fmt.Println("Some fields are missing")
    fmt.Println("", reference)
  }

}

func ExtractQuotes(commands []Command) {
  var buffer *Command = nil
  for ix, cmd := range commands {
    if len(cmd.Cmd) > 2 && cmd.Cmd[0:4] == "réf" {
      if buffer != nil {
        BuildQuote(*buffer, cmd)
      } else {
        fmt.Println("Found ref without quote")
        fmt.Println(cmd)
      }
      buffer = nil
    } else if strings.Index(cmd.Cmd, "citation") == 0 {
      buffer = &commands[ix]
    } else {
      fmt.Println("Unknown command:", cmd.Cmd)
      buffer = nil
    }
  }
}

func ExtractStats(commands []Command) {
  commandPopularity := make(map[string]int, 1000)
  argsPopularity := make(map[string]map[string]int, 100)

  var count int
  var hasCommand bool
  var aCount int
  var hasArg bool

  for _, cmd := range commands {
    count, hasCommand = commandPopularity[cmd.Cmd]
    if hasCommand {
      commandPopularity[cmd.Cmd] = count + 1
    } else {
      commandPopularity[cmd.Cmd] = 1
      argsPopularity[cmd.Cmd] = make(map[string]int, 100)
    }

    for k := range cmd.NamedArguments {
      aCount, hasArg = argsPopularity[cmd.Cmd][k]
      if hasArg {
        argsPopularity[cmd.Cmd][k] = aCount + 1
      } else {
        argsPopularity[cmd.Cmd][k] = 1
      }
    }
  }

  var maxArgsOcc int
  for _, pl := range sortMapByValue(commandPopularity) {
    fmt.Printf("### %s (%d occ.)\n", pl.Key, pl.Value)
    total := pl.Value
    maxArgsOcc = 0
    for _, ar := range sortMapByValue(argsPopularity[pl.Key]) {
      if ar.Value > maxArgsOcc {
        maxArgsOcc = ar.Value
      }
      if ar.Value > maxArgsOcc/10 && 100*ar.Value/total > 1 {
        fmt.Printf("- %s (%d%%)\n", ar.Key, 100*ar.Value/total)
      }
    }
  }
}

func main() {
  pageXPath := xmlpath.MustCompile("/mediawiki/page")
  textXPath := xmlpath.MustCompile(("revision/text"))
  fi, err := os.Open("frwikiquote-20140622-pages-articles-multistream.xml")
  // fi, err := os.Open("sample.xml")

  if err != nil {

    panic(err)
  }
  // close fi on exit and check for its returned error
  defer func() {
    if err := fi.Close(); err != nil {
      panic(err)
    }
  }()
  // make a read buffer
  r := bufio.NewReader(fi)

  root, err := xmlpath.Parse(r)
  if err != nil {
    log.Fatal(err)
  }
  commands := make([]Command, 0)
  iter := pageXPath.Iter(root)
  for iter.Next() {
    page := iter.Node()
    content, _ := textXPath.String(page)
    commands = append(commands, markupExtractor(content)...)
  }
  ExtractStats(commands)
}
<|endoftext|>


<commit_before>package goose

import (
	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
)

// Parser is an HTML parser specialised in extraction of main content and other properties
type Parser struct{}

// NewParser returns an HTML parser
func NewParser() *Parser {
	return &Parser{}
}

func (p Parser) dropTag(selection *goquery.Selection) {
	selection.Each(func(i int, s *goquery.Selection) {
		node := s.Get(0)
		node.Data = s.Text()
		node.Type = html.TextNode
	})
}

func (p Parser) indexOfAttribute(selection *goquery.Selection, attr string) int {
	node := selection.Get(0)
	for i, a := range node.Attr {
		if a.Key == attr {
			return i
		}
	}
	return -1
}

func (p Parser) delAttr(selection *goquery.Selection, attr string) {
	idx := p.indexOfAttribute(selection, attr)
	if idx > -1 {
		node := selection.Get(0)
		node.Attr = append(node.Attr[:idx], node.Attr[idx+1:]...)
	}
}

func (p Parser) getElementsByTags(div *goquery.Selection, tags []string) *goquery.Selection {
	selection := new(goquery.Selection)
	for _, tag := range tags {
		selections := div.Find(tag)
		if selections != nil {
			selection = selection.Union(selections)
		}
	}
	return selection
}

func (p Parser) clear(selection *goquery.Selection) {
	selection.Nodes = make([]*html.Node, 0)
}

func (p Parser) removeNode(selection *goquery.Selection) {
	if selection != nil {
		node := selection.Get(0)
		if node != nil && node.Parent != nil {
			node.Parent.RemoveChild(node)
		}
	}
}

func (p Parser) name(selector string, selection *goquery.Selection) string {
	value, exists := selection.Attr(selector)
	if exists {
		return value
	}
	return ""
}

func (p Parser) setAttr(selection *goquery.Selection, attr string, value string) {
	if selection.Size() > 0 {
		node := selection.Get(0)
		var attrs []html.Attribute
		for _, a := range node.Attr {
			if a.Key != attr {
				newAttr := new(html.Attribute)
				newAttr.Key = a.Key
				newAttr.Val = a.Val
				attrs = append(attrs, *newAttr)
			}
		}
		newAttr := new(html.Attribute)
		newAttr.Key = attr
		newAttr.Val = value
		attrs = append(attrs, *newAttr)
		node.Attr = attrs
	}
}
<commit_msg>fix dropTag - drop all html children nodes or the text of an <a> tag is printed twice<commit_after>package goose

import (
	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
)

// Parser is an HTML parser specialised in extraction of main content and other properties
type Parser struct{}

// NewParser returns an HTML parser
func NewParser() *Parser {
	return &Parser{}
}

func (p Parser) dropTag(selection *goquery.Selection) {
	selection.Each(func(i int, s *goquery.Selection) {
		node := s.Get(0)
		node.Data = s.Text()
		node.Attr = []html.Attribute{}
		node.DataAtom = 0
		node.Type = html.TextNode
		node.FirstChild = nil
		node.LastChild = nil
	})
}

func (p Parser) indexOfAttribute(selection *goquery.Selection, attr string) int {
	node := selection.Get(0)
	for i, a := range node.Attr {
		if a.Key == attr {
			return i
		}
	}
	return -1
}

func (p Parser) delAttr(selection *goquery.Selection, attr string) {
	idx := p.indexOfAttribute(selection, attr)
	if idx > -1 {
		node := selection.Get(0)
		node.Attr = append(node.Attr[:idx], node.Attr[idx+1:]...)
	}
}

func (p Parser) getElementsByTags(div *goquery.Selection, tags []string) *goquery.Selection {
	selection := new(goquery.Selection)
	for _, tag := range tags {
		selections := div.Find(tag)
		if selections != nil {
			selection = selection.Union(selections)
		}
	}
	return selection
}

func (p Parser) clear(selection *goquery.Selection) {
	selection.Nodes = make([]*html.Node, 0)
}

func (p Parser) removeNode(selection *goquery.Selection) {
	if selection != nil {
		node := selection.Get(0)
		if node != nil && node.Parent != nil {
			node.Parent.RemoveChild(node)
		}
	}
}

func (p Parser) name(selector string, selection *goquery.Selection) string {
	value, exists := selection.Attr(selector)
	if exists {
		return value
	}
	return ""
}

func (p Parser) setAttr(selection *goquery.Selection, attr string, value string) {
	if selection.Size() > 0 {
		node := selection.Get(0)
		var attrs []html.Attribute
		for _, a := range node.Attr {
			if a.Key != attr {
				newAttr := new(html.Attribute)
				newAttr.Key = a.Key
				newAttr.Val = a.Val
				attrs = append(attrs, *newAttr)
			}
		}
		newAttr := new(html.Attribute)
		newAttr.Key = attr
		newAttr.Val = value
		attrs = append(attrs, *newAttr)
		node.Attr = attrs
	}
}
<|endoftext|>


<commit_before>package css

import (
	"fmt"
	"github.com/gorilla/css/scanner"
	"strings"
)

type State int

const (
	STATE_NONE State = iota
	STATE_SELECTOR
	STATE_DECLARE_BLOCK
	STATE_PROPERTY
	STATE_VALUE
)

type ParserContext struct {
	State            State
	NowSelectorText  string
	NowProperty      string
	NowValue         string
	NowImportant     int
	NowRuleType      RuleType
	CurrentRule      *CSSRule
	CurrentMediaRule *CSSRule
}

func Parse(csstext string) *CSSStyleSheet {
	context := &ParserContext{
		State:            STATE_NONE,
		NowSelectorText:  "",
		NowProperty:      "",
		NowValue:         "",
		NowImportant:     0,
		NowRuleType:      STYLE_RULE,
		CurrentMediaRule: nil,
	}

	css := &CSSStyleSheet{}
	css.CssRuleList = make([]*CSSRule, 0)
	s := scanner.New(csstext)

	for {
		token := s.Next()

		fmt.Printf("%s:'%s'\n", token.Type.String(), token.Value)

		if token.Type == scanner.TokenEOF || token.Type == scanner.TokenError {
			break
		}

		switch token.Type {
		case scanner.TokenAtKeyword:
			context.State = STATE_SELECTOR
			switch token.Value {
			case "@media":
				context.NowRuleType = MEDIA_RULE
			}

		case scanner.TokenString:
			if context.State == STATE_SELECTOR {
				context.NowSelectorText += token.Value
			}
		case scanner.TokenURI:
		case scanner.TokenUnicodeRange:
		case scanner.TokenCDO:
		case scanner.TokenCDC:
		case scanner.TokenComment:
		case scanner.TokenIdent:

			if context.State == STATE_NONE || context.State == STATE_SELECTOR {
				context.State = STATE_SELECTOR
				context.NowSelectorText += strings.TrimSpace(token.Value)
				break
			}

			if context.State == STATE_DECLARE_BLOCK {
				context.State = STATE_PROPERTY
				context.NowProperty = strings.TrimSpace(token.Value)
				break
			}

			if context.State == STATE_VALUE {
				if token.Value == "important" {
					context.NowImportant = 1
				} else {
					context.NowValue += token.Value
				}
				break
			}

		case scanner.TokenDimension:
			fallthrough
		case scanner.TokenS:
			if context.State == STATE_SELECTOR {
				context.NowSelectorText += token.Value
				break
			}
			if context.State == STATE_VALUE {
				context.NowValue += token.Value
				break
			}
		case scanner.TokenChar:
			if context.State == STATE_NONE {
				if token.Value == "}" && context.CurrentMediaRule != nil {
					// close media rule
					css.CssRuleList = append(css.CssRuleList, context.CurrentMediaRule)
					context.CurrentMediaRule = nil
					break
				}
				if token.Value != "{" {
					context.State = STATE_SELECTOR
					context.NowSelectorText += token.Value
					break
				}
			}
			if context.State == STATE_SELECTOR {
				if "{" == token.Value {
					if context.NowRuleType == MEDIA_RULE {
						context.CurrentMediaRule = NewRule(context.NowRuleType)
						context.CurrentMediaRule.Style.SelectorText = strings.TrimSpace(context.NowSelectorText)
						// reset
						context.NowSelectorText = ""
						context.NowProperty = ""
						context.NowValue = ""
						context.NowImportant = 0
						context.NowRuleType = STYLE_RULE
						context.State = STATE_NONE
					} else {
						context.State = STATE_DECLARE_BLOCK
						context.CurrentRule = NewRule(context.NowRuleType)
						context.CurrentRule.Style.SelectorText = strings.TrimSpace(context.NowSelectorText)
					}
					break
				} else {
					context.NowSelectorText += token.Value
				}
				break
			}

			if context.State == STATE_PROPERTY {
				if token.Value == ":" {
					context.State = STATE_VALUE
				}
				break
			}
			if context.State == STATE_DECLARE_BLOCK {
				if token.Value == "}" {
					if context.CurrentMediaRule != nil {
						context.CurrentMediaRule.Rules = append(context.CurrentMediaRule.Rules, context.CurrentRule)
					} else {
						css.CssRuleList = append(css.CssRuleList, context.CurrentRule)
					}
					context.NowSelectorText = ""
					context.NowProperty = ""
					context.NowValue = ""
					context.NowImportant = 0
					context.NowRuleType = STYLE_RULE
					context.State = STATE_NONE

				}
				break
			}

			if context.State == STATE_VALUE {
				if token.Value == ";" {
					decl := NewCSSStyleDeclaration(context.NowProperty, strings.TrimSpace(context.NowValue), context.NowImportant)
					context.CurrentRule.Style.Styles[context.NowProperty] = decl

					context.NowProperty = ""
					context.NowValue = ""
					context.NowImportant = 0
					context.State = STATE_DECLARE_BLOCK
				} else if token.Value == "}" { // last property in a block can have optional ;
					decl := NewCSSStyleDeclaration(context.NowProperty, strings.TrimSpace(context.NowValue), context.NowImportant)
					context.CurrentRule.Style.Styles[context.NowProperty] = decl
					if context.CurrentMediaRule != nil {
						context.CurrentMediaRule.Rules = append(context.CurrentMediaRule.Rules, context.CurrentRule)
					} else {
						css.CssRuleList = append(css.CssRuleList, context.CurrentRule)
					}
					context.NowSelectorText = ""
					context.NowProperty = ""
					context.NowValue = ""
					context.NowImportant = 0
					context.NowRuleType = STYLE_RULE
					context.State = STATE_NONE
				} else if token.Value != "!" {
					context.NowValue += token.Value
				}
				break

			}
		case scanner.TokenPercentage:
			fallthrough
		case scanner.TokenHash:
			if context.State == STATE_NONE || context.State == STATE_SELECTOR {
				context.State = STATE_SELECTOR
				context.NowSelectorText += strings.TrimSpace(token.Value)
				break
			}

			fallthrough
		case scanner.TokenNumber:
			fallthrough
		case scanner.TokenFunction:
			fallthrough
		case scanner.TokenIncludes:
			fallthrough
		case scanner.TokenDashMatch:
			fallthrough
		case scanner.TokenPrefixMatch:
			fallthrough
		case scanner.TokenSuffixMatch:
			fallthrough
		case scanner.TokenSubstringMatch:
			if context.State == STATE_VALUE {
				context.NowValue += token.Value
				break
			}
			if context.State == STATE_SELECTOR {
				context.NowSelectorText += token.Value
				break
			}
		default:
			fmt.Printf("Unhandled, %s:'%s'\n", token.Type.String(), token.Value)
		}
	}
	return css
}
<commit_msg>Adding css syntax and rearrange token cases<commit_after>package css

import (
	"fmt"
	"github.com/gorilla/css/scanner"
	"strings"
)

/*
	stylesheet  : [ CDO | CDC | S | statement ]*;
	statement   : ruleset | at-rule;
	at-rule     : ATKEYWORD S* any* [ block | ';' S* ];
	block       : '{' S* [ any | block | ATKEYWORD S* | ';' S* ]* '}' S*;
	ruleset     : selector? '{' S* declaration? [ ';' S* declaration? ]* '}' S*;
	selector    : any+;
	declaration : property S* ':' S* value;
	property    : IDENT;
	value       : [ any | block | ATKEYWORD S* ]+;
	any         : [ IDENT | NUMBER | PERCENTAGE | DIMENSION | STRING
	              | DELIM | URI | HASH | UNICODE-RANGE | INCLUDES
	              | DASHMATCH | ':' | FUNCTION S* [any|unused]* ')'
	              | '(' S* [any|unused]* ')' | '[' S* [any|unused]* ']'
	              ] S*;
	unused      : block | ATKEYWORD S* | ';' S* | CDO S* | CDC S*;
*/

type State int

const (
	STATE_NONE State = iota
	STATE_SELECTOR
	STATE_DECLARE_BLOCK
	STATE_PROPERTY
	STATE_VALUE
)

type ParserContext struct {
	State            State
	NowSelectorText  string
	NowProperty      string
	NowValue         string
	NowImportant     int
	NowRuleType      RuleType
	CurrentRule      *CSSRule
	CurrentMediaRule *CSSRule
}

func Parse(csstext string) *CSSStyleSheet {
	context := &ParserContext{
		State:            STATE_NONE,
		NowSelectorText:  "",
		NowProperty:      "",
		NowValue:         "",
		NowImportant:     0,
		NowRuleType:      STYLE_RULE,
		CurrentMediaRule: nil,
	}

	css := &CSSStyleSheet{}
	css.CssRuleList = make([]*CSSRule, 0)
	s := scanner.New(csstext)

	for {
		token := s.Next()

		fmt.Printf("%s:'%s'\n", token.Type.String(), token.Value)

		if token.Type == scanner.TokenEOF || token.Type == scanner.TokenError {
			break
		}

		switch token.Type {
		case scanner.TokenAtKeyword:
			context.State = STATE_SELECTOR
			switch token.Value {
			case "@media":
				context.NowRuleType = MEDIA_RULE
			}

		case scanner.TokenString:
			if context.State == STATE_SELECTOR {
				context.NowSelectorText += token.Value
			}
		case scanner.TokenIdent:

			if context.State == STATE_NONE || context.State == STATE_SELECTOR {
				context.State = STATE_SELECTOR
				context.NowSelectorText += strings.TrimSpace(token.Value)
				break
			}

			if context.State == STATE_DECLARE_BLOCK {
				context.State = STATE_PROPERTY
				context.NowProperty = strings.TrimSpace(token.Value)
				break
			}

			if context.State == STATE_VALUE {
				if token.Value == "important" {
					context.NowImportant = 1
				} else {
					context.NowValue += token.Value
				}
				break
			}

		case scanner.TokenDimension:
			fallthrough
		case scanner.TokenS:
			if context.State == STATE_SELECTOR {
				context.NowSelectorText += token.Value
				break
			}
			if context.State == STATE_VALUE {
				context.NowValue += token.Value
				break
			}
		case scanner.TokenChar:
			if context.State == STATE_NONE {
				if token.Value == "}" && context.CurrentMediaRule != nil {
					// close media rule
					css.CssRuleList = append(css.CssRuleList, context.CurrentMediaRule)
					context.CurrentMediaRule = nil
					break
				}
				if token.Value != "{" {
					context.State = STATE_SELECTOR
					context.NowSelectorText += token.Value
					break
				}
			}
			if context.State == STATE_SELECTOR {
				if "{" == token.Value {
					if context.NowRuleType == MEDIA_RULE {
						context.CurrentMediaRule = NewRule(context.NowRuleType)
						context.CurrentMediaRule.Style.SelectorText = strings.TrimSpace(context.NowSelectorText)
						// reset
						context.NowSelectorText = ""
						context.NowProperty = ""
						context.NowValue = ""
						context.NowImportant = 0
						context.NowRuleType = STYLE_RULE
						context.State = STATE_NONE
					} else {
						context.State = STATE_DECLARE_BLOCK
						context.CurrentRule = NewRule(context.NowRuleType)
						context.CurrentRule.Style.SelectorText = strings.TrimSpace(context.NowSelectorText)
					}
					break
				} else {
					context.NowSelectorText += token.Value
				}
				break
			}

			if context.State == STATE_PROPERTY {
				if token.Value == ":" {
					context.State = STATE_VALUE
				}
				break
			}
			if context.State == STATE_DECLARE_BLOCK {
				if token.Value == "}" {
					if context.CurrentMediaRule != nil {
						context.CurrentMediaRule.Rules = append(context.CurrentMediaRule.Rules, context.CurrentRule)
					} else {
						css.CssRuleList = append(css.CssRuleList, context.CurrentRule)
					}
					context.NowSelectorText = ""
					context.NowProperty = ""
					context.NowValue = ""
					context.NowImportant = 0
					context.NowRuleType = STYLE_RULE
					context.State = STATE_NONE

				}
				break
			}

			if context.State == STATE_VALUE {
				if token.Value == ";" {
					decl := NewCSSStyleDeclaration(context.NowProperty, strings.TrimSpace(context.NowValue), context.NowImportant)
					context.CurrentRule.Style.Styles[context.NowProperty] = decl

					context.NowProperty = ""
					context.NowValue = ""
					context.NowImportant = 0
					context.State = STATE_DECLARE_BLOCK
				} else if token.Value == "}" { // last property in a block can have optional ;
					decl := NewCSSStyleDeclaration(context.NowProperty, strings.TrimSpace(context.NowValue), context.NowImportant)
					context.CurrentRule.Style.Styles[context.NowProperty] = decl
					if context.CurrentMediaRule != nil {
						context.CurrentMediaRule.Rules = append(context.CurrentMediaRule.Rules, context.CurrentRule)
					} else {
						css.CssRuleList = append(css.CssRuleList, context.CurrentRule)
					}
					context.NowSelectorText = ""
					context.NowProperty = ""
					context.NowValue = ""
					context.NowImportant = 0
					context.NowRuleType = STYLE_RULE
					context.State = STATE_NONE
				} else if token.Value != "!" {
					context.NowValue += token.Value
				}
				break

			}
		case scanner.TokenPercentage:
			fallthrough
		case scanner.TokenHash:
			if context.State == STATE_NONE || context.State == STATE_SELECTOR {
				context.State = STATE_SELECTOR
				context.NowSelectorText += strings.TrimSpace(token.Value)
				break
			}

			fallthrough
		case scanner.TokenNumber:
			fallthrough
		case scanner.TokenFunction:
			fallthrough
		case scanner.TokenIncludes:
			fallthrough
		case scanner.TokenDashMatch:
			fallthrough
		case scanner.TokenPrefixMatch:
			fallthrough
		case scanner.TokenSuffixMatch:
			fallthrough
		case scanner.TokenSubstringMatch:
			if context.State == STATE_VALUE {
				context.NowValue += token.Value
				break
			}
			if context.State == STATE_SELECTOR {
				context.NowSelectorText += token.Value
				break
			}

		// Unhandle token
		case scanner.TokenURI:
			fallthrough
		case scanner.TokenUnicodeRange:
			fallthrough
		case scanner.TokenCDO:
			fallthrough
		case scanner.TokenCDC:
			fallthrough
		case scanner.TokenComment:
			fallthrough
		default:
			fmt.Printf("Unhandled, %s:'%s'\n", token.Type.String(), token.Value)
		}
	}
	return css
}
<|endoftext|>


<commit_before>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package constraints

import (
	//"fmt"
	//"github.com/golang/protobuf/proto"
	//mesos "github.com/mesos/mesos-go/mesosproto"
	//"strings"
	"fmt"
	. "github.com/smartystreets/goconvey/convey"
	"testing"
)

func TestConstraintParse(t *testing.T) {
	Convey("Like constraint", t, func() {
		Convey("Should parse on valid input", func() {
			constraint, err := ParseConstraint([]string{"LIKE", "1"})
			So(err, ShouldBeNil)
			like, ok := constraint.(*Like)
			So(ok, ShouldBeTrue)
			So(like.regex, ShouldEqual, "1")
		})
	})

	Convey("Unlike constraint", t, func() {
		Convey("Should parse on valid input", func() {
			constraint, err := ParseConstraint([]string{"UNLIKE", "1"})
			So(err, ShouldBeNil)
			unlike, ok := constraint.(*Unlike)
			So(ok, ShouldBeTrue)
			So(unlike.regex, ShouldEqual, "1")
		})
	})

	Convey("Unique constraint", t, func() {
		Convey("Should parse on valid input", func() {
			constraint, err := ParseConstraint([]string{"UNIQUE"})
			So(err, ShouldBeNil)
			_, ok := constraint.(*Unique)
			So(ok, ShouldBeTrue)
		})
	})

	Convey("Cluster constraint", t, func() {
		Convey("Should parse on empty condition", func() {
			constraint, err := ParseConstraint([]string{"CLUSTER"})
			So(err, ShouldBeNil)
			cluster, ok := constraint.(*Cluster)
			So(ok, ShouldBeTrue)
			So(cluster.value, ShouldBeEmpty)
		})

		Convey("Should parse on non-empty condition", func() {
			constraint, err := ParseConstraint([]string{"CLUSTER", "123"})
			So(err, ShouldBeNil)
			cluster123, ok := constraint.(*Cluster)
			So(ok, ShouldBeTrue)
			So(cluster123.value, ShouldEqual, "123")
		})
	})

	Convey("GroupBy constraint", t, func() {
		Convey("Should parse on empty condition", func() {
			constraint, err := ParseConstraint([]string{"GROUP_BY"})
			So(err, ShouldBeNil)
			groupBy, ok := constraint.(*GroupBy)
			So(ok, ShouldBeTrue)
			So(groupBy.groups, ShouldEqual, 1)
		})

		Convey("Should parse on non-empty condition", func() {
			constraint, err := ParseConstraint([]string{"GROUP_BY", "3"})
			So(err, ShouldBeNil)
			groupBy3, ok := constraint.(*GroupBy)
			So(ok, ShouldBeTrue)
			So(groupBy3.groups, ShouldEqual, 3)
		})
	})

	Convey("Invalid constraint", t, func() {
		Convey("Should not parse", func() {
			constraint, err := ParseConstraint([]string{"unsupported"})
			So(err, ShouldNotBeNil)
			So(constraint, ShouldBeNil)
			So(err.Error(), ShouldContainSubstring, "Unsupported constraint")
		})
	})
}

func TestConstraintMatches(t *testing.T) {
	Convey("Constraints should match", t, func() {
		So(MustParseConstraint([]string{"LIKE", "^abc$"}).Matches("abc", nil), ShouldBeTrue)
		So(MustParseConstraint([]string{"LIKE", "^abc$"}).Matches("abc1", nil), ShouldBeFalse)

		So(MustParseConstraint([]string{"LIKE", "a.*"}).Matches("abc", nil), ShouldBeTrue)
		So(MustParseConstraint([]string{"LIKE", "a.*"}).Matches("bc", nil), ShouldBeFalse)

		So(MustParseConstraint([]string{"UNIQUE"}).Matches("a", nil), ShouldBeTrue)
		So(MustParseConstraint([]string{"UNIQUE"}).Matches("a", []string{"a"}), ShouldBeFalse)

		So(MustParseConstraint([]string{"CLUSTER"}).Matches("a", nil), ShouldBeTrue)
		So(MustParseConstraint([]string{"CLUSTER"}).Matches("a", []string{"b"}), ShouldBeFalse)

		So(MustParseConstraint([]string{"GROUP_BY"}).Matches("a", []string{"a"}), ShouldBeTrue)
		So(MustParseConstraint([]string{"GROUP_BY"}).Matches("a", []string{"b"}), ShouldBeFalse)
	})
}

func TestConstraintString(t *testing.T) {
	Convey("Constraints should be readable strings", t, func() {
		So(fmt.Sprintf("%s", MustParseConstraint([]string{"LIKE", "abc"})), ShouldEqual, "like:abc")
		So(fmt.Sprintf("%s", MustParseConstraint([]string{"UNLIKE", "abc"})), ShouldEqual, "unlike:abc")
		So(fmt.Sprintf("%s", MustParseConstraint([]string{"UNIQUE"})), ShouldEqual, "unique")
		So(fmt.Sprintf("%s", MustParseConstraint([]string{"CLUSTER"})), ShouldEqual, "cluster")
		So(fmt.Sprintf("%s", MustParseConstraint([]string{"CLUSTER", "123"})), ShouldEqual, "cluster:123")
		So(fmt.Sprintf("%s", MustParseConstraint([]string{"GROUP_BY"})), ShouldEqual, "groupBy")
		So(fmt.Sprintf("%s", MustParseConstraint([]string{"GROUP_BY", "2"})), ShouldEqual, "groupBy:2")
	})
}

func TestMatches(t *testing.T) {
	Convey("Constraints should match properly", t, func() {
		Convey("LIKE", func() {
			like := MustParseConstraint([]string{"LIKE", "^1.*2$"})
			So(like.Matches("12", nil), ShouldBeTrue)
			So(like.Matches("1a2", nil), ShouldBeTrue)
			So(like.Matches("1ab2", nil), ShouldBeTrue)

			So(like.Matches("a1a2", nil), ShouldBeFalse)
			So(like.Matches("1a2a", nil), ShouldBeFalse)
		})

		Convey("UNLIKE", func() {
			unlike := MustParseConstraint([]string{"UNLIKE", "1"})
			So(unlike.Matches("1", nil), ShouldBeFalse)
			So(unlike.Matches("2", nil), ShouldBeTrue)
		})

		Convey("UNIQUE", func() {
			unique := MustParseConstraint([]string{"UNIQUE"})
			So(unique.Matches("1", nil), ShouldBeTrue)
			So(unique.Matches("2", []string{"1"}), ShouldBeTrue)
			So(unique.Matches("3", []string{"1", "2"}), ShouldBeTrue)

			So(unique.Matches("1", []string{"1", "2"}), ShouldBeFalse)
			So(unique.Matches("2", []string{"1", "2"}), ShouldBeFalse)
		})

		Convey("CLUSTER", func() {
			cluster := MustParseConstraint([]string{"CLUSTER"})
			So(cluster.Matches("1", nil), ShouldBeTrue)
			So(cluster.Matches("2", nil), ShouldBeTrue)

			So(cluster.Matches("1", []string{"1"}), ShouldBeTrue)
			So(cluster.Matches("1", []string{"1", "1"}), ShouldBeTrue)
			So(cluster.Matches("2", []string{"1"}), ShouldBeFalse)

			cluster3 := MustParseConstraint([]string{"CLUSTER", "3"})
			So(cluster3.Matches("3", nil), ShouldBeTrue)
			So(cluster3.Matches("2", nil), ShouldBeFalse)

			So(cluster3.Matches("3", []string{"3"}), ShouldBeTrue)
			So(cluster3.Matches("3", []string{"3", "3"}), ShouldBeTrue)
			So(cluster3.Matches("2", []string{"3"}), ShouldBeFalse)
		})

		Convey("GROUP_BY", func() {
			groupBy := MustParseConstraint([]string{"GROUP_BY"})
			So(groupBy.Matches("1", nil), ShouldBeTrue)
			So(groupBy.Matches("1", []string{"1"}), ShouldBeTrue)
			So(groupBy.Matches("1", []string{"1", "1"}), ShouldBeTrue)
			So(groupBy.Matches("1", []string{"2"}), ShouldBeFalse)

			groupBy2 := MustParseConstraint([]string{"GROUP_BY", "2"})
			So(groupBy2.Matches("1", nil), ShouldBeTrue)
			So(groupBy2.Matches("1", []string{"1"}), ShouldBeFalse)
			So(groupBy2.Matches("1", []string{"1", "1"}), ShouldBeFalse)
			So(groupBy2.Matches("2", []string{"1"}), ShouldBeTrue)

			So(groupBy2.Matches("1", []string{"1", "2"}), ShouldBeTrue)
			So(groupBy2.Matches("2", []string{"1", "2"}), ShouldBeTrue)

			So(groupBy2.Matches("1", []string{"1", "1", "2"}), ShouldBeFalse)
			So(groupBy2.Matches("2", []string{"1", "1", "2"}), ShouldBeTrue)
		})
	})
}

//func TestMatchesAttributes(t *testing.T) {
//	task1 := &TestConstrained{
//		constraints: map[string][]Constraint{
//			"rack": []Constraint{MustParseConstraint("like:^1-.*")},
//		},
//		attributes: map[string]string{
//			"rack": "1-1",
//		},
//	}
//
//	offer := &mesos.Offer{
//		Attributes: []*mesos.Attribute{
//			&mesos.Attribute{
//				Name: proto.String("rack"),
//				Text: &mesos.Value_Text{
//					Value: proto.String("1-1"),
//				},
//			},
//		},
//	}
//
//	assert(t, CheckConstraints(offer, task1.Constraints(), []Constrained{task1}), "")
//
//	offer.Attributes[0].Text.Value = proto.String("2-1")
//	assert(t, CheckConstraints(offer, task1.Constraints(), []Constrained{task1}), "rack doesn't match like:^1-.*")
//
//	task2 := &TestConstrained{
//		constraints: map[string][]Constraint{
//			"floor": []Constraint{MustParseConstraint("unique")},
//		},
//		attributes: map[string]string{
//			"rack":  "1-1",
//			"floor": "1",
//		},
//	}
//
//	offer.Attributes[0].Name = proto.String("floor")
//	offer.Attributes[0].Text.Value = proto.String("1")
//	assert(t, CheckConstraints(offer, task2.Constraints(), []Constrained{task2}), "floor doesn't match unique")
//}
//
//type TestConstrained struct {
//	constraints map[string][]Constraint
//	attributes  map[string]string
//}
//
//func (tc *TestConstrained) Constraints() map[string][]Constraint {
//	return tc.constraints
//}
//
//func (tc *TestConstrained) Attribute(name string) string {
//	return tc.attributes[name]
//}
<commit_msg>Removed old commented code<commit_after>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package constraints

import (
	"fmt"
	. "github.com/smartystreets/goconvey/convey"
	"testing"
)

func TestConstraintParse(t *testing.T) {
	Convey("Like constraint", t, func() {
		Convey("Should parse on valid input", func() {
			constraint, err := ParseConstraint([]string{"LIKE", "1"})
			So(err, ShouldBeNil)
			like, ok := constraint.(*Like)
			So(ok, ShouldBeTrue)
			So(like.regex, ShouldEqual, "1")
		})
	})

	Convey("Unlike constraint", t, func() {
		Convey("Should parse on valid input", func() {
			constraint, err := ParseConstraint([]string{"UNLIKE", "1"})
			So(err, ShouldBeNil)
			unlike, ok := constraint.(*Unlike)
			So(ok, ShouldBeTrue)
			So(unlike.regex, ShouldEqual, "1")
		})
	})

	Convey("Unique constraint", t, func() {
		Convey("Should parse on valid input", func() {
			constraint, err := ParseConstraint([]string{"UNIQUE"})
			So(err, ShouldBeNil)
			_, ok := constraint.(*Unique)
			So(ok, ShouldBeTrue)
		})
	})

	Convey("Cluster constraint", t, func() {
		Convey("Should parse on empty condition", func() {
			constraint, err := ParseConstraint([]string{"CLUSTER"})
			So(err, ShouldBeNil)
			cluster, ok := constraint.(*Cluster)
			So(ok, ShouldBeTrue)
			So(cluster.value, ShouldBeEmpty)
		})

		Convey("Should parse on non-empty condition", func() {
			constraint, err := ParseConstraint([]string{"CLUSTER", "123"})
			So(err, ShouldBeNil)
			cluster123, ok := constraint.(*Cluster)
			So(ok, ShouldBeTrue)
			So(cluster123.value, ShouldEqual, "123")
		})
	})

	Convey("GroupBy constraint", t, func() {
		Convey("Should parse on empty condition", func() {
			constraint, err := ParseConstraint([]string{"GROUP_BY"})
			So(err, ShouldBeNil)
			groupBy, ok := constraint.(*GroupBy)
			So(ok, ShouldBeTrue)
			So(groupBy.groups, ShouldEqual, 1)
		})

		Convey("Should parse on non-empty condition", func() {
			constraint, err := ParseConstraint([]string{"GROUP_BY", "3"})
			So(err, ShouldBeNil)
			groupBy3, ok := constraint.(*GroupBy)
			So(ok, ShouldBeTrue)
			So(groupBy3.groups, ShouldEqual, 3)
		})
	})

	Convey("Invalid constraint", t, func() {
		Convey("Should not parse", func() {
			constraint, err := ParseConstraint([]string{"unsupported"})
			So(err, ShouldNotBeNil)
			So(constraint, ShouldBeNil)
			So(err.Error(), ShouldContainSubstring, "Unsupported constraint")
		})
	})
}

func TestConstraintMatches(t *testing.T) {
	Convey("Constraints should match", t, func() {
		So(MustParseConstraint([]string{"LIKE", "^abc$"}).Matches("abc", nil), ShouldBeTrue)
		So(MustParseConstraint([]string{"LIKE", "^abc$"}).Matches("abc1", nil), ShouldBeFalse)

		So(MustParseConstraint([]string{"LIKE", "a.*"}).Matches("abc", nil), ShouldBeTrue)
		So(MustParseConstraint([]string{"LIKE", "a.*"}).Matches("bc", nil), ShouldBeFalse)

		So(MustParseConstraint([]string{"UNIQUE"}).Matches("a", nil), ShouldBeTrue)
		So(MustParseConstraint([]string{"UNIQUE"}).Matches("a", []string{"a"}), ShouldBeFalse)

		So(MustParseConstraint([]string{"CLUSTER"}).Matches("a", nil), ShouldBeTrue)
		So(MustParseConstraint([]string{"CLUSTER"}).Matches("a", []string{"b"}), ShouldBeFalse)

		So(MustParseConstraint([]string{"GROUP_BY"}).Matches("a", []string{"a"}), ShouldBeTrue)
		So(MustParseConstraint([]string{"GROUP_BY"}).Matches("a", []string{"b"}), ShouldBeFalse)
	})
}

func TestConstraintString(t *testing.T) {
	Convey("Constraints should be readable strings", t, func() {
		So(fmt.Sprintf("%s", MustParseConstraint([]string{"LIKE", "abc"})), ShouldEqual, "like:abc")
		So(fmt.Sprintf("%s", MustParseConstraint([]string{"UNLIKE", "abc"})), ShouldEqual, "unlike:abc")
		So(fmt.Sprintf("%s", MustParseConstraint([]string{"UNIQUE"})), ShouldEqual, "unique")
		So(fmt.Sprintf("%s", MustParseConstraint([]string{"CLUSTER"})), ShouldEqual, "cluster")
		So(fmt.Sprintf("%s", MustParseConstraint([]string{"CLUSTER", "123"})), ShouldEqual, "cluster:123")
		So(fmt.Sprintf("%s", MustParseConstraint([]string{"GROUP_BY"})), ShouldEqual, "groupBy")
		So(fmt.Sprintf("%s", MustParseConstraint([]string{"GROUP_BY", "2"})), ShouldEqual, "groupBy:2")
	})
}

func TestMatches(t *testing.T) {
	Convey("Constraints should match properly", t, func() {
		Convey("LIKE", func() {
			like := MustParseConstraint([]string{"LIKE", "^1.*2$"})
			So(like.Matches("12", nil), ShouldBeTrue)
			So(like.Matches("1a2", nil), ShouldBeTrue)
			So(like.Matches("1ab2", nil), ShouldBeTrue)

			So(like.Matches("a1a2", nil), ShouldBeFalse)
			So(like.Matches("1a2a", nil), ShouldBeFalse)
		})

		Convey("UNLIKE", func() {
			unlike := MustParseConstraint([]string{"UNLIKE", "1"})
			So(unlike.Matches("1", nil), ShouldBeFalse)
			So(unlike.Matches("2", nil), ShouldBeTrue)
		})

		Convey("UNIQUE", func() {
			unique := MustParseConstraint([]string{"UNIQUE"})
			So(unique.Matches("1", nil), ShouldBeTrue)
			So(unique.Matches("2", []string{"1"}), ShouldBeTrue)
			So(unique.Matches("3", []string{"1", "2"}), ShouldBeTrue)

			So(unique.Matches("1", []string{"1", "2"}), ShouldBeFalse)
			So(unique.Matches("2", []string{"1", "2"}), ShouldBeFalse)
		})

		Convey("CLUSTER", func() {
			cluster := MustParseConstraint([]string{"CLUSTER"})
			So(cluster.Matches("1", nil), ShouldBeTrue)
			So(cluster.Matches("2", nil), ShouldBeTrue)

			So(cluster.Matches("1", []string{"1"}), ShouldBeTrue)
			So(cluster.Matches("1", []string{"1", "1"}), ShouldBeTrue)
			So(cluster.Matches("2", []string{"1"}), ShouldBeFalse)

			cluster3 := MustParseConstraint([]string{"CLUSTER", "3"})
			So(cluster3.Matches("3", nil), ShouldBeTrue)
			So(cluster3.Matches("2", nil), ShouldBeFalse)

			So(cluster3.Matches("3", []string{"3"}), ShouldBeTrue)
			So(cluster3.Matches("3", []string{"3", "3"}), ShouldBeTrue)
			So(cluster3.Matches("2", []string{"3"}), ShouldBeFalse)
		})

		Convey("GROUP_BY", func() {
			groupBy := MustParseConstraint([]string{"GROUP_BY"})
			So(groupBy.Matches("1", nil), ShouldBeTrue)
			So(groupBy.Matches("1", []string{"1"}), ShouldBeTrue)
			So(groupBy.Matches("1", []string{"1", "1"}), ShouldBeTrue)
			So(groupBy.Matches("1", []string{"2"}), ShouldBeFalse)

			groupBy2 := MustParseConstraint([]string{"GROUP_BY", "2"})
			So(groupBy2.Matches("1", nil), ShouldBeTrue)
			So(groupBy2.Matches("1", []string{"1"}), ShouldBeFalse)
			So(groupBy2.Matches("1", []string{"1", "1"}), ShouldBeFalse)
			So(groupBy2.Matches("2", []string{"1"}), ShouldBeTrue)

			So(groupBy2.Matches("1", []string{"1", "2"}), ShouldBeTrue)
			So(groupBy2.Matches("2", []string{"1", "2"}), ShouldBeTrue)

			So(groupBy2.Matches("1", []string{"1", "1", "2"}), ShouldBeFalse)
			So(groupBy2.Matches("2", []string{"1", "1", "2"}), ShouldBeTrue)
		})
	})
}
<|endoftext|>


<commit_before>package consumergroup

import (
	"errors"
	"fmt"
	"sync"
	"time"
)

// OffsetManager is the main interface consumergroup requires to manage offsets of the consumergroup.
type OffsetManager interface {

	// InitializePartition is called when the consumergroup is starting to consume a
	// partition. It should return the last processed offset for this partition. Note:
	// the same partition can be initialized multiple times during a single run of a
	// consumer group due to other consumer instances coming online and offline.
	InitializePartition(topic string, partition int32) (int64, error)

	// Get next offset.
	GetNextOffset(topic string, partition int32) (int64, error)

	// Force offsetManager to commit all messages already stored.
	CommitOffsets() error

	// MarkAsProcessed tells the offset manager than a certain message has been successfully
	// processed by the consumer, and should be committed. The implementation does not have
	// to store this offset right away, but should return true if it intends to do this at
	// some point.
	//
	// Offsets should generally be increasing if the consumer
	// processes events serially, but this cannot be guaranteed if the consumer does any
	// asynchronous processing. This can be handled in various ways, e.g. by only accepting
	// offsets that are higehr than the offsets seen before for the same partition.
	MarkAsProcessed(topic string, partition int32, offset int64) bool

	// FinalizePartition is called when the consumergroup is done consuming a
	// partition. In this method, the offset manager can flush any remaining offsets to its
	// backend store. It should return an error if it was not able to commit the offset.
	// Note: it's possible that the consumergroup instance will start to consume the same
	// partition again after this function is called.
	FinalizePartition(topic string, partition int32, lastOffset int64, timeout time.Duration) error

	// Close is called when the consumergroup is shutting down. In normal circumstances, all
	// offsets are committed because FinalizePartition is called for all the running partition
	// consumers. You may want to check for this to be true, and try to commit any outstanding
	// offsets. If this doesn't succeed, it should return an error.
	Close() error
}

var (
	UncleanClose = errors.New("Not all offsets were committed before shutdown was completed")
)

// OffsetManagerConfig holds configuration setting son how the offset manager should behave.
type OffsetManagerConfig struct {
	CommitInterval time.Duration // Interval between offset flushes to the backend store.
	VerboseLogging bool          // Whether to enable verbose logging.
	EnableAutoCommit bool        // Whether to enable auto commit.
}

// NewOffsetManagerConfig returns a new OffsetManagerConfig with sane defaults.
func NewOffsetManagerConfig() *OffsetManagerConfig {
	return &OffsetManagerConfig{
		CommitInterval: 10 * time.Second,
	}
}

type (
	topicOffsets    map[int32]*partitionOffsetTracker
	offsetsMap      map[string]topicOffsets
	offsetCommitter func(int64) error
)

type partitionOffsetTracker struct {
	l                      sync.Mutex
	waitingForOffset       int64
	highestProcessedOffset int64
	lastCommittedOffset    int64
	done                   chan struct{}
}

type zookeeperOffsetManager struct {
	config  *OffsetManagerConfig
	l       sync.RWMutex
	offsets offsetsMap
	cg      *ConsumerGroup

	closing, closed chan struct{}
}

// NewZookeeperOffsetManager returns an offset manager that uses Zookeeper
// to store offsets.
func NewZookeeperOffsetManager(cg *ConsumerGroup, config *OffsetManagerConfig) OffsetManager {
	if config == nil {
		config = NewOffsetManagerConfig()
	}

	zom := &zookeeperOffsetManager{
		config:  config,
		cg:      cg,
		offsets: make(offsetsMap),
		closing: make(chan struct{}),
		closed:  make(chan struct{}),
	}

	if config.EnableAutoCommit {
		go zom.offsetCommitter()
	}

	return zom
}

func (zom *zookeeperOffsetManager) InitializePartition(topic string, partition int32) (int64, error) {
	zom.l.Lock()
	defer zom.l.Unlock()

	if zom.offsets[topic] == nil {
		zom.offsets[topic] = make(topicOffsets)
	}

	nextOffset, err := zom.cg.group.FetchOffset(topic, partition)
	if err != nil {
		return 0, err
	}

	zom.offsets[topic][partition] = &partitionOffsetTracker{
		highestProcessedOffset: nextOffset - 1,
		lastCommittedOffset:    nextOffset - 1,
		done:                   make(chan struct{}),
	}

	return nextOffset, nil
}

func (zom *zookeeperOffsetManager) FinalizePartition(topic string, partition int32, lastOffset int64, timeout time.Duration) error {
	zom.l.RLock()
	tracker := zom.offsets[topic][partition]
	zom.l.RUnlock()

	if lastOffset >= 0 {
		if lastOffset-tracker.highestProcessedOffset > 0 {
			zom.cg.Logf("%s/%d :: Last processed offset: %d. Waiting up to %ds for another %d messages to process...", topic, partition, tracker.highestProcessedOffset, timeout/time.Second, lastOffset-tracker.highestProcessedOffset)
			if !tracker.waitForOffset(lastOffset, timeout) {
				return fmt.Errorf("TIMEOUT waiting for offset %d. Last committed offset: %d", lastOffset, tracker.lastCommittedOffset)
			}
		}

		if err := zom.commitOffset(topic, partition, tracker); err != nil {
			return fmt.Errorf("FAILED to commit offset %d to Zookeeper. Last committed offset: %d", tracker.highestProcessedOffset, tracker.lastCommittedOffset)
		}
	}

	zom.l.Lock()
	delete(zom.offsets[topic], partition)
	zom.l.Unlock()

	return nil
}

func (zom *zookeeperOffsetManager) GetNextOffset(topic string, partition int32) (int64, error) {

	return zom.cg.group.FetchOffset(topic, partition)
}

func (zom *zookeeperOffsetManager) CommitOffsets() error {
	zom.commitOffsets()
	return nil
}

func (zom *zookeeperOffsetManager) MarkAsProcessed(topic string, partition int32, offset int64) bool {
	zom.l.RLock()
	defer zom.l.RUnlock()
	if p, ok := zom.offsets[topic][partition]; ok {
		return p.markAsProcessed(offset)
	} else {
		return false
	}
}

func (zom *zookeeperOffsetManager) Close() error {
	close(zom.closing)
	<-zom.closed

	zom.l.Lock()
	defer zom.l.Unlock()

	var closeError error
	for _, partitionOffsets := range zom.offsets {
		if len(partitionOffsets) > 0 {
			closeError = UncleanClose
		}
	}

	return closeError
}

func (zom *zookeeperOffsetManager) offsetCommitter() {
	commitTicker := time.NewTicker(zom.config.CommitInterval)
	defer commitTicker.Stop()

	for {
		select {
		case <-zom.closing:
			close(zom.closed)
			return
		case <-commitTicker.C:
			zom.commitOffsets()
		}
	}
}

func (zom *zookeeperOffsetManager) commitOffsets() error {
	zom.l.RLock()
	defer zom.l.RUnlock()

	var returnErr error
	for topic, partitionOffsets := range zom.offsets {
		for partition, offsetTracker := range partitionOffsets {
			err := zom.commitOffset(topic, partition, offsetTracker)
			switch err {
			case nil:
				// noop
			default:
				returnErr = err
			}
		}
	}
	return returnErr
}

func (zom *zookeeperOffsetManager) commitOffset(topic string, partition int32, tracker *partitionOffsetTracker) error {
	err := tracker.commit(func(offset int64) error {
		if offset >= 0 {
			return zom.cg.group.CommitOffset(topic, partition, offset+1)
		} else {
			return nil
		}
	})

	if err != nil {
		zom.cg.Logf("FAILED to commit offset %d for %s/%d!", tracker.highestProcessedOffset, topic, partition)
	} else if zom.config.VerboseLogging {
		zom.cg.Logf("Committed offset %d for %s/%d!", tracker.lastCommittedOffset, topic, partition)
	}

	return err
}

// MarkAsProcessed marks the provided offset as highest processed offset if
// it's higehr than any previous offset it has received.
func (pot *partitionOffsetTracker) markAsProcessed(offset int64) bool {
	pot.l.Lock()
	defer pot.l.Unlock()
	if offset > pot.highestProcessedOffset {
		pot.highestProcessedOffset = offset
		if pot.waitingForOffset == pot.highestProcessedOffset {
			close(pot.done)
		}
		return true
	} else {
		return false
	}
}

// Commit calls a committer function if the highest processed offset is out
// of sync with the last committed offset.
func (pot *partitionOffsetTracker) commit(committer offsetCommitter) error {
	pot.l.Lock()
	defer pot.l.Unlock()

	if pot.highestProcessedOffset > pot.lastCommittedOffset {
		if err := committer(pot.highestProcessedOffset); err != nil {
			return err
		}
		pot.lastCommittedOffset = pot.highestProcessedOffset
		return nil
	} else {
		return nil
	}
}

func (pot *partitionOffsetTracker) waitForOffset(offset int64, timeout time.Duration) bool {
	pot.l.Lock()
	if offset > pot.highestProcessedOffset {
		pot.waitingForOffset = offset
		pot.l.Unlock()
		select {
		case <-pot.done:
			return true
		case <-time.After(timeout):
			return false
		}
	} else {
		pot.l.Unlock()
		return true
	}
}
<commit_msg>Fixing not committing offset after finalizing partitions<commit_after>package consumergroup

import (
	"errors"
	"fmt"
	"sync"
	"time"
)

// OffsetManager is the main interface consumergroup requires to manage offsets of the consumergroup.
type OffsetManager interface {

	// InitializePartition is called when the consumergroup is starting to consume a
	// partition. It should return the last processed offset for this partition. Note:
	// the same partition can be initialized multiple times during a single run of a
	// consumer group due to other consumer instances coming online and offline.
	InitializePartition(topic string, partition int32) (int64, error)

	// Get next offset.
	GetNextOffset(topic string, partition int32) (int64, error)

	// Force offsetManager to commit all messages already stored.
	CommitOffsets() error

	// MarkAsProcessed tells the offset manager than a certain message has been successfully
	// processed by the consumer, and should be committed. The implementation does not have
	// to store this offset right away, but should return true if it intends to do this at
	// some point.
	//
	// Offsets should generally be increasing if the consumer
	// processes events serially, but this cannot be guaranteed if the consumer does any
	// asynchronous processing. This can be handled in various ways, e.g. by only accepting
	// offsets that are higehr than the offsets seen before for the same partition.
	MarkAsProcessed(topic string, partition int32, offset int64) bool

	// FinalizePartition is called when the consumergroup is done consuming a
	// partition. In this method, the offset manager can flush any remaining offsets to its
	// backend store. It should return an error if it was not able to commit the offset.
	// Note: it's possible that the consumergroup instance will start to consume the same
	// partition again after this function is called.
	FinalizePartition(topic string, partition int32, lastOffset int64, timeout time.Duration) error

	// Close is called when the consumergroup is shutting down. In normal circumstances, all
	// offsets are committed because FinalizePartition is called for all the running partition
	// consumers. You may want to check for this to be true, and try to commit any outstanding
	// offsets. If this doesn't succeed, it should return an error.
	Close() error
}

var (
	UncleanClose = errors.New("Not all offsets were committed before shutdown was completed")
)

// OffsetManagerConfig holds configuration setting son how the offset manager should behave.
type OffsetManagerConfig struct {
	CommitInterval time.Duration // Interval between offset flushes to the backend store.
	VerboseLogging bool          // Whether to enable verbose logging.
	EnableAutoCommit bool        // Whether to enable auto commit.
}

// NewOffsetManagerConfig returns a new OffsetManagerConfig with sane defaults.
func NewOffsetManagerConfig() *OffsetManagerConfig {
	return &OffsetManagerConfig{
		CommitInterval: 10 * time.Second,
	}
}

type (
	topicOffsets    map[int32]*partitionOffsetTracker
	offsetsMap      map[string]topicOffsets
	offsetCommitter func(int64) error
)

type partitionOffsetTracker struct {
	l                      sync.Mutex
	waitingForOffset       int64
	highestProcessedOffset int64
	lastCommittedOffset    int64
	done                   chan struct{}
}

type zookeeperOffsetManager struct {
	config  *OffsetManagerConfig
	l       sync.RWMutex
	offsets offsetsMap
	cg      *ConsumerGroup

	closing, closed chan struct{}
}

// NewZookeeperOffsetManager returns an offset manager that uses Zookeeper
// to store offsets.
func NewZookeeperOffsetManager(cg *ConsumerGroup, config *OffsetManagerConfig) OffsetManager {
	if config == nil {
		config = NewOffsetManagerConfig()
	}

	zom := &zookeeperOffsetManager{
		config:  config,
		cg:      cg,
		offsets: make(offsetsMap),
		closing: make(chan struct{}),
		closed:  make(chan struct{}),
	}

	if config.EnableAutoCommit {
		go zom.offsetCommitter()
	}

	return zom
}

func (zom *zookeeperOffsetManager) InitializePartition(topic string, partition int32) (int64, error) {
	zom.l.Lock()
	defer zom.l.Unlock()

	if zom.offsets[topic] == nil {
		zom.offsets[topic] = make(topicOffsets)
	}

	nextOffset, err := zom.cg.group.FetchOffset(topic, partition)
	if err != nil {
		return 0, err
	}

	zom.offsets[topic][partition] = &partitionOffsetTracker{
		highestProcessedOffset: nextOffset - 1,
		lastCommittedOffset:    nextOffset - 1,
		done:                   make(chan struct{}),
	}

	return nextOffset, nil
}

func (zom *zookeeperOffsetManager) FinalizePartition(topic string, partition int32, lastOffset int64, timeout time.Duration) error {
	zom.l.RLock()
	tracker := zom.offsets[topic][partition]
	zom.l.RUnlock()

	if lastOffset >= 0 {
		if lastOffset-tracker.highestProcessedOffset > 0 {
			zom.cg.Logf("%s/%d :: Last processed offset: %d. Waiting up to %ds for another %d messages to process...", topic, partition, tracker.highestProcessedOffset, timeout/time.Second, lastOffset-tracker.highestProcessedOffset)
			zom.CommitOffsets()
			if !tracker.waitForOffset(lastOffset, timeout) {
				return fmt.Errorf("TIMEOUT waiting for offset %d. Last committed offset: %d", lastOffset, tracker.lastCommittedOffset)
			}
		}

		if err := zom.commitOffset(topic, partition, tracker); err != nil {
			return fmt.Errorf("FAILED to commit offset %d to Zookeeper. Last committed offset: %d", tracker.highestProcessedOffset, tracker.lastCommittedOffset)
		}
	}

	zom.l.Lock()
	delete(zom.offsets[topic], partition)
	zom.l.Unlock()

	return nil
}

func (zom *zookeeperOffsetManager) GetNextOffset(topic string, partition int32) (int64, error) {

	return zom.cg.group.FetchOffset(topic, partition)
}

func (zom *zookeeperOffsetManager) CommitOffsets() error {
	return zom.commitOffsets()
}

func (zom *zookeeperOffsetManager) MarkAsProcessed(topic string, partition int32, offset int64) bool {
	zom.l.RLock()
	defer zom.l.RUnlock()
	if p, ok := zom.offsets[topic][partition]; ok {
		return p.markAsProcessed(offset)
	} else {
		return false
	}
}

func (zom *zookeeperOffsetManager) Close() error {
	close(zom.closing)
	<-zom.closed

	zom.l.Lock()
	defer zom.l.Unlock()

	var closeError error
	for _, partitionOffsets := range zom.offsets {
		if len(partitionOffsets) > 0 {
			closeError = UncleanClose
		}
	}

	return closeError
}

func (zom *zookeeperOffsetManager) offsetCommitter() {
	commitTicker := time.NewTicker(zom.config.CommitInterval)
	defer commitTicker.Stop()

	for {
		select {
		case <-zom.closing:
			close(zom.closed)
			return
		case <-commitTicker.C:
			zom.commitOffsets()
		}
	}
}

func (zom *zookeeperOffsetManager) commitOffsets() error {
	zom.l.RLock()
	defer zom.l.RUnlock()

	var returnErr error
	for topic, partitionOffsets := range zom.offsets {
		for partition, offsetTracker := range partitionOffsets {
			err := zom.commitOffset(topic, partition, offsetTracker)
			switch err {
			case nil:
				// noop
			default:
				returnErr = err
			}
		}
	}
	return returnErr
}

func (zom *zookeeperOffsetManager) commitOffset(topic string, partition int32, tracker *partitionOffsetTracker) error {
	err := tracker.commit(func(offset int64) error {
		if offset >= 0 {
			return zom.cg.group.CommitOffset(topic, partition, offset+1)
		} else {
			return nil
		}
	})

	if err != nil {
		zom.cg.Logf("FAILED to commit offset %d for %s/%d!", tracker.highestProcessedOffset, topic, partition)
	} else if zom.config.VerboseLogging {
		zom.cg.Logf("Committed offset %d for %s/%d!", tracker.lastCommittedOffset, topic, partition)
	}

	return err
}

// MarkAsProcessed marks the provided offset as highest processed offset if
// it's higehr than any previous offset it has received.
func (pot *partitionOffsetTracker) markAsProcessed(offset int64) bool {
	pot.l.Lock()
	defer pot.l.Unlock()
	if offset > pot.highestProcessedOffset {
		pot.highestProcessedOffset = offset
		if pot.waitingForOffset == pot.highestProcessedOffset {
			close(pot.done)
		}
		return true
	} else {
		return false
	}
}

// Commit calls a committer function if the highest processed offset is out
// of sync with the last committed offset.
func (pot *partitionOffsetTracker) commit(committer offsetCommitter) error {
	pot.l.Lock()
	defer pot.l.Unlock()

	if pot.highestProcessedOffset > pot.lastCommittedOffset {
		if err := committer(pot.highestProcessedOffset); err != nil {
			return err
		}
		pot.lastCommittedOffset = pot.highestProcessedOffset
		return nil
	} else {
		return nil
	}
}

func (pot *partitionOffsetTracker) waitForOffset(offset int64, timeout time.Duration) bool {
	pot.l.Lock()
	if offset > pot.highestProcessedOffset {
		pot.waitingForOffset = offset
		pot.l.Unlock()
		select {
		case <-pot.done:
			return true
		case <-time.After(timeout):
			return false
		}
	} else {
		pot.l.Unlock()
		return true
	}
}
<|endoftext|>


<commit_before>package impl

import . "github.com/gonium/gosdm630/meters"

func init() {
	Register(NewABBProducer)
}

const (
	METERTYPE_ABB = "ABB"
)

type ABBProducer struct {
	RS485Core
	Opcodes
}

func NewABBProducer() Producer {
	/***
	 * http://datenblatt.stark-elektronik.de/Energiezaehler_B-Serie_Handbuch.pdf
	 */
	ops := Opcodes{
		VoltageL1: 0x5B00,
		VoltageL2: 0x5B02,
		VoltageL3: 0x5B04,

		CurrentL1: 0x5B0C,
		CurrentL2: 0x5B0E,
		CurrentL3: 0x5B10,

		Power:   0x5B14,
		PowerL1: 0x5B16,
		PowerL2: 0x5B18,
		PowerL3: 0x5B1A,

		ImportL1:  0x5460,
		ImportL2:  0x5464,
		ImportL3:  0x5468,
		Import:    0x5000,

		ExportL1:  0x546C,
		ExportL2:  0x5470,
		ExportL3:  0x5474,
		Export:    0x5004,

		Cosphi:   0x5B3A,
		CosphiL1: 0x5B3B,
		CosphiL2: 0x5B3C,
		CosphiL3: 0x5B3D,

		Frequency: 0x5B2C,
	}
	return &ABBProducer{Opcodes: ops}
}

func (p *ABBProducer) Type() string {
	return METERTYPE_ABB
}

func (p *ABBProducer) Description() string {
	return "ABB A/B-Series meters"
}

func (p *ABBProducer) snip(iec Measurement, readlen uint16) Operation {
	opcode := p.Opcodes[iec]
	return Operation{
		FuncCode: ReadHoldingReg,
		OpCode:   opcode,
		ReadLen:  readlen,
		IEC61850: iec,
	}
}

// snip16 creates modbus operation for single register
func (p *ABBProducer) snip16(iec Measurement, scaler ...float64) Operation {
	snip := p.snip(iec, 1)

	snip.Transform = RTUUint16ToFloat64 // default conversion
	if len(scaler) > 0 {
		snip.Transform = MakeScaledTransform(snip.Transform, scaler[0])
	}

	return snip
}

// snip16i creates modbus operation for single register
func (p *ABBProducer) snip16i(iec Measurement, scaler ...float64) Operation {
	snip := p.snip(iec, 1)

	snip.Transform = RTUInt16ToFloat64 // default conversion
	if len(scaler) > 0 {
		snip.Transform = MakeScaledTransform(snip.Transform, scaler[0])
	}

	return snip
}

// snip32 creates modbus operation for double register
func (p *ABBProducer) snip32(iec Measurement, scaler ...float64) Operation {
	snip := p.snip(iec, 2)

	snip.Transform = RTUUint32ToFloat64 // default conversion
	if len(scaler) > 0 {
		snip.Transform = MakeScaledTransform(snip.Transform, scaler[0])
	}

	return snip
}

// snip32i creates modbus operation for double register
func (p *ABBProducer) snip32i(iec Measurement, scaler ...float64) Operation {
	snip := p.snip(iec, 2)

	snip.Transform = RTUInt32ToFloat64 // default conversion
	if len(scaler) > 0 {
		snip.Transform = MakeScaledTransform(snip.Transform, scaler[0])
	}

	return snip
}

// snip64 creates modbus operation for double register
func (p *ABBProducer) snip64(iec Measurement, scaler ...float64) Operation {
	snip := p.snip(iec, 4)

	snip.Transform = RTUUint64ToFloat64 // default conversion
	if len(scaler) > 0 {
		snip.Transform = MakeScaledTransform(snip.Transform, scaler[0])
	}

	return snip
}

func (p *ABBProducer) Probe() Operation {
	return p.snip16(VoltageL1)
}

func (p *ABBProducer) Produce() (res []Operation) {
	for _, op := range []Measurement{
		VoltageL1, VoltageL2, VoltageL3,
		CurrentL1, CurrentL2, CurrentL3,
	} {
		res = append(res, p.snip32(op, 10))
	}

	for _, op := range []Measurement{
		Cosphi, CosphiL1, CosphiL2, CosphiL3,
	} {
		res = append(res, p.snip16i(op, 1000))
	}

	for _, op := range []Measurement{
		Frequency,
	} {
		res = append(res, p.snip16(op,100))
	}

	for _, op := range []Measurement{
		Power, PowerL1, PowerL2, PowerL3,
	} {
		res = append(res, p.snip32i(op, 10))
	}

	for _, op := range []Measurement{
		Import, ImportL1, ImportL2, ImportL3,
		Export, ExportL1, ExportL2, ExportL3,
	} {
		res = append(res, p.snip64(op,100))
	}

	return res
}
<commit_msg>Simplify transformations (#125)<commit_after>package impl

import . "github.com/gonium/gosdm630/meters"

func init() {
	Register(NewABBProducer)
}

const (
	METERTYPE_ABB = "ABB"
)

type ABBProducer struct {
	RS485Core
	Opcodes
}

func NewABBProducer() Producer {
	/***
	 * http://datenblatt.stark-elektronik.de/Energiezaehler_B-Serie_Handbuch.pdf
	 */
	ops := Opcodes{
		VoltageL1: 0x5B00,
		VoltageL2: 0x5B02,
		VoltageL3: 0x5B04,

		CurrentL1: 0x5B0C,
		CurrentL2: 0x5B0E,
		CurrentL3: 0x5B10,

		Power:   0x5B14,
		PowerL1: 0x5B16,
		PowerL2: 0x5B18,
		PowerL3: 0x5B1A,

		ImportL1: 0x5460,
		ImportL2: 0x5464,
		ImportL3: 0x5468,
		Import:   0x5000,

		ExportL1: 0x546C,
		ExportL2: 0x5470,
		ExportL3: 0x5474,
		Export:   0x5004,

		Cosphi:   0x5B3A,
		CosphiL1: 0x5B3B,
		CosphiL2: 0x5B3C,
		CosphiL3: 0x5B3D,

		Frequency: 0x5B2C,
	}
	return &ABBProducer{Opcodes: ops}
}

func (p *ABBProducer) Type() string {
	return METERTYPE_ABB
}

func (p *ABBProducer) Description() string {
	return "ABB A/B-Series meters"
}

func (p *ABBProducer) snip(iec Measurement, readlen uint16, transform RTUTransform, scaler ...float64) Operation {
	snip := Operation{
		FuncCode:  ReadHoldingReg,
		OpCode:    p.Opcodes[iec],
		ReadLen:   readlen,
		Transform: transform,
		IEC61850:  iec,
	}

	if len(scaler) > 0 {
		snip.Transform = MakeScaledTransform(snip.Transform, scaler[0])
	}

	return snip
}

// snip16u creates modbus operation for single register
func (p *ABBProducer) snip16u(iec Measurement, scaler ...float64) Operation {
	return p.snip(iec, 1, RTUUint16ToFloat64, scaler...)
}

// snip16i creates modbus operation for single register
func (p *ABBProducer) snip16i(iec Measurement, scaler ...float64) Operation {
	return p.snip(iec, 1, RTUInt16ToFloat64, scaler...)
}

// snip32u creates modbus operation for double register
func (p *ABBProducer) snip32u(iec Measurement, scaler ...float64) Operation {
	return p.snip(iec, 2, RTUUint32ToFloat64, scaler...)
}

// snip32i creates modbus operation for double register
func (p *ABBProducer) snip32i(iec Measurement, scaler ...float64) Operation {
	return p.snip(iec, 2, RTUInt32ToFloat64, scaler...)
}

// snip64u creates modbus operation for double register
func (p *ABBProducer) snip64u(iec Measurement, scaler ...float64) Operation {
	return p.snip(iec, 4, RTUUint64ToFloat64, scaler...)
}

func (p *ABBProducer) Probe() Operation {
	return p.snip32u(VoltageL1, 10)
}

func (p *ABBProducer) Produce() (res []Operation) {
	for _, op := range []Measurement{
		VoltageL1, VoltageL2, VoltageL3,
		CurrentL1, CurrentL2, CurrentL3,
	} {
		res = append(res, p.snip32u(op, 10))
	}

	for _, op := range []Measurement{
		Cosphi, CosphiL1, CosphiL2, CosphiL3,
	} {
		res = append(res, p.snip16i(op, 1000))
	}

	for _, op := range []Measurement{
		Frequency,
	} {
		res = append(res, p.snip16u(op, 100))
	}

	for _, op := range []Measurement{
		Power, PowerL1, PowerL2, PowerL3,
	} {
		res = append(res, p.snip32i(op, 10))
	}

	for _, op := range []Measurement{
		Import, ImportL1, ImportL2, ImportL3,
		Export, ExportL1, ExportL2, ExportL3,
	} {
		res = append(res, p.snip64u(op, 100))
	}

	return res
}
<|endoftext|>