<commit_before>package lfs

import (
	"bufio"
	"bytes"
	"io"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

const (
	// blobSizeCutoff is used to determine which files to scan for Git LFS
	// pointers.  Any file with a size below this cutoff will be scanned.
	blobSizeCutoff = 140

	// stdoutBufSize is the size of the buffers given to a sub-process stdout
	stdoutBufSize = 16384

	// chanBufSize is the size of the channels used to pass data from one
	// sub-process to another.
	chanBufSize = 100
)

// WrappedPointer wraps a pointer.Pointer and provides the git sha1
// and the file name associated with the object, taken from the
// rev-list output.
type WrappedPointer struct {
	Sha1    string
	Name    string
	SrcName string
	Size    int64
	Status  string
	*Pointer
}

// indexFile is used when scanning the index. It stores the name of
// the file, the status of the file in the index, and, in the case of
// a moved or copied file, the original name of the file.
type indexFile struct {
	Name    string
	SrcName string
	Status  string
}

var z40 = regexp.MustCompile(`\^?0{40}`)

// ScanRefs takes a ref and returns a slice of WrappedPointer objects
// for all Git LFS pointers it finds for that ref.
func ScanRefs(refLeft, refRight string) ([]*WrappedPointer, error) {
	nameMap := make(map[string]string, 0)

	start := time.Now()
	defer func() {
		tracerx.PerformanceSince("scan", start)
	}()

	revs, err := revListShas(refLeft, refRight, refLeft == "", nameMap)
	if err != nil {
		return nil, err
	}

	smallShas, err := catFileBatchCheck(revs)
	if err != nil {
		return nil, err
	}

	pointerc, err := catFileBatch(smallShas)
	if err != nil {
		return nil, err
	}

	pointers := make([]*WrappedPointer, 0)
	for p := range pointerc {
		if name, ok := nameMap[p.Sha1]; ok {
			p.Name = name
		}
		pointers = append(pointers, p)
	}

	return pointers, nil
}

// ScanIndex returns a slice of WrappedPointer objects for all
// Git LFS pointers it finds in the index.
func ScanIndex() ([]*WrappedPointer, error) {
	nameMap := make(map[string]*indexFile, 0)

	start := time.Now()
	defer func() {
		tracerx.PerformanceSince("scan-staging", start)
	}()

	revs, err := revListIndex(false, nameMap)
	if err != nil {
		return nil, err
	}

	cachedRevs, err := revListIndex(true, nameMap)
	if err != nil {
		return nil, err
	}

	allRevs := make(chan string)
	go func() {
		seenRevs := make(map[string]bool, 0)

		for rev := range revs {
			seenRevs[rev] = true
			allRevs <- rev
		}

		for rev := range cachedRevs {
			if _, ok := seenRevs[rev]; !ok {
				allRevs <- rev
			}
		}
		close(allRevs)
	}()

	smallShas, err := catFileBatchCheck(allRevs)
	if err != nil {
		return nil, err
	}

	pointerc, err := catFileBatch(smallShas)
	if err != nil {
		return nil, err
	}

	pointers := make([]*WrappedPointer, 0)
	for p := range pointerc {
		if e, ok := nameMap[p.Sha1]; ok {
			p.Name = e.Name
			p.Status = e.Status
			p.SrcName = e.SrcName
		}
		pointers = append(pointers, p)
	}

	return pointers, nil

}

// revListShas uses git rev-list to return the list of object sha1s
// for the given ref. If all is true, ref is ignored. It returns a
// channel from which sha1 strings can be read.
func revListShas(refLeft, refRight string, all bool, nameMap map[string]string) (chan string, error) {
	refArgs := []string{"rev-list", "--objects"}
	if all {
		refArgs = append(refArgs, "--all")
	} else {
		refArgs = append(refArgs, "--no-walk")
		refArgs = append(refArgs, refLeft)
		if refRight != "" && !z40.MatchString(refRight) {
			refArgs = append(refArgs, refRight)
		}
	}

	cmd, err := startCommand("git", refArgs...)
	if err != nil {
		return nil, err
	}

	cmd.Stdin.Close()

	revs := make(chan string, chanBufSize)

	go func() {
		scanner := bufio.NewScanner(cmd.Stdout)
		for scanner.Scan() {
			line := strings.TrimSpace(scanner.Text())
			if len(line) < 40 {
				continue
			}

			sha1 := line[0:40]
			if len(line) > 40 {
				nameMap[sha1] = line[41:len(line)]
			}
			revs <- sha1
		}
		close(revs)
	}()

	return revs, nil
}

// revListIndex uses git diff-index to return the list of object sha1s
// for in the indexf. It returns a channel from which sha1 strings can be read.
// The namMap will be filled indexFile pointers mapping sha1s to indexFiles.
func revListIndex(cache bool, nameMap map[string]*indexFile) (chan string, error) {
	cmdArgs := []string{"diff-index", "-M"}
	if cache {
		cmdArgs = append(cmdArgs, "--cached")
	}
	cmdArgs = append(cmdArgs, "HEAD")

	cmd, err := startCommand("git", cmdArgs...)
	if err != nil {
		return nil, err
	}

	cmd.Stdin.Close()

	revs := make(chan string, chanBufSize)

	go func() {
		scanner := bufio.NewScanner(cmd.Stdout)
		for scanner.Scan() {
			// Format is:
			// :100644 100644 c5b3d83a7542255ec7856487baa5e83d65b1624c 9e82ac1b514be060945392291b5b3108c22f6fe3 M foo.gif
			// :<old mode> <new mode> <old sha1> <new sha1> <status>\t<file name>[\t<file name>]
			line := scanner.Text()
			parts := strings.Split(line, "\t")
			if len(parts) < 2 {
				continue
			}

			description := strings.Split(parts[0], " ")
			files := parts[1:len(parts)]

			if len(description) >= 5 {
				status := description[4][0:1]
				sha1 := description[3]
				if status == "M" {
					sha1 = description[2] // This one is modified but not added
				}
				nameMap[sha1] = &indexFile{files[len(files)-1], files[0], status}
				revs <- sha1
			}
		}
		close(revs)
	}()

	return revs, nil
}

// catFileBatchCheck uses git cat-file --batch-check to get the type
// and size of a git object. Any object that isn't of type blob and
// under the blobSizeCutoff will be ignored. revs is a channel over
// which strings containing git sha1s will be sent. It returns a channel
// from which sha1 strings can be read.
func catFileBatchCheck(revs chan string) (chan string, error) {
	cmd, err := startCommand("git", "cat-file", "--batch-check")
	if err != nil {
		return nil, err
	}

	smallRevs := make(chan string, chanBufSize)

	go func() {
		scanner := bufio.NewScanner(cmd.Stdout)
		for scanner.Scan() {
			line := scanner.Text()
			// Format is:
			// <sha1> <type> <size>
			// type is at a fixed spot, if we see that it's "blob", we can avoid
			// splitting the line just to get the size.
			if line[41:45] == "blob" {
				size, err := strconv.Atoi(line[46:len(line)])
				if err != nil {
					continue
				}
				if size < blobSizeCutoff {
					smallRevs <- line[0:40]
				}
			}
		}
		close(smallRevs)
	}()

	go func() {
		for r := range revs {
			cmd.Stdin.Write([]byte(r + "\n"))
		}
		cmd.Stdin.Close()
	}()

	return smallRevs, nil
}

// catFileBatch uses git cat-file --batch to get the object contents
// of a git object, given its sha1. The contents will be decoded into
// a Git LFS pointer. revs is a channel over which strings containing Git SHA1s
// will be sent. It returns a channel from which point.Pointers can be read.
func catFileBatch(revs chan string) (chan *WrappedPointer, error) {
	cmd, err := startCommand("git", "cat-file", "--batch")
	if err != nil {
		return nil, err
	}

	pointers := make(chan *WrappedPointer, chanBufSize)

	go func() {
		for {
			l, err := cmd.Stdout.ReadBytes('\n')
			if err != nil {
				break
			}

			// Line is formatted:
			// <sha1> <type> <size>
			fields := bytes.Fields(l)
			s, _ := strconv.Atoi(string(fields[2]))

			nbuf := make([]byte, s)
			_, err = io.ReadFull(cmd.Stdout, nbuf)
			if err != nil {
				break // Legit errors
			}

			p, err := DecodePointer(bytes.NewBuffer(nbuf))
			if err == nil {
				pointers <- &WrappedPointer{
					Sha1:    string(fields[0]),
					Size:    p.Size,
					Pointer: p,
				}
			}

			_, err = cmd.Stdout.ReadBytes('\n') // Extra \n inserted by cat-file
			if err != nil {
				break
			}
		}
		close(pointers)
	}()

	go func() {
		for r := range revs {
			cmd.Stdin.Write([]byte(r + "\n"))
		}
		cmd.Stdin.Close()
	}()

	return pointers, nil
}

type wrappedCmd struct {
	Stdin  io.WriteCloser
	Stdout *bufio.Reader
	*exec.Cmd
}

// startCommand starts up a command and creates a stdin pipe and a buffered
// stdout pipe, wrapped in a wrappedCmd. The stdout buffer will be of stdoutBufSize
// bytes.
func startCommand(command string, args ...string) (*wrappedCmd, error) {
	cmd := exec.Command(command, args...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}

	tracerx.Printf("run_command: %s %s", command, strings.Join(args, " "))
	if err := cmd.Start(); err != nil {
		return nil, err
	}

	return &wrappedCmd{stdin, bufio.NewReaderSize(stdout, stdoutBufSize), cmd}, nil
}
<commit_msg>this forces the scanner to look for deleted objects<commit_after>package lfs

import (
	"bufio"
	"bytes"
	"io"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

const (
	// blobSizeCutoff is used to determine which files to scan for Git LFS
	// pointers.  Any file with a size below this cutoff will be scanned.
	blobSizeCutoff = 140

	// stdoutBufSize is the size of the buffers given to a sub-process stdout
	stdoutBufSize = 16384

	// chanBufSize is the size of the channels used to pass data from one
	// sub-process to another.
	chanBufSize = 100
)

// WrappedPointer wraps a pointer.Pointer and provides the git sha1
// and the file name associated with the object, taken from the
// rev-list output.
type WrappedPointer struct {
	Sha1    string
	Name    string
	SrcName string
	Size    int64
	Status  string
	*Pointer
}

// indexFile is used when scanning the index. It stores the name of
// the file, the status of the file in the index, and, in the case of
// a moved or copied file, the original name of the file.
type indexFile struct {
	Name    string
	SrcName string
	Status  string
}

var z40 = regexp.MustCompile(`\^?0{40}`)

// ScanRefs takes a ref and returns a slice of WrappedPointer objects
// for all Git LFS pointers it finds for that ref.
func ScanRefs(refLeft, refRight string) ([]*WrappedPointer, error) {
	nameMap := make(map[string]string, 0)

	start := time.Now()
	defer func() {
		tracerx.PerformanceSince("scan", start)
	}()

	revs, err := revListShas(refLeft, refRight, refLeft == "", nameMap)
	if err != nil {
		return nil, err
	}

	smallShas, err := catFileBatchCheck(revs)
	if err != nil {
		return nil, err
	}

	pointerc, err := catFileBatch(smallShas)
	if err != nil {
		return nil, err
	}

	pointers := make([]*WrappedPointer, 0)
	for p := range pointerc {
		if name, ok := nameMap[p.Sha1]; ok {
			p.Name = name
		}
		pointers = append(pointers, p)
	}

	return pointers, nil
}

// ScanIndex returns a slice of WrappedPointer objects for all
// Git LFS pointers it finds in the index.
func ScanIndex() ([]*WrappedPointer, error) {
	nameMap := make(map[string]*indexFile, 0)

	start := time.Now()
	defer func() {
		tracerx.PerformanceSince("scan-staging", start)
	}()

	revs, err := revListIndex(false, nameMap)
	if err != nil {
		return nil, err
	}

	cachedRevs, err := revListIndex(true, nameMap)
	if err != nil {
		return nil, err
	}

	allRevs := make(chan string)
	go func() {
		seenRevs := make(map[string]bool, 0)

		for rev := range revs {
			seenRevs[rev] = true
			allRevs <- rev
		}

		for rev := range cachedRevs {
			if _, ok := seenRevs[rev]; !ok {
				allRevs <- rev
			}
		}
		close(allRevs)
	}()

	smallShas, err := catFileBatchCheck(allRevs)
	if err != nil {
		return nil, err
	}

	pointerc, err := catFileBatch(smallShas)
	if err != nil {
		return nil, err
	}

	pointers := make([]*WrappedPointer, 0)
	for p := range pointerc {
		if e, ok := nameMap[p.Sha1]; ok {
			p.Name = e.Name
			p.Status = e.Status
			p.SrcName = e.SrcName
		}
		pointers = append(pointers, p)
	}

	return pointers, nil

}

// revListShas uses git rev-list to return the list of object sha1s
// for the given ref. If all is true, ref is ignored. It returns a
// channel from which sha1 strings can be read.
func revListShas(refLeft, refRight string, all bool, nameMap map[string]string) (chan string, error) {
	refArgs := []string{"rev-list", "--objects"}
	if all {
		refArgs = append(refArgs, "--all")
	} else {
		refArgs = append(refArgs, "--do-walk")
		refArgs = append(refArgs, refLeft)
		if refRight != "" && !z40.MatchString(refRight) {
			refArgs = append(refArgs, refRight)
		}
	}

	cmd, err := startCommand("git", refArgs...)
	if err != nil {
		return nil, err
	}

	cmd.Stdin.Close()

	revs := make(chan string, chanBufSize)

	go func() {
		scanner := bufio.NewScanner(cmd.Stdout)
		for scanner.Scan() {
			line := strings.TrimSpace(scanner.Text())
			if len(line) < 40 {
				continue
			}

			sha1 := line[0:40]
			if len(line) > 40 {
				nameMap[sha1] = line[41:len(line)]
			}
			revs <- sha1
		}
		close(revs)
	}()

	return revs, nil
}

// revListIndex uses git diff-index to return the list of object sha1s
// for in the indexf. It returns a channel from which sha1 strings can be read.
// The namMap will be filled indexFile pointers mapping sha1s to indexFiles.
func revListIndex(cache bool, nameMap map[string]*indexFile) (chan string, error) {
	cmdArgs := []string{"diff-index", "-M"}
	if cache {
		cmdArgs = append(cmdArgs, "--cached")
	}
	cmdArgs = append(cmdArgs, "HEAD")

	cmd, err := startCommand("git", cmdArgs...)
	if err != nil {
		return nil, err
	}

	cmd.Stdin.Close()

	revs := make(chan string, chanBufSize)

	go func() {
		scanner := bufio.NewScanner(cmd.Stdout)
		for scanner.Scan() {
			// Format is:
			// :100644 100644 c5b3d83a7542255ec7856487baa5e83d65b1624c 9e82ac1b514be060945392291b5b3108c22f6fe3 M foo.gif
			// :<old mode> <new mode> <old sha1> <new sha1> <status>\t<file name>[\t<file name>]
			line := scanner.Text()
			parts := strings.Split(line, "\t")
			if len(parts) < 2 {
				continue
			}

			description := strings.Split(parts[0], " ")
			files := parts[1:len(parts)]

			if len(description) >= 5 {
				status := description[4][0:1]
				sha1 := description[3]
				if status == "M" {
					sha1 = description[2] // This one is modified but not added
				}
				nameMap[sha1] = &indexFile{files[len(files)-1], files[0], status}
				revs <- sha1
			}
		}
		close(revs)
	}()

	return revs, nil
}

// catFileBatchCheck uses git cat-file --batch-check to get the type
// and size of a git object. Any object that isn't of type blob and
// under the blobSizeCutoff will be ignored. revs is a channel over
// which strings containing git sha1s will be sent. It returns a channel
// from which sha1 strings can be read.
func catFileBatchCheck(revs chan string) (chan string, error) {
	cmd, err := startCommand("git", "cat-file", "--batch-check")
	if err != nil {
		return nil, err
	}

	smallRevs := make(chan string, chanBufSize)

	go func() {
		scanner := bufio.NewScanner(cmd.Stdout)
		for scanner.Scan() {
			line := scanner.Text()
			// Format is:
			// <sha1> <type> <size>
			// type is at a fixed spot, if we see that it's "blob", we can avoid
			// splitting the line just to get the size.
			if line[41:45] == "blob" {
				size, err := strconv.Atoi(line[46:len(line)])
				if err != nil {
					continue
				}
				if size < blobSizeCutoff {
					smallRevs <- line[0:40]
				}
			}
		}
		close(smallRevs)
	}()

	go func() {
		for r := range revs {
			cmd.Stdin.Write([]byte(r + "\n"))
		}
		cmd.Stdin.Close()
	}()

	return smallRevs, nil
}

// catFileBatch uses git cat-file --batch to get the object contents
// of a git object, given its sha1. The contents will be decoded into
// a Git LFS pointer. revs is a channel over which strings containing Git SHA1s
// will be sent. It returns a channel from which point.Pointers can be read.
func catFileBatch(revs chan string) (chan *WrappedPointer, error) {
	cmd, err := startCommand("git", "cat-file", "--batch")
	if err != nil {
		return nil, err
	}

	pointers := make(chan *WrappedPointer, chanBufSize)

	go func() {
		for {
			l, err := cmd.Stdout.ReadBytes('\n')
			if err != nil {
				break
			}

			// Line is formatted:
			// <sha1> <type> <size>
			fields := bytes.Fields(l)
			s, _ := strconv.Atoi(string(fields[2]))

			nbuf := make([]byte, s)
			_, err = io.ReadFull(cmd.Stdout, nbuf)
			if err != nil {
				break // Legit errors
			}

			p, err := DecodePointer(bytes.NewBuffer(nbuf))
			if err == nil {
				pointers <- &WrappedPointer{
					Sha1:    string(fields[0]),
					Size:    p.Size,
					Pointer: p,
				}
			}

			_, err = cmd.Stdout.ReadBytes('\n') // Extra \n inserted by cat-file
			if err != nil {
				break
			}
		}
		close(pointers)
	}()

	go func() {
		for r := range revs {
			cmd.Stdin.Write([]byte(r + "\n"))
		}
		cmd.Stdin.Close()
	}()

	return pointers, nil
}

type wrappedCmd struct {
	Stdin  io.WriteCloser
	Stdout *bufio.Reader
	*exec.Cmd
}

// startCommand starts up a command and creates a stdin pipe and a buffered
// stdout pipe, wrapped in a wrappedCmd. The stdout buffer will be of stdoutBufSize
// bytes.
func startCommand(command string, args ...string) (*wrappedCmd, error) {
	cmd := exec.Command(command, args...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}

	tracerx.Printf("run_command: %s %s", command, strings.Join(args, " "))
	if err := cmd.Start(); err != nil {
		return nil, err
	}

	return &wrappedCmd{stdin, bufio.NewReaderSize(stdout, stdoutBufSize), cmd}, nil
}
<|endoftext|>


<commit_before>// Package taskq implements an intra-process task queue with a simple
// retry model.
package taskq

import (
	"errors"
	"log"
	"sync"
	"time"
)

// Task is the execution unit.
type Task interface {
	// Do executes the task. Errors returned are logged to the default
	// logger.
	Do() error
}

// Func represents a function that is a Task.
type Func func() error

// Do executes the task function.
func (t Func) Do() error {
	return t()
}

// Queue manages a fixed-sized queue of tasks to be executed at a fixed
// rate by a number of parallel workers.
type Queue struct {
	taskc         chan Task
	waitg         sync.WaitGroup
	tickermu      sync.Mutex
	ticker        *time.Ticker
	retries       int
	retrydelay    time.Duration
	retrymaxdelay time.Duration
}

var errWorkersLessThanOne = errors.New("taskq: workers less than one")

// New creates a new queue with the specified queue size. Panics if size is
// less than zero.
func New(size int) *Queue {
	q := &Queue{
		taskc:         make(chan Task, size),
		ticker:        time.NewTicker(time.Second),
		retrydelay:    1 * time.Second,
		retrymaxdelay: 5 * time.Minute,
	}
	return q
}

// Start spawns n worker goroutines to execute tasks. Panics if n is
// less than 1.
func (q *Queue) Start(n int) {
	if n < 1 {
		panic(errWorkersLessThanOne)
	}
	q.waitg.Add(n)
	for i := 0; i < n; i++ {
		go q.worker()
	}
}

// Add adds a task to the queue, blocking if the queue is full.
func (q *Queue) Add(t Task) {
	q.taskc <- t
}

// Stop closes the queue and waits for the remaining tasks to be
// executed.
func (q *Queue) Stop() {
	close(q.taskc)
	q.waitg.Wait()
	q.ticker.Stop()
}

// SetInterval sets minimum time interval between tasks. Panics if not
// greater than zero. Default is 1 * time.Second.
func (q *Queue) SetInterval(dur time.Duration) {
	q.tickermu.Lock()
	defer q.tickermu.Unlock()
	q.ticker.Stop()
	q.ticker = time.NewTicker(dur)
}

// SetRetries sets maximum number of retries before dropping a task.  If
// retries is zero, never drop the task. Default is zero.
func (q *Queue) SetRetries(retries int) {
	q.retries = retries
}

// SetRetryDelay sets the first retry delay duration, and maximum delay
// after doubling the delay for each retry. Default delay of
// 1 * time.Second, max of 5 * time.Minute.
func (q *Queue) SetRetryDelay(delay, maxdelay time.Duration) {
	q.retrydelay = delay
	q.retrymaxdelay = maxdelay
}

// throttle waits for ticker or new interval duration
func (q *Queue) throttle() {
	q.tickermu.Lock()
	<-q.ticker.C
	q.tickermu.Unlock()
}

type retryState struct {
	n        int
	max      int
	delay    time.Duration
	maxdelay time.Duration
}

// backoff sleeps then returns true if task should be retried
func (r *retryState) backoff() bool {
	r.n++
	time.Sleep(r.delay)
	if r.delay < r.maxdelay {
		r.delay *= 2
		if r.delay > r.maxdelay {
			r.delay = r.maxdelay
		}
	}
	return r.max > 0 && r.n < r.max
}

func (r *retryState) reset(q *Queue) {
	r.n = 0
	r.max = q.retries
	r.delay = q.retrydelay
	r.maxdelay = q.retrymaxdelay
}

// worker pulls tasks from the queue and executes them until the queue
// is stopped.
func (q *Queue) worker() {
	defer q.waitg.Done()
	var retry retryState
	retry.reset(q)
	for t := range q.taskc {
		q.throttle()
		for err := t.Do(); err != nil; err = t.Do() {
			log.Println(err)
			if !retry.backoff() {
				break
			}
		}
		retry.reset(q)
	}
}
<commit_msg>allow restarting of a Queue<commit_after>// Package taskq implements an intra-process task queue with a simple
// retry model.
package taskq

import (
	"errors"
	"log"
	"sync"
	"time"
)

// Task is the execution unit.
type Task interface {
	// Do executes the task. Errors returned are logged to the default
	// logger.
	Do() error
}

// Func represents a function that is a Task.
type Func func() error

// Do executes the task function.
func (t Func) Do() error {
	return t()
}

// Queue manages a fixed-sized queue of tasks to be executed at a fixed
// rate by a number of parallel workers.
type Queue struct {
	size          int
	taskc         chan Task
	waitg         sync.WaitGroup
	tickermu      sync.Mutex
	ticker        *time.Ticker
	interval      time.Duration
	retries       int
	retrydelay    time.Duration
	retrymaxdelay time.Duration
}

var errWorkersLessThanOne = errors.New("taskq: workers less than one")

// New creates a new queue with the specified queue size. Panics if size is
// less than zero.
func New(size int) *Queue {
	q := &Queue{
		size:          size,
		interval:      1 * time.Second,
		retrydelay:    1 * time.Second,
		retrymaxdelay: 5 * time.Minute,
	}
	return q
}

// Start spawns n worker goroutines to execute tasks. Panics if n is
// less than 1.
func (q *Queue) Start(n int) {
	if n < 1 {
		panic(errWorkersLessThanOne)
	}
	q.taskc = make(chan Task, q.size)
	q.ticker = time.NewTicker(q.interval)
	q.waitg.Add(n)
	for i := 0; i < n; i++ {
		go q.worker()
	}
}

// Add adds a task to the queue, blocking if the queue is full.
func (q *Queue) Add(t Task) {
	q.taskc <- t
}

// Stop closes the queue and waits for the remaining tasks to be
// executed.
func (q *Queue) Stop() {
	close(q.taskc)
	q.waitg.Wait()
	q.ticker.Stop()
}

// SetInterval sets minimum time interval between tasks. Panics if not
// greater than zero. Default is 1 * time.Second.
func (q *Queue) SetInterval(dur time.Duration) {
	q.tickermu.Lock()
	defer q.tickermu.Unlock()
	if q.ticker != nil {
		q.ticker.Stop()
	}
	q.ticker = time.NewTicker(dur)
	q.interval = dur
}

// SetRetries sets maximum number of retries before dropping a task.  If
// retries is zero, never drop the task. Default is zero.
func (q *Queue) SetRetries(retries int) {
	q.retries = retries
}

// SetRetryDelay sets the first retry delay duration, and maximum delay
// after doubling the delay for each retry. Default delay of
// 1 * time.Second, max of 5 * time.Minute.
func (q *Queue) SetRetryDelay(delay, maxdelay time.Duration) {
	q.retrydelay = delay
	q.retrymaxdelay = maxdelay
}

// throttle waits for ticker or new interval duration
func (q *Queue) throttle() {
	q.tickermu.Lock()
	<-q.ticker.C
	q.tickermu.Unlock()
}

type retryState struct {
	n        int
	max      int
	delay    time.Duration
	maxdelay time.Duration
}

// backoff sleeps then returns true if task should be retried
func (r *retryState) backoff() bool {
	r.n++
	time.Sleep(r.delay)
	if r.delay < r.maxdelay {
		r.delay *= 2
		if r.delay > r.maxdelay {
			r.delay = r.maxdelay
		}
	}
	return r.max > 0 && r.n < r.max
}

func (r *retryState) reset(q *Queue) {
	r.n = 0
	r.max = q.retries
	r.delay = q.retrydelay
	r.maxdelay = q.retrymaxdelay
}

// worker pulls tasks from the queue and executes them until the queue
// is stopped.
func (q *Queue) worker() {
	defer q.waitg.Done()
	var retry retryState
	retry.reset(q)
	for t := range q.taskc {
		q.throttle()
		for err := t.Do(); err != nil; err = t.Do() {
			log.Println(err)
			if !retry.backoff() {
				break
			}
		}
		retry.reset(q)
	}
}
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"

	"github.com/hashicorp/terraform/addrs"
)

// DeposedTransformer is a GraphTransformer that adds deposed resources
// to the graph.
type DeposedTransformer struct {
	// State is the global state. We'll automatically find the correct
	// ModuleState based on the Graph.Path that is being transformed.
	State *State

	// View, if non-empty, is the ModuleState.View used around the state
	// to find deposed resources.
	View string

	// The provider used by the resourced which were deposed
	ResolvedProvider addrs.AbsProviderConfig
}

func (t *DeposedTransformer) Transform(g *Graph) error {
	state := t.State.ModuleByPath(g.Path)
	if state == nil {
		// If there is no state for our module there can't be any deposed
		// resources, since they live in the state.
		return nil
	}

	// If we have a view, apply it now
	if t.View != "" {
		state = state.View(t.View)
	}

	// Go through all the resources in our state to look for deposed resources
	for k, rs := range state.Resources {
		// If we have no deposed resources, then move on
		if len(rs.Deposed) == 0 {
			continue
		}

		legacyAddr, err := parseResourceAddressInternal(k)
		if err != nil {
			return fmt.Errorf("invalid instance key %q in state: %s", k, err)
		}
		addr := legacyAddr.AbsResourceInstanceAddr()

		providerAddr, err := rs.ProviderAddr()
		if err != nil {
			return fmt.Errorf("invalid instance provider address %q in state: %s", rs.Provider, err)
		}

		for i := range rs.Deposed {
			g.Add(&graphNodeDeposedResource{
				Addr:             addr,
				Index:            i,
				RecordedProvider: providerAddr,
			})
		}
	}

	return nil
}

// graphNodeDeposedResource is the graph vertex representing a deposed resource.
type graphNodeDeposedResource struct {
	Addr             addrs.AbsResourceInstance
	Index            int // Index into the "deposed" list in state
	RecordedProvider addrs.AbsProviderConfig
	ResolvedProvider addrs.AbsProviderConfig
}

var (
	_ GraphNodeProviderConsumer = (*graphNodeDeposedResource)(nil)
	_ GraphNodeEvalable         = (*graphNodeDeposedResource)(nil)
)

func (n *graphNodeDeposedResource) Name() string {
	return fmt.Sprintf("%s (deposed #%d)", n.Addr.String(), n.Index)
}

func (n *graphNodeDeposedResource) ProvidedBy() (addrs.AbsProviderConfig, bool) {
	return n.RecordedProvider, true
}

func (n *graphNodeDeposedResource) SetProvider(addr addrs.AbsProviderConfig) {
	// Because our ProvidedBy returns exact=true, this is actually rather
	// pointless and should always just be the address we asked for.
	n.RecordedProvider = addr
}

// GraphNodeEvalable impl.
func (n *graphNodeDeposedResource) EvalTree() EvalNode {
	addr := n.Addr

	var provider ResourceProvider
	var state *InstanceState

	seq := &EvalSequence{Nodes: make([]EvalNode, 0, 5)}

	stateKey := NewLegacyResourceInstanceAddress(addr).stateId()

	// Refresh the resource
	seq.Nodes = append(seq.Nodes, &EvalOpFilter{
		Ops: []walkOperation{walkRefresh},
		Node: &EvalSequence{
			Nodes: []EvalNode{
				&EvalGetProvider{
					Addr:   n.ResolvedProvider,
					Output: &provider,
				},
				&EvalReadStateDeposed{
					Name:   stateKey,
					Output: &state,
					Index:  n.Index,
				},
				&EvalRefresh{
					Addr:     addr.Resource,
					Provider: &provider,
					State:    &state,
					Output:   &state,
				},
				&EvalWriteStateDeposed{
					Name:         stateKey,
					ResourceType: n.Addr.Resource.Resource.Type,
					Provider:     n.ResolvedProvider.String(), // FIXME: Change underlying struct to use addrs.AbsProviderConfig itself
					State:        &state,
					Index:        n.Index,
				},
			},
		},
	})

	// Apply
	var diff *InstanceDiff
	var err error
	seq.Nodes = append(seq.Nodes, &EvalOpFilter{
		Ops: []walkOperation{walkApply, walkDestroy},
		Node: &EvalSequence{
			Nodes: []EvalNode{
				&EvalGetProvider{
					Addr:   n.ResolvedProvider,
					Output: &provider,
				},
				&EvalReadStateDeposed{
					Name:   stateKey,
					Output: &state,
					Index:  n.Index,
				},
				&EvalDiffDestroy{
					Addr:   addr.Resource,
					State:  &state,
					Output: &diff,
				},
				// Call pre-apply hook
				&EvalApplyPre{
					Addr:  addr.Resource,
					State: &state,
					Diff:  &diff,
				},
				&EvalApply{
					Addr:     addr.Resource,
					State:    &state,
					Diff:     &diff,
					Provider: &provider,
					Output:   &state,
					Error:    &err,
				},
				// Always write the resource back to the state deposed... if it
				// was successfully destroyed it will be pruned. If it was not, it will
				// be caught on the next run.
				&EvalWriteStateDeposed{
					Name:         stateKey,
					ResourceType: n.Addr.Resource.Resource.Type,
					Provider:     n.ResolvedProvider.String(), // FIXME: Change underlying struct to use addrs.AbsProviderConfig itself
					State:        &state,
					Index:        n.Index,
				},
				&EvalApplyPost{
					Addr:  addr.Resource,
					State: &state,
					Error: &err,
				},
				&EvalReturnError{
					Error: &err,
				},
				&EvalUpdateStateHook{},
			},
		},
	})

	return seq
}
<commit_msg>deposed nodes need the resolved provider too<commit_after>package terraform

import (
	"fmt"

	"github.com/hashicorp/terraform/addrs"
)

// DeposedTransformer is a GraphTransformer that adds deposed resources
// to the graph.
type DeposedTransformer struct {
	// State is the global state. We'll automatically find the correct
	// ModuleState based on the Graph.Path that is being transformed.
	State *State

	// View, if non-empty, is the ModuleState.View used around the state
	// to find deposed resources.
	View string

	// The provider used by the resourced which were deposed
	ResolvedProvider addrs.AbsProviderConfig
}

func (t *DeposedTransformer) Transform(g *Graph) error {
	state := t.State.ModuleByPath(g.Path)
	if state == nil {
		// If there is no state for our module there can't be any deposed
		// resources, since they live in the state.
		return nil
	}

	// If we have a view, apply it now
	if t.View != "" {
		state = state.View(t.View)
	}

	// Go through all the resources in our state to look for deposed resources
	for k, rs := range state.Resources {
		// If we have no deposed resources, then move on
		if len(rs.Deposed) == 0 {
			continue
		}

		legacyAddr, err := parseResourceAddressInternal(k)
		if err != nil {
			return fmt.Errorf("invalid instance key %q in state: %s", k, err)
		}
		addr := legacyAddr.AbsResourceInstanceAddr()

		providerAddr, err := rs.ProviderAddr()
		if err != nil {
			return fmt.Errorf("invalid instance provider address %q in state: %s", rs.Provider, err)
		}

		for i := range rs.Deposed {
			g.Add(&graphNodeDeposedResource{
				Addr:             addr,
				Index:            i,
				RecordedProvider: providerAddr,
				ResolvedProvider: providerAddr,
			})
		}
	}

	return nil
}

// graphNodeDeposedResource is the graph vertex representing a deposed resource.
type graphNodeDeposedResource struct {
	Addr             addrs.AbsResourceInstance
	Index            int // Index into the "deposed" list in state
	RecordedProvider addrs.AbsProviderConfig
	ResolvedProvider addrs.AbsProviderConfig
}

var (
	_ GraphNodeProviderConsumer = (*graphNodeDeposedResource)(nil)
	_ GraphNodeEvalable         = (*graphNodeDeposedResource)(nil)
)

func (n *graphNodeDeposedResource) Name() string {
	return fmt.Sprintf("%s (deposed #%d)", n.Addr.String(), n.Index)
}

func (n *graphNodeDeposedResource) ProvidedBy() (addrs.AbsProviderConfig, bool) {
	return n.RecordedProvider, true
}

func (n *graphNodeDeposedResource) SetProvider(addr addrs.AbsProviderConfig) {
	// Because our ProvidedBy returns exact=true, this is actually rather
	// pointless and should always just be the address we asked for.
	n.RecordedProvider = addr
}

// GraphNodeEvalable impl.
func (n *graphNodeDeposedResource) EvalTree() EvalNode {
	addr := n.Addr

	var provider ResourceProvider
	var state *InstanceState

	seq := &EvalSequence{Nodes: make([]EvalNode, 0, 5)}

	stateKey := NewLegacyResourceInstanceAddress(addr).stateId()

	// Refresh the resource
	seq.Nodes = append(seq.Nodes, &EvalOpFilter{
		Ops: []walkOperation{walkRefresh},
		Node: &EvalSequence{
			Nodes: []EvalNode{
				&EvalGetProvider{
					Addr:   n.ResolvedProvider,
					Output: &provider,
				},
				&EvalReadStateDeposed{
					Name:   stateKey,
					Output: &state,
					Index:  n.Index,
				},
				&EvalRefresh{
					Addr:     addr.Resource,
					Provider: &provider,
					State:    &state,
					Output:   &state,
				},
				&EvalWriteStateDeposed{
					Name:         stateKey,
					ResourceType: n.Addr.Resource.Resource.Type,
					Provider:     n.ResolvedProvider.String(), // FIXME: Change underlying struct to use addrs.AbsProviderConfig itself
					State:        &state,
					Index:        n.Index,
				},
			},
		},
	})

	// Apply
	var diff *InstanceDiff
	var err error
	seq.Nodes = append(seq.Nodes, &EvalOpFilter{
		Ops: []walkOperation{walkApply, walkDestroy},
		Node: &EvalSequence{
			Nodes: []EvalNode{
				&EvalGetProvider{
					Addr:   n.ResolvedProvider,
					Output: &provider,
				},
				&EvalReadStateDeposed{
					Name:   stateKey,
					Output: &state,
					Index:  n.Index,
				},
				&EvalDiffDestroy{
					Addr:   addr.Resource,
					State:  &state,
					Output: &diff,
				},
				// Call pre-apply hook
				&EvalApplyPre{
					Addr:  addr.Resource,
					State: &state,
					Diff:  &diff,
				},
				&EvalApply{
					Addr:     addr.Resource,
					State:    &state,
					Diff:     &diff,
					Provider: &provider,
					Output:   &state,
					Error:    &err,
				},
				// Always write the resource back to the state deposed... if it
				// was successfully destroyed it will be pruned. If it was not, it will
				// be caught on the next run.
				&EvalWriteStateDeposed{
					Name:         stateKey,
					ResourceType: n.Addr.Resource.Resource.Type,
					Provider:     n.ResolvedProvider.String(), // FIXME: Change underlying struct to use addrs.AbsProviderConfig itself
					State:        &state,
					Index:        n.Index,
				},
				&EvalApplyPost{
					Addr:  addr.Resource,
					State: &state,
					Error: &err,
				},
				&EvalReturnError{
					Error: &err,
				},
				&EvalUpdateStateHook{},
			},
		},
	})

	return seq
}
<|endoftext|>


<commit_before>package pangu

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"regexp"
	"text/template"
)

const VERSION = "1.1.0"

// CJK is short for Chinese, Japanese and Korean.
//
// The constant cjk contains following Unicode blocks:
// 	\u2e80-\u2eff CJK Radicals Supplement
// 	\u2f00-\u2fdf Kangxi Radicals
// 	\u3040-\u309f Hiragana
// 	\u30a0-\u30ff Katakana
// 	\u3100-\u312f Bopomofo
// 	\u3200-\u32ff Enclosed CJK Letters and Months
// 	\u3400-\u4dbf CJK Unified Ideographs Extension A
// 	\u4e00-\u9fff CJK Unified Ideographs
// 	\uf900-\ufaff CJK Compatibility Ideographs
//
// For more information about Unicode blocks, see
// 	http://unicode-table.com/en/
const cjk = "" +
	"\u2e80-\u2eff" +
	"\u2f00-\u2fdf" +
	"\u3040-\u309f" +
	"\u30a0-\u30ff" +
	"\u3100-\u312f" +
	"\u3200-\u32ff" +
	"\u3400-\u4dbf" +
	"\u4e00-\u9fff" +
	"\uf900-\ufaff"

// ANS is short for Alphabets, Numbers
// and Symbols (`~!@#$%^&*()-_=+[]{}\|;:'",<.>/?).
//
// The constant ans doesn't contain all symbols above.
const ans = "A-Za-z0-9`\\$%\\^&\\*\\-=\\+\\\\|/\u00a1-\u00ff\u2022\u2027\u2150-\u218f"

var cjk_quote = regexp.MustCompile(re("([{{ .CJK }}])" + "([\"'])"))
var quote_cjk = regexp.MustCompile(re("([\"'])" + "([{{ .CJK }}])"))
var fix_quote = regexp.MustCompile(re("([\"'\\(\\[\\{<\u201c])" + "(\\s*)" + "(.+?)" + "(\\s*)" + "([\"'\\)\\]\\}>\u201d])"))
var fix_single_quote = regexp.MustCompile(re("([{{ .CJK }}])" + "( )" + "(')" + "([A-Za-z])"))

var cjk_hash = regexp.MustCompile(re("([{{ .CJK }}])" + "(#(\\S+))"))
var hash_cjk = regexp.MustCompile(re("((\\S+)#)" + "([{{ .CJK }}])"))

var cjk_operator_ans = regexp.MustCompile(re("([{{ .CJK }}])" + "([\\+\\-\\*/=&\\|<>])" + "([A-Za-z0-9])"))
var ans_operator_cjk = regexp.MustCompile(re("([A-Za-z0-9])" + "([\\+\\-\\*/=&\\|<>])" + "([{{ .CJK }}])"))

var cjk_bracket_cjk = regexp.MustCompile(re("([{{ .CJK }}])" + "([\\(\\[\\{<\u201c]+(.*?)[\\)\\]\\}>\u201d]+)" + "([{{ .CJK }}])"))
var cjk_bracket = regexp.MustCompile(re("([{{ .CJK }}])" + "([\\(\\[\\{<\u201c>])"))
var bracket_cjk = regexp.MustCompile(re("([\\)\\]\\}>\u201d<])" + "([{{ .CJK }}])"))
var fix_bracket = regexp.MustCompile(re("([\\(\\[\\{<\u201c]+)" + "(\\s*)" + "(.+?)" + "(\\s*)" + "([\\)\\]\\}>\u201d]+)"))

var fix_symbol = regexp.MustCompile(re("([{{ .CJK }}])" + "([~!;:,\\.\\?\u2026])" + "([A-Za-z0-9])"))

var cjk_ans = regexp.MustCompile(re("([{{ .CJK }}])([{{ .ANS }}@])"))
var ans_cjk = regexp.MustCompile(re("([{{ .ANS }}~!;:,\\.\\?\u2026])([{{ .CJK }}])"))

type pattern struct {
	CJK string
	ANS string
}

func re(exp string) string {
	var buf bytes.Buffer

	var tmpl = template.New("pangu")
	tmpl, _ = tmpl.Parse(exp)
	pat := pattern{
		CJK: cjk,
		ANS: ans,
	}
	tmpl.Execute(&buf, pat)
	expr := buf.String()

	return expr
}

// TextSpacing performs paranoid text spacing on text.
// It returns the processed text, with love.
func TextSpacing(text string) string {
	if len(text) < 2 {
		return text
	}

	text = cjk_quote.ReplaceAllString(text, "$1 $2")
	text = quote_cjk.ReplaceAllString(text, "$1 $2")
	text = fix_quote.ReplaceAllString(text, "$1$3$5")
	text = fix_single_quote.ReplaceAllString(text, "$1$3$4")

	text = cjk_hash.ReplaceAllString(text, "$1 $2")
	text = hash_cjk.ReplaceAllString(text, "$1 $3")

	text = cjk_operator_ans.ReplaceAllString(text, "$1 $2 $3")
	text = ans_operator_cjk.ReplaceAllString(text, "$1 $2 $3")

	oldText := text
	newText := cjk_bracket_cjk.ReplaceAllString(oldText, "$1 $2 $4")
	text = newText
	if oldText == newText {
		text = cjk_bracket.ReplaceAllString(text, "$1 $2")
		text = bracket_cjk.ReplaceAllString(text, "$1 $2")
	}
	text = fix_bracket.ReplaceAllString(text, "$1$3$5")

	text = fix_symbol.ReplaceAllString(text, "$1$2 $3")

	text = cjk_ans.ReplaceAllString(text, "$1 $2")
	text = ans_cjk.ReplaceAllString(text, "$1 $2")

	return text
}

// FileSpacing reads the file named by filename, performs paranoid text
// spacing on its contents and writes the processed content to w.
// A successful call returns err == nil.
func FileSpacing(filename string, w io.Writer) error {
	// filename, err := filepath.Abs(filename)
	// if err != nil {
	// 	return err
	// }

	fr, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer fr.Close()

	br := bufio.NewReader(fr)
	bw := bufio.NewWriter(w)

	for {
		line, err := br.ReadString('\n')
		if err == nil {
			fmt.Fprint(bw, TextSpacing(line))
		} else {
			if err == io.EOF {
				fmt.Fprint(bw, TextSpacing(line))
				break
			}
			return err
		}
	}
	defer bw.Flush()

	return nil
}
<commit_msg>name the return variable<commit_after>package pangu

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"regexp"
	"text/template"
)

const VERSION = "1.1.0"

// CJK is short for Chinese, Japanese and Korean.
//
// The constant cjk contains following Unicode blocks:
// 	\u2e80-\u2eff CJK Radicals Supplement
// 	\u2f00-\u2fdf Kangxi Radicals
// 	\u3040-\u309f Hiragana
// 	\u30a0-\u30ff Katakana
// 	\u3100-\u312f Bopomofo
// 	\u3200-\u32ff Enclosed CJK Letters and Months
// 	\u3400-\u4dbf CJK Unified Ideographs Extension A
// 	\u4e00-\u9fff CJK Unified Ideographs
// 	\uf900-\ufaff CJK Compatibility Ideographs
//
// For more information about Unicode blocks, see
// 	http://unicode-table.com/en/
const cjk = "" +
	"\u2e80-\u2eff" +
	"\u2f00-\u2fdf" +
	"\u3040-\u309f" +
	"\u30a0-\u30ff" +
	"\u3100-\u312f" +
	"\u3200-\u32ff" +
	"\u3400-\u4dbf" +
	"\u4e00-\u9fff" +
	"\uf900-\ufaff"

// ANS is short for Alphabets, Numbers
// and Symbols (`~!@#$%^&*()-_=+[]{}\|;:'",<.>/?).
//
// The constant ans doesn't contain all symbols above.
const ans = "A-Za-z0-9`\\$%\\^&\\*\\-=\\+\\\\|/\u00a1-\u00ff\u2022\u2027\u2150-\u218f"

var cjk_quote = regexp.MustCompile(re("([{{ .CJK }}])" + "([\"'])"))
var quote_cjk = regexp.MustCompile(re("([\"'])" + "([{{ .CJK }}])"))
var fix_quote = regexp.MustCompile(re("([\"'\\(\\[\\{<\u201c])" + "(\\s*)" + "(.+?)" + "(\\s*)" + "([\"'\\)\\]\\}>\u201d])"))
var fix_single_quote = regexp.MustCompile(re("([{{ .CJK }}])" + "( )" + "(')" + "([A-Za-z])"))

var cjk_hash = regexp.MustCompile(re("([{{ .CJK }}])" + "(#(\\S+))"))
var hash_cjk = regexp.MustCompile(re("((\\S+)#)" + "([{{ .CJK }}])"))

var cjk_operator_ans = regexp.MustCompile(re("([{{ .CJK }}])" + "([\\+\\-\\*/=&\\|<>])" + "([A-Za-z0-9])"))
var ans_operator_cjk = regexp.MustCompile(re("([A-Za-z0-9])" + "([\\+\\-\\*/=&\\|<>])" + "([{{ .CJK }}])"))

var cjk_bracket_cjk = regexp.MustCompile(re("([{{ .CJK }}])" + "([\\(\\[\\{<\u201c]+(.*?)[\\)\\]\\}>\u201d]+)" + "([{{ .CJK }}])"))
var cjk_bracket = regexp.MustCompile(re("([{{ .CJK }}])" + "([\\(\\[\\{<\u201c>])"))
var bracket_cjk = regexp.MustCompile(re("([\\)\\]\\}>\u201d<])" + "([{{ .CJK }}])"))
var fix_bracket = regexp.MustCompile(re("([\\(\\[\\{<\u201c]+)" + "(\\s*)" + "(.+?)" + "(\\s*)" + "([\\)\\]\\}>\u201d]+)"))

var fix_symbol = regexp.MustCompile(re("([{{ .CJK }}])" + "([~!;:,\\.\\?\u2026])" + "([A-Za-z0-9])"))

var cjk_ans = regexp.MustCompile(re("([{{ .CJK }}])([{{ .ANS }}@])"))
var ans_cjk = regexp.MustCompile(re("([{{ .ANS }}~!;:,\\.\\?\u2026])([{{ .CJK }}])"))

type pattern struct {
	CJK string
	ANS string
}

func re(exp string) string {
	var buf bytes.Buffer

	var tmpl = template.New("pangu")
	tmpl, _ = tmpl.Parse(exp)
	pat := pattern{
		CJK: cjk,
		ANS: ans,
	}
	tmpl.Execute(&buf, pat)
	expr := buf.String()

	return expr
}

// TextSpacing performs paranoid text spacing on text.
// It returns the processed text, with love.
func TextSpacing(text string) string {
	if len(text) < 2 {
		return text
	}

	text = cjk_quote.ReplaceAllString(text, "$1 $2")
	text = quote_cjk.ReplaceAllString(text, "$1 $2")
	text = fix_quote.ReplaceAllString(text, "$1$3$5")
	text = fix_single_quote.ReplaceAllString(text, "$1$3$4")

	text = cjk_hash.ReplaceAllString(text, "$1 $2")
	text = hash_cjk.ReplaceAllString(text, "$1 $3")

	text = cjk_operator_ans.ReplaceAllString(text, "$1 $2 $3")
	text = ans_operator_cjk.ReplaceAllString(text, "$1 $2 $3")

	oldText := text
	newText := cjk_bracket_cjk.ReplaceAllString(oldText, "$1 $2 $4")
	text = newText
	if oldText == newText {
		text = cjk_bracket.ReplaceAllString(text, "$1 $2")
		text = bracket_cjk.ReplaceAllString(text, "$1 $2")
	}
	text = fix_bracket.ReplaceAllString(text, "$1$3$5")

	text = fix_symbol.ReplaceAllString(text, "$1$2 $3")

	text = cjk_ans.ReplaceAllString(text, "$1 $2")
	text = ans_cjk.ReplaceAllString(text, "$1 $2")

	return text
}

// FileSpacing reads the file named by filename, performs paranoid text
// spacing on its contents and writes the processed content to w.
// A successful call returns err == nil.
func FileSpacing(filename string, w io.Writer) (err error) {
	// filename, err := filepath.Abs(filename)
	// if err != nil {
	// 	return err
	// }

	fr, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer fr.Close()

	br := bufio.NewReader(fr)
	bw := bufio.NewWriter(w)

	for {
		line, err := br.ReadString('\n')
		if err == nil {
			fmt.Fprint(bw, TextSpacing(line))
		} else {
			if err == io.EOF {
				fmt.Fprint(bw, TextSpacing(line))
				break
			}
			return err
		}
	}
	defer bw.Flush()

	return nil
}
<|endoftext|>


<commit_before>// Package panos interacts with Palo Alto firewalls using the REST API.
package panos

import (
	"encoding/xml"
	"errors"
	"fmt"
	"github.com/scottdware/go-rested"
)

// PaloAlto is a container for our session state.
type PaloAlto struct {
	Host string
	Key  string
}

// authKey holds our API key.
type authKey struct {
	XMLName xml.Name `xml:"response"`
	Key     string   `xml:"result>key"`
}

// requestError contains information about any error we get from a request.
type requestError struct {
	XMLName xml.Name `xml:"response"`
	Message string   `xml:"msg>line,omitempty"`
}

// NewSession sets up our connection to the Palo Alto firewall system.
func NewSession(host, user, passwd string) *PaloAlto {
	var key authKey

	resp := rested.Send(fmt.Sprintf("https://%s/api/?type=keygen&user=%s&password=%s", host, user, passwd), nil)
	if resp.Error != nil {
		fmt.Println(resp.Error)
	}

	err := xml.Unmarshal(resp.Body, &key)
	if err != nil {
		fmt.Println(err)
	}

	return &PaloAlto{
		Host: host,
		Key:  key.Key,
	}
}

// checkError handles any errors we get from our API requests. It returns either the
// message of the error, if any, or nil.
func (p *PaloAlto) checkError(resp []byte) error {
	var reqError requestError

	err := xml.Unmarshal(resp, &reqError)
	if err != nil {
		return err
	}

	if reqError.Message != "" {
		return errors.New(reqError.Message)
	}

	return nil
}
<commit_msg>Added URL field<commit_after>// Package panos interacts with Palo Alto firewalls using the REST API.
package panos

import (
	"encoding/xml"
	"errors"
	"fmt"
	"github.com/scottdware/go-rested"
)

// PaloAlto is a container for our session state.
type PaloAlto struct {
	Host string
	Key  string
	URI  string
}

// authKey holds our API key.
type authKey struct {
	XMLName xml.Name `xml:"response"`
	Key     string   `xml:"result>key"`
}

// requestError contains information about any error we get from a request.
type requestError struct {
	XMLName xml.Name `xml:"response"`
	Message string   `xml:"msg>line,omitempty"`
}

// NewSession sets up our connection to the Palo Alto firewall system.
func NewSession(host, user, passwd string) *PaloAlto {
	var key authKey

	resp := rested.Send(fmt.Sprintf("https://%s/api/?type=keygen&user=%s&password=%s", host, user, passwd), nil)
	if resp.Error != nil {
		fmt.Println(resp.Error)
	}

	err := xml.Unmarshal(resp.Body, &key)
	if err != nil {
		fmt.Println(err)
	}

	return &PaloAlto{
		Host: host,
		Key:  key.Key,
		URI:  fmt.Sprintf("https://%s/api/?", host),
	}
}

// checkError handles any errors we get from our API requests. It returns either the
// message of the error, if any, or nil.
func (p *PaloAlto) checkError(resp []byte) error {
	var reqError requestError

	err := xml.Unmarshal(resp, &reqError)
	if err != nil {
		return err
	}

	if reqError.Message != "" {
		return errors.New(reqError.Message)
	}

	return nil
}
<|endoftext|>


<commit_before>package gosfdoc

import (
	"bytes"
	"regexp"
	// "fmt"
)

var (
	_defaultAbout = []byte(`
## About
------

gosfdoc document generator

More references: [https://github.com/slowfei/gosfdoc][0]

The MIT license (MIT) - [http://opensource.org/licenses/MIT][1]

Copyright (c) 2014 slowfei

Email: slowfei#foxmail.com

[0]:https://github.com/slowfei/gosfdoc
[1]:http://opensource.org/licenses/MIT
`)

	_defaultIntro = []byte(`
##Document Introduction

Sorry! Document author did not write any information.

----
This is a good tool, Can help you make beautiful documents.

More references: [https://github.com/slowfei/gosfdoc][0]

[0]:https://github.com/slowfei/gosfdoc
`)

	//  主要用于去除注释的前缀
	_prefixFilterTags = [][]byte{
		[]byte(" *\t"),
		[]byte(" *  "),
		[]byte(" * "),
		[]byte("//\t"),
		[]byte("//  "),
		[]byte("// "),
		[]byte("//"),
	}

	_tagStar   = []byte("*")
	_tagDSlash = []byte("//")
)

/**
 *  commons parse file about content
 *
 *  @param `fileBuf`
 *  @return about content
 */
func ParseAbout(fileBuf *FileBuf) []byte {
	return parseAboutAndIntro(fileBuf, REXAbout)
}

/**
 *  commons parse file introduction content
 *
 *  @param `fileBuf`
 *  @return introduction content
 */
func ParseIntro(fileBuf *FileBuf) []byte {
	return parseAboutAndIntro(fileBuf, REXIntro)
}

/**
 *  commons about intro parse
 *
 *  @param fileBuf
 *  @param rex
 */
func parseAboutAndIntro(fileBuf *FileBuf, rex *regexp.Regexp) []byte {
	var result []byte = nil
	var prefixTag []byte = nil
	prefixLen := 0

	aboutBuf := fileBuf.Find(rex)

	if 0 < len(aboutBuf) {
		appendLine := bytes.NewBuffer(nil)

		lines := bytes.Split(aboutBuf, []byte("\n"))
		linesCount := len(lines)

		for i := 1; i < linesCount-1; i++ {
			newLine := lines[i]

			if i == 1 {
				//  记录第一个前缀的标识，以第一个为准，后面的根据要求都要是相同的注释前缀。
				/**
				  (*)remove prefix tag
				  (*)
				  (*)
				*/
				prefixTag = findPrefixFilterTag(newLine)
				prefixLen = len(prefixTag)
			}

			if 0 != len(prefixTag) {

				if 0 == bytes.Index(newLine, prefixTag) {
					appendLine.Write(newLine[prefixLen:])
				} else {
					trimed := bytes.TrimSpace(newLine)
					//  有可能存在空行，所以需要过滤只存在 "*" || "//"
					if !bytes.Equal(trimed, _tagStar) && !bytes.Equal(trimed, _tagDSlash) {
						appendLine.Write(newLine)
					}
				}

			} else {
				appendLine.Write(newLine)
			}

			appendLine.WriteByte('\n')
		}

		if 0 < appendLine.Len() {
			result = appendLine.Bytes()
		}
	}

	return result
}

/**
 *  find prefix filter tag index
 *  see var _prefixFilterTags
 */
func findPrefixFilterTag(src []byte) []byte {
	var pftCount = len(_prefixFilterTags)

	for i := 0; i < pftCount; i++ {
		prefix := _prefixFilterTags[i]
		if 0 == bytes.Index(src, prefix) {
			return prefix
		}
	}

	return nil
}
<commit_msg>增加document解析（未实现）<commit_after>package gosfdoc

import (
	"bytes"
	"regexp"
	// "fmt"
)

var (
	_defaultAbout = []byte(`
## About
------

gosfdoc document generator

More references: [https://github.com/slowfei/gosfdoc][0]

The MIT license (MIT) - [http://opensource.org/licenses/MIT][1]

Copyright (c) 2014 slowfei

Email: slowfei#foxmail.com

[0]:https://github.com/slowfei/gosfdoc
[1]:http://opensource.org/licenses/MIT
`)

	_defaultIntro = []byte(`
##Document Introduction

Sorry! Document author did not write any information.

----
This is a good tool, Can help you make beautiful documents.

More references: [https://github.com/slowfei/gosfdoc][0]

[0]:https://github.com/slowfei/gosfdoc
`)

	//  主要用于去除注释的前缀
	_prefixFilterTags = [][]byte{
		[]byte(" *\t"),
		[]byte(" *  "),
		[]byte(" * "),
		[]byte("//\t"),
		[]byte("//  "),
		[]byte("// "),
		[]byte("//"),
	}

	_tagStar   = []byte("*")
	_tagDSlash = []byte("//")
)

/**
 *	parse public document content
 *
 *	@param `fileBuf`
 *	@return document array
 */
func ParseDocument(fileBuf *FileBuf) []Document {
	return nil
}

/**
 *  commons parse file about content
 *
 *  @param `fileBuf`
 *  @return about content
 */
func ParseAbout(fileBuf *FileBuf) []byte {
	return parseAboutAndIntro(fileBuf, REXAbout)
}

/**
 *  commons parse file introduction content
 *
 *  @param `fileBuf`
 *  @return introduction content
 */
func ParseIntro(fileBuf *FileBuf) []byte {
	return parseAboutAndIntro(fileBuf, REXIntro)
}

/**
 *  commons about intro parse
 *
 *  @param fileBuf
 *  @param rex
 */
func parseAboutAndIntro(fileBuf *FileBuf, rex *regexp.Regexp) []byte {
	var result []byte = nil
	var prefixTag []byte = nil
	prefixLen := 0

	aboutBuf := fileBuf.Find(rex)

	if 0 < len(aboutBuf) {
		appendLine := bytes.NewBuffer(nil)

		lines := bytes.Split(aboutBuf, []byte("\n"))
		linesCount := len(lines)

		for i := 1; i < linesCount-1; i++ {
			newLine := lines[i]

			if i == 1 {
				//  记录第一个前缀的标识，以第一个为准，后面的根据要求都要是相同的注释前缀。
				/**
				  (*)remove prefix tag
				  (*)
				  (*)
				*/
				prefixTag = findPrefixFilterTag(newLine)
				prefixLen = len(prefixTag)
			}

			if 0 != len(prefixTag) {

				if 0 == bytes.Index(newLine, prefixTag) {
					appendLine.Write(newLine[prefixLen:])
				} else {
					trimed := bytes.TrimSpace(newLine)
					//  有可能存在空行，所以需要过滤只存在 "*" || "//"
					if !bytes.Equal(trimed, _tagStar) && !bytes.Equal(trimed, _tagDSlash) {
						appendLine.Write(newLine)
					}
				}

			} else {
				appendLine.Write(newLine)
			}

			appendLine.WriteByte('\n')
		}

		if 0 < appendLine.Len() {
			result = appendLine.Bytes()
		}
	}

	return result
}

/**
 *  find prefix filter tag index
 *  see var _prefixFilterTags
 */
func findPrefixFilterTag(src []byte) []byte {
	var pftCount = len(_prefixFilterTags)

	for i := 0; i < pftCount; i++ {
		prefix := _prefixFilterTags[i]
		if 0 == bytes.Index(src, prefix) {
			return prefix
		}
	}

	return nil
}
<|endoftext|>


<commit_before>/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/sbinet/go-clang"
)

type Parser struct {
	cas map[string][]string
}

/*
 * There is so much path manipulation in the construction of the compilation
 * aguments database that I think this deserves a long explanation. Compilation
 * database (compile_command.json) provides absolute path of the file with its
 * compilation options. We are storing this compilation options/arguments in
 * the cas field of the Parser struct to be used during parsing. This is a map
 * of file name to list of arguments. The name file should match the one
 * returned by the directory traversing in main, i.e., the minimum relative
 * path of the file (the path returned by filepath.Clean) or the absolute path
 * depending on the input. For each input directory (provided in the command
 * line) we try to read the compile command database from disk. For each of the
 * file path read, we fix the full path to match the relative or absolute path
 * of the input (fixPaths) and clean it with filepath.Clean.
 *
 * Then, we need to make sure that the directories in the -I options also match
 * the relative or absolute path from the input. This is fixed in fixCompDirArg
 * right before populating the arguments for some specific file.
 */

type compArgs struct {
	Directory string
	Command   string
	File      string
}

func fixPaths(cas []compArgs, path string) {
	// first, find absolute path of @path
	if filepath.IsAbs(path) {
		return
	}

	wd, err := os.Getwd()
	if err != nil {
		log.Panic("unable to get working directoy: ", err)
	}

	// second, replace absolute path with relative path and clean
	for i := range cas {
		ca := &cas[i]
		rel, err := filepath.Rel(wd, ca.File)
		if err != nil {
			log.Panic("unable to get relative path: ", err)
		}
		ca.File = filepath.Clean(rel)
	}
}

func fixCompDirArg(argDir, path string) string {
	if filepath.IsAbs(path) {
		if filepath.IsAbs(argDir) {
			return argDir
		}

		abs, err := filepath.Abs(argDir)
		if err != nil {
			log.Panic("unable to get absolute path: ",
				err)
		}
		return filepath.Clean(abs)
	}
	if filepath.IsAbs(argDir) {
		wd, err := os.Getwd()
		if err != nil {
			log.Panic("unable to get working directoy: ",
				err)
		}
		rel, err := filepath.Rel(wd, argDir)
		if err != nil {
			log.Panic("unable to get relative path: ",
				err)
		}
		return filepath.Clean(rel)
	}
	return filepath.Clean(path + "/" + argDir)
}

func getCompArgs(command, path string) []string {
	args := []string{}

	argsList := strings.Fields(command)

	for i, arg := range argsList {
		switch {
		case arg == "-D":
			args = append(args, arg, argsList[i+1])
		case strings.HasPrefix(arg, "-D"):
			args = append(args, arg)
		case arg == "-I":
			argDir := fixCompDirArg(argsList[i+1], path)
			args = append(args, "-I", argDir)
		case strings.HasPrefix(arg, "-I"):
			argDir := fixCompDirArg(
				strings.Replace(arg, "-I", "", 1),
				path)
			args = append(args, "-I", argDir)
		}
	}

	return args
}

func NewParser(inputDirs []string) *Parser {
	ret := &Parser{make(map[string][]string)}

	// read compilation args db and fix files paths
	for _, path := range inputDirs {
		f, err := os.Open(path + "/compile_commands.json")
		if os.IsPermission(err) {
			log.Panic("error opening compile db: ", err)
		} else if err != nil {
			continue
		}
		defer f.Close()

		dec := json.NewDecoder(f)
		var cas []compArgs
		err = dec.Decode(&cas)
		if err != nil {
			log.Panic(err)
		}

		fixPaths(cas, path)

		// index compArgs by file names
		for _, ca := range cas {
			ret.cas[ca.File] = getCompArgs(ca.Command, path)
		}
	}

	return ret
}

func getSymbolFromCursor(db *TUSymbolsDB, cursor *clang.Cursor) *SymbolInfo {
	if cursor.IsNull() {
		return nil
	}

	f, line, col, _ := cursor.Location().GetFileLocation()
	fName := filepath.Clean(f.Name())
	return &SymbolInfo{
		name: cursor.Spelling(),
		usr:  cursor.USR(),
		loc: SymbolLocReq{
			fName,
			int(line),
			int(col),
		},
	}
}

func (pa *Parser) Parse(file string) *TUSymbolsDB {
	idx := clang.NewIndex(0, 0)
	defer idx.Dispose()

	args, ok := pa.cas[file]
	if !ok {
		args = []string{}
	}
	tu := idx.Parse(file, args, nil, clang.TU_DetailedPreprocessingRecord)
	defer tu.Dispose()

	db := NewTUSymbolsDB(file, tu.File(file).ModTime())
	defer db.TempSaveDB()

	visitNode := func(cursor, parent clang.Cursor) clang.ChildVisitResult {
		if cursor.IsNull() {
			return clang.CVR_Continue
		}

		cur := getSymbolFromCursor(db, &cursor)
		curFile := cur.loc.File

		if curFile == "" || curFile == "." {
			// ignore system code
			return clang.CVR_Continue
		}

		// TODO: erase! this is not required
		if false {
			log.Printf("%s: %s (%s)\n",
				cursor.Kind().Spelling(),
				cursor.Spelling(),
				cursor.USR())
			log.Println(curFile, ":", cur.loc.Line, cur.loc.Col)
		}
		////////////////////////////////////

		switch cursor.Kind() {
		case clang.CK_FunctionDecl:
			defCursor := cursor.DefinitionCursor()
			if !defCursor.IsNull() {
				def := getSymbolFromCursor(db, &defCursor)
				db.InsertSymbolDeclWithDef(cur, def)
			} else {
				db.InsertSymbolDecl(cur)
			}
		case clang.CK_VarDecl:
			db.InsertSymbolDecl(cur)
		case clang.CK_ParmDecl:
			if cursor.Spelling() != "" {
				db.InsertSymbolDecl(cur)
			}
		case clang.CK_CallExpr:
			decCursor := cursor.Referenced()
			dec := getSymbolFromCursor(db, &decCursor)
			db.InsertSymbolUse(cur, dec, true)
		case clang.CK_DeclRefExpr:
			decCursor := cursor.Referenced()
			dec := getSymbolFromCursor(db, &decCursor)
			db.InsertSymbolUse(cur, dec, false)
		case clang.CK_InclusionDirective:
			incFile := cursor.IncludedFile()
			db.InsertHeader(cursor.Spelling(), incFile)
		}

		// TODO: eventually we need to continue on some cases for
		// faster run
		return clang.CVR_Recurse
	}

	tu.ToCursor().Visit(visitNode)

	return db
}
<commit_msg>DB not necessary to get SymbolInfo in parser<commit_after>/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/sbinet/go-clang"
)

type Parser struct {
	cas map[string][]string
}

/*
 * There is so much path manipulation in the construction of the compilation
 * aguments database that I think this deserves a long explanation. Compilation
 * database (compile_command.json) provides absolute path of the file with its
 * compilation options. We are storing this compilation options/arguments in
 * the cas field of the Parser struct to be used during parsing. This is a map
 * of file name to list of arguments. The name file should match the one
 * returned by the directory traversing in main, i.e., the minimum relative
 * path of the file (the path returned by filepath.Clean) or the absolute path
 * depending on the input. For each input directory (provided in the command
 * line) we try to read the compile command database from disk. For each of the
 * file path read, we fix the full path to match the relative or absolute path
 * of the input (fixPaths) and clean it with filepath.Clean.
 *
 * Then, we need to make sure that the directories in the -I options also match
 * the relative or absolute path from the input. This is fixed in fixCompDirArg
 * right before populating the arguments for some specific file.
 */

type compArgs struct {
	Directory string
	Command   string
	File      string
}

func fixPaths(cas []compArgs, path string) {
	// first, find absolute path of @path
	if filepath.IsAbs(path) {
		return
	}

	wd, err := os.Getwd()
	if err != nil {
		log.Panic("unable to get working directoy: ", err)
	}

	// second, replace absolute path with relative path and clean
	for i := range cas {
		ca := &cas[i]
		rel, err := filepath.Rel(wd, ca.File)
		if err != nil {
			log.Panic("unable to get relative path: ", err)
		}
		ca.File = filepath.Clean(rel)
	}
}

func fixCompDirArg(argDir, path string) string {
	if filepath.IsAbs(path) {
		if filepath.IsAbs(argDir) {
			return argDir
		}

		abs, err := filepath.Abs(argDir)
		if err != nil {
			log.Panic("unable to get absolute path: ",
				err)
		}
		return filepath.Clean(abs)
	}
	if filepath.IsAbs(argDir) {
		wd, err := os.Getwd()
		if err != nil {
			log.Panic("unable to get working directoy: ",
				err)
		}
		rel, err := filepath.Rel(wd, argDir)
		if err != nil {
			log.Panic("unable to get relative path: ",
				err)
		}
		return filepath.Clean(rel)
	}
	return filepath.Clean(path + "/" + argDir)
}

func getCompArgs(command, path string) []string {
	args := []string{}

	argsList := strings.Fields(command)

	for i, arg := range argsList {
		switch {
		case arg == "-D":
			args = append(args, arg, argsList[i+1])
		case strings.HasPrefix(arg, "-D"):
			args = append(args, arg)
		case arg == "-I":
			argDir := fixCompDirArg(argsList[i+1], path)
			args = append(args, "-I", argDir)
		case strings.HasPrefix(arg, "-I"):
			argDir := fixCompDirArg(
				strings.Replace(arg, "-I", "", 1),
				path)
			args = append(args, "-I", argDir)
		}
	}

	return args
}

func NewParser(inputDirs []string) *Parser {
	ret := &Parser{make(map[string][]string)}

	// read compilation args db and fix files paths
	for _, path := range inputDirs {
		f, err := os.Open(path + "/compile_commands.json")
		if os.IsPermission(err) {
			log.Panic("error opening compile db: ", err)
		} else if err != nil {
			continue
		}
		defer f.Close()

		dec := json.NewDecoder(f)
		var cas []compArgs
		err = dec.Decode(&cas)
		if err != nil {
			log.Panic(err)
		}

		fixPaths(cas, path)

		// index compArgs by file names
		for _, ca := range cas {
			ret.cas[ca.File] = getCompArgs(ca.Command, path)
		}
	}

	return ret
}

func getSymbolFromCursor(cursor *clang.Cursor) *SymbolInfo {
	if cursor.IsNull() {
		return nil
	}

	f, line, col, _ := cursor.Location().GetFileLocation()
	fName := filepath.Clean(f.Name())
	return &SymbolInfo{
		name: cursor.Spelling(),
		usr:  cursor.USR(),
		loc: SymbolLocReq{
			fName,
			int(line),
			int(col),
		},
	}
}

func (pa *Parser) Parse(file string) *TUSymbolsDB {
	idx := clang.NewIndex(0, 0)
	defer idx.Dispose()

	args, ok := pa.cas[file]
	if !ok {
		args = []string{}
	}
	tu := idx.Parse(file, args, nil, clang.TU_DetailedPreprocessingRecord)
	defer tu.Dispose()

	db := NewTUSymbolsDB(file, tu.File(file).ModTime())
	defer db.TempSaveDB()

	visitNode := func(cursor, parent clang.Cursor) clang.ChildVisitResult {
		if cursor.IsNull() {
			return clang.CVR_Continue
		}

		cur := getSymbolFromCursor(&cursor)
		curFile := cur.loc.File

		if curFile == "" || curFile == "." {
			// ignore system code
			return clang.CVR_Continue
		}

		// TODO: erase! this is not required
		if false {
			log.Printf("%s: %s (%s)\n",
				cursor.Kind().Spelling(),
				cursor.Spelling(),
				cursor.USR())
			log.Println(curFile, ":", cur.loc.Line, cur.loc.Col)
		}
		////////////////////////////////////
		switch cursor.Kind() {
		case clang.CK_FunctionDecl:
			defCursor := cursor.DefinitionCursor()
			if !defCursor.IsNull() {
				def := getSymbolFromCursor(&defCursor)
				db.InsertSymbolDeclWithDef(cur, def)
			} else {
				db.InsertSymbolDecl(cur)
			}
		case clang.CK_VarDecl:
			db.InsertSymbolDecl(cur)
		case clang.CK_ParmDecl:
			if cursor.Spelling() != "" {
				db.InsertSymbolDecl(cur)
			}
		case clang.CK_CallExpr:
			decCursor := cursor.Referenced()
			dec := getSymbolFromCursor(&decCursor)
			db.InsertSymbolUse(cur, dec, true)
		case clang.CK_DeclRefExpr:
			decCursor := cursor.Referenced()
			dec := getSymbolFromCursor(&decCursor)
			db.InsertSymbolUse(cur, dec, false)
		case clang.CK_InclusionDirective:
			incFile := cursor.IncludedFile()
			db.InsertHeader(cursor.Spelling(), incFile)
		}

		// TODO: eventually we need to continue on some cases for
		// faster run
		return clang.CVR_Recurse
	}

	tu.ToCursor().Visit(visitNode)

	return db
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

type parser struct {
	lex    *lexer
	peeks  [2]token
	npeeks int
}

func (p *parser) next() token {
	if p.npeeks == 0 {
		return p.lex.token()
	}
	t := p.peeks[0]
	for i := 1; i < p.npeeks; i++ {
		p.peeks[i-1] = p.peeks[i]
	}
	p.npeeks--
	return t
}

func parse(lex *lexer) *parser {
	return &parser{
		lex: lex,
	}
}

// peek at the nth token
func (p *parser) peekn(n int) token {
	if n > len(p.peeks) {
		panic("Too much peeking in the parser")
	}
	for ; p.npeeks < n; p.npeeks++ {
		p.peeks[p.npeeks] = p.lex.token()
	}
	return p.peeks[n-1]
}

func (p *parser) peek() token {
	return p.peekn(1)
}

func (p *parser) junk(n int) {
	for i := 0; i < n; i++ {
		p.next()
	}
}

func (p *parser) accept(typ ttype) (t token, ok bool) {
	if p.peek().typ == typ {
		t = p.next()
		ok = true
	}
	return
}

func (p *parser) acceptNamedList(name string) bool {
	if p.peek().typ != tokOpen || p.peekn(2).txt != name {
		return false
	}
	p.junk(2)
	return true
}

func (p *parser) errorf(format string, args ...interface{}) {
	pre := fmt.Sprintf("%s:%d", p.lex.name, p.lex.lineno)
	suf := fmt.Sprintf(format, args...)
	panic(fmt.Errorf("%s: %s", pre, suf))
}

func (p *parser) expect(typ ttype) token {
	t := p.peek()
	if t.typ != typ {
		p.errorf("expected %v, got %v", typ, t)
	}
	return p.next()
}

func (p *parser) expectId(s string) token {
	t := p.peek()
	typ := tokId
	if s[0] == ':' {
		typ = tokCid
	} else if s[0] == '?' {
		typ = tokQid
	}
	if t.typ != typ || t.txt != s {
		p.errorf("expected identifier [\"%s\"], got %v", s, t)
	}
	return p.next()
}

func (p *parser) parseDomain() *domain {
	p.expect(tokOpen)
	p.expectId("define")
	d := &domain{
		name:   p.parseDomainName(),
		reqs:   p.parseReqsDef(),
		types:  p.parseTypesDef(),
		consts: p.parseConstsDef(),
		preds:  p.parsePredsDef(),
		acts:   make([]action, 0),
	}
	// Ignore :functions for now
	if p.acceptNamedList(":functions") {
		for nesting := 1; nesting > 0; {
			switch p.next().typ {
			case tokClose:
				nesting--
			case tokOpen:
				nesting++
			}
		}
	}
	for p.peek().typ == tokOpen {
		d.acts = append(d.acts, p.parseActionDef())
	}

	//p.expect(tokClose)
	return d
}

func (p *parser) parseDomainName() string {
	p.expect(tokOpen)
	p.expectId("domain")
	n := p.expect(tokId)
	p.expect(tokClose)
	return n.txt
}

func (p *parser) parseReqsDef() (reqs []string) {
	reqs = make([]string, 0)
	if !p.acceptNamedList(":requirements") {
		return
	}
	for t, ok := p.accept(tokCid); ok; t, ok = p.accept(tokCid) {
		reqs = append(reqs, t.txt)
	}
	p.expect(tokClose)
	return
}

func (p *parser) parseTypesDef() (types []tname) {
	types = make([]tname, 0)
	if !p.acceptNamedList(":types") {
		return
	}
	types = p.parseTypedListString(tokId)
	p.expect(tokClose)
	return
}

func (p *parser) parseConstsDef() (consts []tname) {
	consts = make([]tname, 0)
	if !p.acceptNamedList(":constants") {
		return
	}
	consts = p.parseTypedListString(tokId)
	p.expect(tokClose)
	return
}

func (p *parser) parsePredsDef() (preds []pred) {
	preds = make([]pred, 0)
	if !p.acceptNamedList(":predicates") {
		return
	}
	preds = append(preds, p.parseAtomicFormSkele())
	for p.peek().typ == tokOpen {
		preds = append(preds, p.parseAtomicFormSkele())
	}
	p.expect(tokClose)
	return
}

func (p *parser) parseAtomicFormSkele() pred {
	p.expect(tokOpen)
	pred := pred{
		name:  p.expect(tokId).txt,
		parms: p.parseTypedListString(tokQid),
	}
	p.expect(tokClose)
	return pred
}

func (p *parser) parseActionDef() action {
	p.expect(tokOpen)
	p.expectId(":action")

	act := action{name: p.expect(tokId).txt, parms: p.parseActParms()}

	if p.peek().txt == ":precondition" {
		p.junk(1)
		if p.peek().typ == tokOpen && p.peekn(2).typ == tokClose {
			p.junk(2)
		} else {
			act.prec = p.parsePreGd()
		}
	}
	if p.peek().txt == ":effect" {
		p.junk(1)
		if p.peek().typ == tokOpen && p.peekn(2).typ == tokClose {
			p.junk(2)
		} else {
			act.effect = p.parseEffect()
		}
	}

	p.expect(tokClose)
	return act
}

func (p *parser) parseActParms() []tname {
	p.expectId(":parameters")
	p.expect(tokOpen)
	res := p.parseTypedListString(tokQid)
	p.expect(tokClose)
	return res
}

func (p *parser) parsePreGd() (res *gd) {
	parseNested := func(p *parser) *gd { return p.parsePreGd() }
	switch {
	case p.acceptNamedList("and"):
		res = p.parseAndGd(parseNested)
	case p.acceptNamedList("forall"):
		res = p.parseQuantGd(gdForall, parseNested)
	default:
		res = p.parsePrefGd()
	}
	return
}

func (p *parser) parsePrefGd() *gd {
	return p.parseGd()
}

func (p *parser) parseGd() (res *gd) {
	parseNested := func(p *parser) *gd { return p.parseGd() }
	switch {
	case p.acceptNamedList("and"):
		res = p.parseAndGd(parseNested)
	case p.acceptNamedList("or"):
		res = p.parseOrGd(parseNested)
	case p.acceptNamedList("not"):
		res = &gd{typ: gdNot, left: p.parseGd()}
		p.expect(tokClose)
	case p.acceptNamedList("imply"):
		res = &gd{
			typ:   gdOr,
			left:  &gd{typ: gdNot, left: p.parseGd()},
			right: p.parseGd(),
		}
		p.expect(tokClose)
	case p.acceptNamedList("exists"):
		res = p.parseQuantGd(gdExists, parseNested)
	case p.acceptNamedList("forall"):
		res = p.parseQuantGd(gdForall, parseNested)
	default:
		p.expect(tokOpen)
		res = &gd{
			typ:   gdPred,
			name:  p.expect(tokId).txt,
			parms: p.parseTerms(),
		}
		p.expect(tokClose)
	}
	return
}

func (p *parser) parseAndGd(nested func(*parser) *gd) *gd {
	conj := make([]*gd, 0)
	for p.peek().typ == tokOpen {
		conj = append(conj, nested(p))
	}
	res := seqAnd(conj)
	p.expect(tokClose)
	return res
}

func seqAnd(conj []*gd) (res *gd) {
	switch len(conj) {
	case 0:
		res = &gd{typ: gdTrue}
	case 1:
		res = conj[0]
	default:
		res = &gd{typ: gdAnd, left: conj[0], right: seqAnd(conj[1:])}
	}
	return
}

func (p *parser) parseOrGd(nested func(*parser) *gd) *gd {
	disj := make([]*gd, 0)
	for p.peek().typ == tokOpen  {
		disj = append(disj, nested(p))
	}
	res := seqOr(disj)

	p.expect(tokClose)
	return res
}

func seqOr(disj []*gd) (res *gd) {
	switch len(disj) {
	case 0:
		res = &gd{typ: gdFalse}
	case 1:
		res = disj[0]
	default:
		res = &gd{typ: gdOr, left: disj[0], right: seqOr(disj[1:])}
	}
	return
}

func (p *parser) parseQuantGd(q gdtype, nested func(*parser) *gd) *gd {
	res := &gd{typ: gdForall}

	p.expect(tokOpen)
	vrs := p.parseTypedListString(tokQid)
	p.expect(tokClose)

	bottom := res
	for i, vr := range vrs {
		bottom.vr = vr
		if i < len(vrs)-1 {
			bottom.left = &gd{typ: q}
			bottom = bottom.left
		}
	}

	bottom.left = nested(p)
	p.expect(tokClose)
	return res
}

func (p *parser) parseTerms() []string {
	lst := make([]string, 0)
	for {
		if t, ok := p.accept(tokId); ok {
			lst = append(lst, t.txt)
			continue
		}
		if t, ok := p.accept(tokQid); ok {
			lst = append(lst, t.txt)
			continue
		}
		break
	}
	return lst

}

func (p *parser) parseEffect() *effect {
	// ignore
	p.expect(tokOpen)
	for nesting := 1; nesting > 0; {
		switch p.next().typ {
		case tokClose:
			nesting--
		case tokOpen:
			nesting++
		}
	}
	return nil
}

func (p *parser) parseTypedListString(typ ttype) []tname {
	lst := make([]tname, 0)

	for {
		names := p.parseStrings(typ)
		if len(names) == 0 {
			break
		}
		typ := p.parseType()
		for _, n := range names {
			lst = append(lst, tname{name: n, typ: typ})
		}
	}
	return lst
}

func (p *parser) parseType() []string {
	if _, ok := p.accept(tokMinus); !ok {
		return []string{"object"}
	}
	if _, ok := p.accept(tokOpen); ok {
		p.expectId("either")
		lst := p.parseStringPlus(tokId)
		p.expect(tokClose)
		return lst
	}
	t := p.expect(tokId)
	return []string{t.txt}
}

func (p *parser) parseStringPlus(typ ttype) []string {
	lst := []string{p.expect(typ).txt}
	lst = append(lst, p.parseStrings(typ)...)
	return lst
}

func (p *parser) parseStrings(typ ttype) []string {
	lst := make([]string, 0)
	for t, ok := p.accept(typ); ok; t, ok = p.accept(typ) {
		lst = append(lst, t.txt)
	}
	return lst
}
<commit_msg>Expect the closed paren on domain definitions.<commit_after>package main

import (
	"fmt"
)

type parser struct {
	lex    *lexer
	peeks  [2]token
	npeeks int
}

func (p *parser) next() token {
	if p.npeeks == 0 {
		return p.lex.token()
	}
	t := p.peeks[0]
	for i := 1; i < p.npeeks; i++ {
		p.peeks[i-1] = p.peeks[i]
	}
	p.npeeks--
	return t
}

func parse(lex *lexer) *parser {
	return &parser{
		lex: lex,
	}
}

// peek at the nth token
func (p *parser) peekn(n int) token {
	if n > len(p.peeks) {
		panic("Too much peeking in the parser")
	}
	for ; p.npeeks < n; p.npeeks++ {
		p.peeks[p.npeeks] = p.lex.token()
	}
	return p.peeks[n-1]
}

func (p *parser) peek() token {
	return p.peekn(1)
}

func (p *parser) junk(n int) {
	for i := 0; i < n; i++ {
		p.next()
	}
}

func (p *parser) accept(typ ttype) (t token, ok bool) {
	if p.peek().typ == typ {
		t = p.next()
		ok = true
	}
	return
}

func (p *parser) acceptNamedList(name string) bool {
	if p.peek().typ != tokOpen || p.peekn(2).txt != name {
		return false
	}
	p.junk(2)
	return true
}

func (p *parser) errorf(format string, args ...interface{}) {
	pre := fmt.Sprintf("%s:%d", p.lex.name, p.lex.lineno)
	suf := fmt.Sprintf(format, args...)
	panic(fmt.Errorf("%s: %s", pre, suf))
}

func (p *parser) expect(typ ttype) token {
	t := p.peek()
	if t.typ != typ {
		p.errorf("expected %v, got %v", typ, t)
	}
	return p.next()
}

func (p *parser) expectId(s string) token {
	t := p.peek()
	typ := tokId
	if s[0] == ':' {
		typ = tokCid
	} else if s[0] == '?' {
		typ = tokQid
	}
	if t.typ != typ || t.txt != s {
		p.errorf("expected identifier [\"%s\"], got %v", s, t)
	}
	return p.next()
}

func (p *parser) parseDomain() *domain {
	p.expect(tokOpen)
	p.expectId("define")
	d := &domain{
		name:   p.parseDomainName(),
		reqs:   p.parseReqsDef(),
		types:  p.parseTypesDef(),
		consts: p.parseConstsDef(),
		preds:  p.parsePredsDef(),
		acts:   make([]action, 0),
	}
	// Ignore :functions for now
	if p.acceptNamedList(":functions") {
		for nesting := 1; nesting > 0; {
			switch p.next().typ {
			case tokClose:
				nesting--
			case tokOpen:
				nesting++
			}
		}
	}
	for p.peek().typ == tokOpen {
		d.acts = append(d.acts, p.parseActionDef())
	}

	p.expect(tokClose)
	return d
}

func (p *parser) parseDomainName() string {
	p.expect(tokOpen)
	p.expectId("domain")
	n := p.expect(tokId)
	p.expect(tokClose)
	return n.txt
}

func (p *parser) parseReqsDef() (reqs []string) {
	reqs = make([]string, 0)
	if !p.acceptNamedList(":requirements") {
		return
	}
	for t, ok := p.accept(tokCid); ok; t, ok = p.accept(tokCid) {
		reqs = append(reqs, t.txt)
	}
	p.expect(tokClose)
	return
}

func (p *parser) parseTypesDef() (types []tname) {
	types = make([]tname, 0)
	if !p.acceptNamedList(":types") {
		return
	}
	types = p.parseTypedListString(tokId)
	p.expect(tokClose)
	return
}

func (p *parser) parseConstsDef() (consts []tname) {
	consts = make([]tname, 0)
	if !p.acceptNamedList(":constants") {
		return
	}
	consts = p.parseTypedListString(tokId)
	p.expect(tokClose)
	return
}

func (p *parser) parsePredsDef() (preds []pred) {
	preds = make([]pred, 0)
	if !p.acceptNamedList(":predicates") {
		return
	}
	preds = append(preds, p.parseAtomicFormSkele())
	for p.peek().typ == tokOpen {
		preds = append(preds, p.parseAtomicFormSkele())
	}
	p.expect(tokClose)
	return
}

func (p *parser) parseAtomicFormSkele() pred {
	p.expect(tokOpen)
	pred := pred{
		name:  p.expect(tokId).txt,
		parms: p.parseTypedListString(tokQid),
	}
	p.expect(tokClose)
	return pred
}

func (p *parser) parseActionDef() action {
	p.expect(tokOpen)
	p.expectId(":action")

	act := action{name: p.expect(tokId).txt, parms: p.parseActParms()}

	if p.peek().txt == ":precondition" {
		p.junk(1)
		if p.peek().typ == tokOpen && p.peekn(2).typ == tokClose {
			p.junk(2)
		} else {
			act.prec = p.parsePreGd()
		}
	}
	if p.peek().txt == ":effect" {
		p.junk(1)
		if p.peek().typ == tokOpen && p.peekn(2).typ == tokClose {
			p.junk(2)
		} else {
			act.effect = p.parseEffect()
		}
	}

	p.expect(tokClose)
	return act
}

func (p *parser) parseActParms() []tname {
	p.expectId(":parameters")
	p.expect(tokOpen)
	res := p.parseTypedListString(tokQid)
	p.expect(tokClose)
	return res
}

func (p *parser) parsePreGd() (res *gd) {
	parseNested := func(p *parser) *gd { return p.parsePreGd() }
	switch {
	case p.acceptNamedList("and"):
		res = p.parseAndGd(parseNested)
	case p.acceptNamedList("forall"):
		res = p.parseQuantGd(gdForall, parseNested)
	default:
		res = p.parsePrefGd()
	}
	return
}

func (p *parser) parsePrefGd() *gd {
	return p.parseGd()
}

func (p *parser) parseGd() (res *gd) {
	parseNested := func(p *parser) *gd { return p.parseGd() }
	switch {
	case p.acceptNamedList("and"):
		res = p.parseAndGd(parseNested)
	case p.acceptNamedList("or"):
		res = p.parseOrGd(parseNested)
	case p.acceptNamedList("not"):
		res = &gd{typ: gdNot, left: p.parseGd()}
		p.expect(tokClose)
	case p.acceptNamedList("imply"):
		res = &gd{
			typ:   gdOr,
			left:  &gd{typ: gdNot, left: p.parseGd()},
			right: p.parseGd(),
		}
		p.expect(tokClose)
	case p.acceptNamedList("exists"):
		res = p.parseQuantGd(gdExists, parseNested)
	case p.acceptNamedList("forall"):
		res = p.parseQuantGd(gdForall, parseNested)
	default:
		p.expect(tokOpen)
		res = &gd{
			typ:   gdPred,
			name:  p.expect(tokId).txt,
			parms: p.parseTerms(),
		}
		p.expect(tokClose)
	}
	return
}

func (p *parser) parseAndGd(nested func(*parser) *gd) *gd {
	conj := make([]*gd, 0)
	for p.peek().typ == tokOpen {
		conj = append(conj, nested(p))
	}
	res := seqAnd(conj)
	p.expect(tokClose)
	return res
}

func seqAnd(conj []*gd) (res *gd) {
	switch len(conj) {
	case 0:
		res = &gd{typ: gdTrue}
	case 1:
		res = conj[0]
	default:
		res = &gd{typ: gdAnd, left: conj[0], right: seqAnd(conj[1:])}
	}
	return
}

func (p *parser) parseOrGd(nested func(*parser) *gd) *gd {
	disj := make([]*gd, 0)
	for p.peek().typ == tokOpen  {
		disj = append(disj, nested(p))
	}
	res := seqOr(disj)

	p.expect(tokClose)
	return res
}

func seqOr(disj []*gd) (res *gd) {
	switch len(disj) {
	case 0:
		res = &gd{typ: gdFalse}
	case 1:
		res = disj[0]
	default:
		res = &gd{typ: gdOr, left: disj[0], right: seqOr(disj[1:])}
	}
	return
}

func (p *parser) parseQuantGd(q gdtype, nested func(*parser) *gd) *gd {
	res := &gd{typ: gdForall}

	p.expect(tokOpen)
	vrs := p.parseTypedListString(tokQid)
	p.expect(tokClose)

	bottom := res
	for i, vr := range vrs {
		bottom.vr = vr
		if i < len(vrs)-1 {
			bottom.left = &gd{typ: q}
			bottom = bottom.left
		}
	}

	bottom.left = nested(p)
	p.expect(tokClose)
	return res
}

func (p *parser) parseTerms() []string {
	lst := make([]string, 0)
	for {
		if t, ok := p.accept(tokId); ok {
			lst = append(lst, t.txt)
			continue
		}
		if t, ok := p.accept(tokQid); ok {
			lst = append(lst, t.txt)
			continue
		}
		break
	}
	return lst

}

func (p *parser) parseEffect() *effect {
	// ignore
	p.expect(tokOpen)
	for nesting := 1; nesting > 0; {
		switch p.next().typ {
		case tokClose:
			nesting--
		case tokOpen:
			nesting++
		}
	}
	return nil
}

func (p *parser) parseTypedListString(typ ttype) []tname {
	lst := make([]tname, 0)

	for {
		names := p.parseStrings(typ)
		if len(names) == 0 {
			break
		}
		typ := p.parseType()
		for _, n := range names {
			lst = append(lst, tname{name: n, typ: typ})
		}
	}
	return lst
}

func (p *parser) parseType() []string {
	if _, ok := p.accept(tokMinus); !ok {
		return []string{"object"}
	}
	if _, ok := p.accept(tokOpen); ok {
		p.expectId("either")
		lst := p.parseStringPlus(tokId)
		p.expect(tokClose)
		return lst
	}
	t := p.expect(tokId)
	return []string{t.txt}
}

func (p *parser) parseStringPlus(typ ttype) []string {
	lst := []string{p.expect(typ).txt}
	lst = append(lst, p.parseStrings(typ)...)
	return lst
}

func (p *parser) parseStrings(typ ttype) []string {
	lst := make([]string, 0)
	for t, ok := p.accept(typ); ok; t, ok = p.accept(typ) {
		lst = append(lst, t.txt)
	}
	return lst
}
<|endoftext|>


<commit_before>package entities

import (
	"encoding/json"
	"errors"

	gostErrors "github.com/geodan/gost/src/errors"
)

// FeatureOfInterest in SensorThings represents the phenomena an Observation is detecting. In some cases a FeatureOfInterest
// can be the Location of the Sensor and therefore of the Observation. A FeatureOfInterest is linked to a single Observation
type FeatureOfInterest struct {
	BaseEntity
	Description     string                 `json:"description,omitempty"`
	EncodingType    string                 `json:"encodingType,omitempty"`
	Feature         map[string]interface{} `json:"feature,omitempty"`
	NavObservations string                 `json:"Observations@iot.navigationLink,omitempty"`
	Observations    []*Observation         `json:"Observations,omitempty"`
}

// GetEntityType returns the EntityType for FeatureOfInterest
func (f FeatureOfInterest) GetEntityType() EntityType {
	return EntityTypeFeatureOfInterest
}

// GetPropertyNames returns the available properties for a FeatureOfInterest
func (f *FeatureOfInterest) GetPropertyNames() []string {
	return []string{"id", "description", "encodingtype", "feature"}
}

// ParseEntity tries to parse the given json byte array into the current entity
func (f *FeatureOfInterest) ParseEntity(data []byte) error {
	foi := &f
	err := json.Unmarshal(data, foi)
	if err != nil {
		return gostErrors.NewBadRequestError(errors.New("Unable to parse FeatureOfInterest"))
	}

	return nil
}

// ContainsMandatoryParams checks if all mandatory params for a FeatureOfInterest are available before posting
func (f *FeatureOfInterest) ContainsMandatoryParams() (bool, []error) {
	err := []error{}
	CheckMandatoryParam(&err, f.Description, f.GetEntityType(), "description")
	CheckMandatoryParam(&err, f.EncodingType, f.GetEntityType(), "encodingtype")
	CheckMandatoryParam(&err, f.Feature, f.GetEntityType(), "feature")

	if len(err) != 0 {
		return false, err
	}

	return true, nil
}

// SetAllLinks sets the self link and relational links
func (f *FeatureOfInterest) SetAllLinks(externalURL string) {
	f.SetSelfLink(externalURL)
	f.SetLinks(externalURL)
}

// SetSelfLink sets the self link for the entity
func (f *FeatureOfInterest) SetSelfLink(externalURL string) {
	f.NavSelf = CreateEntitySelfLink(externalURL, EntityLinkFeatureOfInterests.ToString(), f.ID)
}

// SetLinks sets the entity specific navigation links, empty string if linked(expanded) data is not nil
func (f *FeatureOfInterest) SetLinks(externalURL string) {
	f.NavObservations = CreateEntityLink(f.Observations == nil, externalURL, EntityLinkFeatureOfInterests.ToString(), EntityLinkObservations.ToString(), f.ID)
}

// GetSupportedEncoding returns the supported encoding tye for this entity
func (f FeatureOfInterest) GetSupportedEncoding() map[int]EncodingType {
	return map[int]EncodingType{EncodingGeoJSON.Code: EncodingGeoJSON}
}
<commit_msg>add encodingtype to featureofinterest<commit_after>package entities

import (
	"encoding/json"
	"errors"

	gostErrors "github.com/geodan/gost/src/errors"
)

// FeatureOfInterest in SensorThings represents the phenomena an Observation is detecting. In some cases a FeatureOfInterest
// can be the Location of the Sensor and therefore of the Observation. A FeatureOfInterest is linked to a single Observation
type FeatureOfInterest struct {
	BaseEntity
	Description     string                 `json:"description,omitempty"`
	EncodingType    string                 `json:"encodingType,omitempty"`
	Feature         map[string]interface{} `json:"feature,omitempty"`
	NavObservations string                 `json:"Observations@iot.navigationLink,omitempty"`
	Observations    []*Observation         `json:"Observations,omitempty"`
}

// GetEntityType returns the EntityType for FeatureOfInterest
func (f FeatureOfInterest) GetEntityType() EntityType {
	return EntityTypeFeatureOfInterest
}

// GetPropertyNames returns the available properties for a FeatureOfInterest
func (f *FeatureOfInterest) GetPropertyNames() []string {
	return []string{"id", "description", "encodingtype", "feature"}
}

// ParseEntity tries to parse the given json byte array into the current entity
func (f *FeatureOfInterest) ParseEntity(data []byte) error {
	foi := &f
	err := json.Unmarshal(data, foi)
	if err != nil {
		return gostErrors.NewBadRequestError(errors.New("Unable to parse FeatureOfInterest"))
	}

	return nil
}

// ContainsMandatoryParams checks if all mandatory params for a FeatureOfInterest are available before posting
func (f *FeatureOfInterest) ContainsMandatoryParams() (bool, []error) {
	err := []error{}
	CheckMandatoryParam(&err, f.Description, f.GetEntityType(), "description")
	CheckMandatoryParam(&err, f.EncodingType, f.GetEntityType(), "encodingtype")
	CheckMandatoryParam(&err, f.Feature, f.GetEntityType(), "feature")

	if len(err) != 0 {
		return false, err
	}

	return true, nil
}

// SetAllLinks sets the self link and relational links
func (f *FeatureOfInterest) SetAllLinks(externalURL string) {
	f.SetSelfLink(externalURL)
	f.SetLinks(externalURL)
}

// SetSelfLink sets the self link for the entity
func (f *FeatureOfInterest) SetSelfLink(externalURL string) {
	f.NavSelf = CreateEntitySelfLink(externalURL, EntityLinkFeatureOfInterests.ToString(), f.ID)
}

// SetLinks sets the entity specific navigation links, empty string if linked(expanded) data is not nil
func (f *FeatureOfInterest) SetLinks(externalURL string) {
	f.NavObservations = CreateEntityLink(f.Observations == nil, externalURL, EntityLinkFeatureOfInterests.ToString(), EntityLinkObservations.ToString(), f.ID)
}

// GetSupportedEncoding returns the supported encoding tye for this entity
func (f FeatureOfInterest) GetSupportedEncoding() map[int]EncodingType {
	return map[int]EncodingType{EncodingGeoJSON.Code: EncodingGeoJSON, EncodingLocationType.Code: EncodingLocationType}
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Damon Revoe. All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

package main

import (
	"fmt"
	"strings"
)

type templateParams map[string]interface{}

type fileParams struct {
	filename string
	params   templateParams
}

type array struct {
	paramName    string
	paramValues  []string
	continuation verbatim
}

type verbatim struct {
	text string
	next *array
}

// Subst updates the 'verbatim' receiver by replacing all instances
// of paramName surrounded by braces with paramValue, which can be
// either a string or a slice of strings. In the latter case, the
// text in the receiver structure gets truncated by the substitution
// and the receiver structure gets extended by a new array structure.
// Subst returns the number of substitution values.
func (v *verbatim) subst(paramName string, paramValue interface{}) int {
	if textValue, ok := paramValue.(string); ok {
		v.text = strings.Replace(v.text, "{"+paramName+"}",
			textValue, -1)
	} else if arrayValue, ok := paramValue.([]string); !ok {
		v.text = strings.Replace(v.text, "{"+paramName+"}",
			fmt.Sprint(paramValue), -1)
	} else if pos := strings.Index(v.text, "{"+paramName+"}"); pos >= 0 {
		v.next = &array{paramName, arrayValue,
			verbatim{v.text[pos+len(paramName)+2:],
				v.next}}
		v.text = v.text[:pos]
		return len(arrayValue)
	}

	return 1
}

// ExpandPathnameTemplate takes a pathname template and subsitutes
// template parameter names with their values. Parameter values can be
// either strings or slices of strings. Each template value that is a
// slice of strings multiplies the number of output strings by the number
// of strings in the slice.
func expandPathnameTemplate(pathname string,
	params templateParams) []fileParams {
	root := verbatim{pathname, nil}

	resultSize := 1

	for paramName, paramValue := range params {
		resultSize *= root.subst(paramName, paramValue)

		for n := root.next; n != nil; n = n.continuation.next {
			resultSize *= n.continuation.subst(paramName, paramValue)
		}
	}

	result := make([]fileParams, resultSize)

	for i := 0; i < resultSize; i++ {
		result[i].filename = root.text
		copyOfParams := templateParams{}
		for paramName, paramValue := range params {
			copyOfParams[paramName] = paramValue
		}
		result[i].params = copyOfParams
	}

	if resultSize == 0 {
		return result
	}

	sliceSize := resultSize

	for a := root.next; a != nil; a = a.continuation.next {
		numberOfSlices := resultSize / sliceSize
		numberOfValues := len(a.paramValues)
		sliceSize /= numberOfValues

		filenameIndex := 0
		verbatimText := a.continuation.text
		for i := 0; i < numberOfSlices; i++ {
			for j := 0; j < numberOfValues; j++ {
				paramValue := a.paramValues[j]
				filenameFragment := paramValue + verbatimText
				for k := 0; k < sliceSize; k++ {
					result[filenameIndex].filename += filenameFragment
					result[filenameIndex].params[a.paramName] = paramValue
					filenameIndex++
				}
			}
		}
	}

	return result
}
<commit_msg>Improve expandPathnameTemplate implementation<commit_after>// Copyright (C) 2017 Damon Revoe. All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

package main

import (
	"fmt"
	"strings"
)

type templateParams map[string]interface{}

type fileParams struct {
	filename string
	params   templateParams
}

type array struct {
	paramName    string
	paramValues  []string
	continuation verbatim
}

type verbatim struct {
	text string
	next *array
}

// Subst updates the 'verbatim' receiver by replacing all instances
// of 'name' surrounded by braces with 'value', which can be
// either a string or a slice of strings. In the latter case, the
// text in the receiver structure gets truncated by the substitution
// and the receiver structure gets extended by a new array structure.
// Subst returns the number of substitution values.
func (v *verbatim) subst(name string, value interface{}) int {
	if textValue, ok := value.(string); ok {
		v.text = strings.Replace(v.text, "{"+name+"}",
			textValue, -1)
	} else if arrayValue, ok := value.([]string); !ok {
		v.text = strings.Replace(v.text, "{"+name+"}",
			fmt.Sprint(value), -1)
	} else if pos := strings.Index(v.text, "{"+name+"}"); pos >= 0 {
		v.next = &array{name, arrayValue,
			verbatim{v.text[pos+len(name)+2:], v.next}}
		v.text = v.text[:pos]
		return len(arrayValue)
	}

	return 1
}

// ExpandPathnameTemplate takes a pathname template and subsitutes
// template parameter names with their values. Parameter values can be
// either strings or slices of strings. Each template value that is a
// slice of strings multiplies the number of output strings by the number
// of strings in the slice.
func expandPathnameTemplate(pathname string,
	params templateParams) []fileParams {
	root := verbatim{pathname, nil}

	resultSize := 1

	for name, value := range params {
		resultSize *= root.subst(name, value)

		for n := root.next; n != nil; n = n.continuation.next {
			resultSize *= n.continuation.subst(name, value)
		}
	}

	result := make([]fileParams, resultSize)

	for i := 0; i < resultSize; i++ {
		result[i].filename = root.text
		copyOfParams := templateParams{}
		for name, value := range params {
			copyOfParams[name] = value
		}
		result[i].params = copyOfParams
	}

	if resultSize == 0 {
		return result
	}

	sliceSize := 1

	for a := root.next; a != nil; a = a.continuation.next {
		numberOfValues := len(a.paramValues)

		verbatimText := a.continuation.text

		for i := 0; i < resultSize; {
			for j := 0; j < numberOfValues; j++ {
				value := a.paramValues[j]
				filenameFragment := value + verbatimText
				for k := 0; k < sliceSize; k++ {
					result[i].filename += filenameFragment
					result[i].params[a.paramName] = value
					i++
				}
			}
		}

		sliceSize *= numberOfValues
	}

	return result
}
<|endoftext|>


<commit_before>// Copyright 2014, David Minor. All rights reserved.
// Use of this source code is governed by the MIT
// license which can be found in the LICENSE file.

package pcg

import (
  . "github.com/davidminor/uint128"
)

var multiplier = Uint128{2549297995355413924,4865540595714422341}
var default_stream = Uint128{6364136223846793005,1442695040888963407}

type Pcg64 struct {
  state, stream Uint128
}

// create a new PCG with the given seed and the default stream
func NewPcg64(seed Uint128) *Pcg64 {
  rng := &Pcg64{}
  rng.Seed(seed)
  return rng
}

// create a new PCG with the given seed and stream
func NewPcg64Stream(state, stream Uint128) *Pcg64 {
  rng := &Pcg64{}
  rng.Stream(stream)
  rng.Seed(state)
  return rng
}

// seed the PCG's initial state
func (rng *Pcg64) Seed(seed Uint128) {
  if rng.stream.L == 0 && rng.stream.H == 0 {
    rng.Stream(default_stream)
  }
  rng.state = seed.Add(rng.stream)
  rng.bump()
}

// set the stream of this PCG
func (rng *Pcg64) Stream(stream Uint128) {
  // stream must be odd for LCG, so shift left 1 and turn on the 1 bit
  rng.stream = stream.ShiftLeft(1)
  rng.stream.L |= uint64(1)
}

// advance the LCG one step
func (rng *Pcg64) bump() {
  rng.state = rng.state.Mult(multiplier).Add(rng.stream)
}

// xor the top bits with the bottom and randomly rotate them
func output(internal Uint128) uint64 {
  rot := internal.H >> 58;
  shift := internal.L ^ internal.H;
  return (shift >> rot) | (shift << (64 - rot));
}

// returns the next uint64 from the generator
func (rng *Pcg64) Uint64() uint64 {
  rng.bump()
  return output(rng.state)
}

// returns an uint64 uniformly distributed in [0,bounds)
func (rng *Pcg64) Uint64n(bounds uint64) uint64 {
  threshold := (uint64(0) - bounds) % bounds
  for {
    result := rng.Uint64()
    if result >= threshold {
      return result % bounds;
    }
  }
}

<commit_msg>rename argument for clarity<commit_after>// Copyright 2014, David Minor. All rights reserved.
// Use of this source code is governed by the MIT
// license which can be found in the LICENSE file.

package pcg

import (
  . "github.com/davidminor/uint128"
)

var multiplier = Uint128{2549297995355413924,4865540595714422341}
var default_stream = Uint128{6364136223846793005,1442695040888963407}

type Pcg64 struct {
  state, stream Uint128
}

// create a new PCG with the given seed and the default stream
func NewPcg64(seed Uint128) *Pcg64 {
  rng := &Pcg64{}
  rng.Seed(seed)
  return rng
}

// create a new PCG with the given seed and stream
func NewPcg64Stream(seed, stream Uint128) *Pcg64 {
  rng := &Pcg64{}
  rng.Stream(stream)
  rng.Seed(seed)
  return rng
}

// seed the PCG's initial state
func (rng *Pcg64) Seed(seed Uint128) {
  if rng.stream.L == 0 && rng.stream.H == 0 {
    rng.Stream(default_stream)
  }
  rng.state = seed.Add(rng.stream)
  rng.bump()
}

// set the stream of this PCG
func (rng *Pcg64) Stream(stream Uint128) {
  // stream must be odd for LCG, so shift left 1 and turn on the 1 bit
  rng.stream = stream.ShiftLeft(1)
  rng.stream.L |= uint64(1)
}

// advance the LCG one step
func (rng *Pcg64) bump() {
  rng.state = rng.state.Mult(multiplier).Add(rng.stream)
}

// xor the top bits with the bottom and randomly rotate them
func output(internal Uint128) uint64 {
  rot := internal.H >> 58;
  shift := internal.L ^ internal.H;
  return (shift >> rot) | (shift << (64 - rot));
}

// returns the next uint64 from the generator
func (rng *Pcg64) Uint64() uint64 {
  rng.bump()
  return output(rng.state)
}

// returns an uint64 uniformly distributed in [0,bounds)
func (rng *Pcg64) Uint64n(bounds uint64) uint64 {
  threshold := (uint64(0) - bounds) % bounds
  for {
    result := rng.Uint64()
    if result >= threshold {
      return result % bounds;
    }
  }
}

<|endoftext|>