<commit_before>/*
Copyright 2014 CoreOS Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package wal

import (
	"errors"
	"fmt"
	"hash/crc32"
	"io"
	"log"
	"os"
	"path"
	"sort"

	"github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/wal/walpb"
)

const (
	infoType int64 = iota + 1
	entryType
	stateType
	crcType

	// the owner can make/remove files inside the directory
	privateDirMode = 0700
)

var (
	ErrIDMismatch  = errors.New("wal: unmatch id")
	ErrNotFound    = errors.New("wal: file is not found")
	ErrCRCMismatch = errors.New("wal: crc mismatch")
	crcTable       = crc32.MakeTable(crc32.Castagnoli)
)

// WAL is a logical repersentation of the stable storage.
// WAL is either in read mode or append mode but not both.
// A newly created WAL is in append mode, and ready for appending records.
// A just opened WAL is in read mode, and ready for reading records.
// The WAL will be ready for appending after reading out all the previous records.
type WAL struct {
	dir string // the living directory of the underlay files

	ri      int64    // index of entry to start reading
	decoder *decoder // decoder to decode records

	f       *os.File // underlay file opened for appending, sync
	seq     int64    // current sequence of the wal file to be written
	enti    int64    // index of the last entry that has been saved to wal
	encoder *encoder // encoder to encode records
}

// Create creates a WAL ready for appending records.
func Create(dirpath string) (*WAL, error) {
	log.Printf("path=%s wal.create", dirpath)
	if Exist(dirpath) {
		return nil, os.ErrExist
	}

	if err := os.MkdirAll(dirpath, privateDirMode); err != nil {
		return nil, err
	}

	p := path.Join(dirpath, fmt.Sprintf("%016x-%016x.wal", 0, 0))
	f, err := os.OpenFile(p, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
	if err != nil {
		return nil, err
	}
	w := &WAL{
		dir:     dirpath,
		seq:     0,
		f:       f,
		encoder: newEncoder(f, 0),
	}
	if err := w.saveCrc(0); err != nil {
		return nil, err
	}
	return w, nil
}

// OpenAtIndex opens the WAL at the given index.
// The returned WAL is ready to read and the first record will be the given
// index. The WAL cannot be appended to before reading out all of its
// previous records.
func OpenAtIndex(dirpath string, index int64) (*WAL, error) {
	log.Printf("path=%s wal.load index=%d", dirpath, index)
	names, err := readDir(dirpath)
	if err != nil {
		return nil, err
	}
	names = checkWalNames(names)
	if len(names) == 0 {
		return nil, ErrNotFound
	}

	sort.Sort(sort.StringSlice(names))

	nameIndex, ok := searchIndex(names, index)
	if !ok || !isValidSeq(names[nameIndex:]) {
		return nil, ErrNotFound
	}

	// open the wal files for reading
	rcs := make([]io.ReadCloser, 0)
	for _, name := range names[nameIndex:] {
		f, err := os.Open(path.Join(dirpath, name))
		if err != nil {
			return nil, err
		}
		rcs = append(rcs, f)
	}
	rc := MultiReadCloser(rcs...)

	// open the lastest wal file for appending
	seq, _, err := parseWalName(names[len(names)-1])
	if err != nil {
		rc.Close()
		return nil, err
	}
	last := path.Join(dirpath, names[len(names)-1])
	f, err := os.OpenFile(last, os.O_WRONLY|os.O_APPEND, 0)
	if err != nil {
		rc.Close()
		return nil, err
	}

	// create a WAL ready for reading
	w := &WAL{
		ri:      index,
		decoder: newDecoder(rc),

		f:   f,
		seq: seq,
	}
	return w, nil
}

// ReadAll reads out all records of the current WAL.
// After ReadAll, the WAL will be ready for appending new records.
func (w *WAL) ReadAll() (id int64, state raftpb.State, ents []raftpb.Entry, err error) {
	rec := &walpb.Record{}
	decoder := w.decoder

	for err = decoder.decode(rec); err == nil; err = decoder.decode(rec) {
		switch rec.Type {
		case entryType:
			e := mustUnmarshalEntry(rec.Data)
			if e.Index >= w.ri {
				ents = append(ents[:e.Index-w.ri], e)
			}
			w.enti = e.Index
		case stateType:
			state = mustUnmarshalState(rec.Data)
		case infoType:
			i := mustUnmarshalInfo(rec.Data)
			if id != 0 && id != i.Id {
				state.Reset()
				return 0, state, nil, ErrIDMismatch
			}
			id = i.Id
		case crcType:
			crc := decoder.crc.Sum32()
			// current crc of decoder must match the crc of the record.
			// do no need to match 0 crc, since the decoder is a new one at this case.
			if crc != 0 && rec.Validate(crc) != nil {
				state.Reset()
				return 0, state, nil, ErrCRCMismatch
			}
			decoder.updateCRC(rec.Crc)
		default:
			state.Reset()
			return 0, state, nil, fmt.Errorf("unexpected block type %d", rec.Type)
		}
	}
	if err != io.EOF {
		state.Reset()
		return 0, state, nil, err
	}

	// close decoder, disable reading
	w.decoder.close()
	w.ri = 0

	// create encoder (chain crc with the decoder), enable appending
	w.encoder = newEncoder(w.f, w.decoder.lastCRC())
	w.decoder = nil
	return id, state, ents, nil
}

// index should be the index of last log entry.
// Cut closes current file written and creates a new one ready to append.
func (w *WAL) Cut() error {
	log.Printf("wal.cut index=%d", w.enti+1)

	// create a new wal file with name sequence + 1
	fpath := path.Join(w.dir, fmt.Sprintf("%016x-%016x.wal", w.seq+1, w.enti+1))
	f, err := os.OpenFile(fpath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
	if err != nil {
		return err
	}

	w.Sync()
	w.f.Close()

	// update writer and save the previous crc
	w.f = f
	w.seq++
	prevCrc := w.encoder.crc.Sum32()
	w.encoder = newEncoder(w.f, prevCrc)
	return w.saveCrc(prevCrc)
}

func (w *WAL) Sync() error {
	if w.encoder != nil {
		if err := w.encoder.flush(); err != nil {
			return err
		}
	}
	return w.f.Sync()
}

func (w *WAL) Close() {
	log.Printf("path=%s wal.close", w.f.Name())
	if w.f != nil {
		w.Sync()
		w.f.Close()
	}
}

func (w *WAL) SaveInfo(i *raftpb.Info) error {
	log.Printf("path=%s wal.saveInfo id=%d", w.f.Name(), i.Id)
	b, err := i.Marshal()
	if err != nil {
		panic(err)
	}
	rec := &walpb.Record{Type: infoType, Data: b}
	return w.encoder.encode(rec)
}

func (w *WAL) SaveEntry(e *raftpb.Entry) error {
	b, err := e.Marshal()
	if err != nil {
		panic(err)
	}
	rec := &walpb.Record{Type: entryType, Data: b}
	if err := w.encoder.encode(rec); err != nil {
		return err
	}
	w.enti = e.Index
	return nil
}

func (w *WAL) SaveState(s *raftpb.State) error {
	if raft.IsEmptyState(*s) {
		return nil
	}
	log.Printf("path=%s wal.saveState state=\"%+v\"", w.f.Name(), s)
	b, err := s.Marshal()
	if err != nil {
		panic(err)
	}
	rec := &walpb.Record{Type: stateType, Data: b}
	return w.encoder.encode(rec)
}

func (w *WAL) Save(st raftpb.State, ents []raftpb.Entry) {
	// TODO(xiangli): no more reference operator
	w.SaveState(&st)
	for i := range ents {
		w.SaveEntry(&ents[i])
	}
	w.Sync()
}

func (w *WAL) saveCrc(prevCrc uint32) error {
	return w.encoder.encode(&walpb.Record{Type: crcType, Crc: prevCrc})
}
<commit_msg>wal: remove wrong comment for cut<commit_after>/*
Copyright 2014 CoreOS Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package wal

import (
	"errors"
	"fmt"
	"hash/crc32"
	"io"
	"log"
	"os"
	"path"
	"sort"

	"github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/wal/walpb"
)

const (
	infoType int64 = iota + 1
	entryType
	stateType
	crcType

	// the owner can make/remove files inside the directory
	privateDirMode = 0700
)

var (
	ErrIDMismatch  = errors.New("wal: unmatch id")
	ErrNotFound    = errors.New("wal: file is not found")
	ErrCRCMismatch = errors.New("wal: crc mismatch")
	crcTable       = crc32.MakeTable(crc32.Castagnoli)
)

// WAL is a logical repersentation of the stable storage.
// WAL is either in read mode or append mode but not both.
// A newly created WAL is in append mode, and ready for appending records.
// A just opened WAL is in read mode, and ready for reading records.
// The WAL will be ready for appending after reading out all the previous records.
type WAL struct {
	dir string // the living directory of the underlay files

	ri      int64    // index of entry to start reading
	decoder *decoder // decoder to decode records

	f       *os.File // underlay file opened for appending, sync
	seq     int64    // current sequence of the wal file to be written
	enti    int64    // index of the last entry that has been saved to wal
	encoder *encoder // encoder to encode records
}

// Create creates a WAL ready for appending records.
func Create(dirpath string) (*WAL, error) {
	log.Printf("path=%s wal.create", dirpath)
	if Exist(dirpath) {
		return nil, os.ErrExist
	}

	if err := os.MkdirAll(dirpath, privateDirMode); err != nil {
		return nil, err
	}

	p := path.Join(dirpath, fmt.Sprintf("%016x-%016x.wal", 0, 0))
	f, err := os.OpenFile(p, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
	if err != nil {
		return nil, err
	}
	w := &WAL{
		dir:     dirpath,
		seq:     0,
		f:       f,
		encoder: newEncoder(f, 0),
	}
	if err := w.saveCrc(0); err != nil {
		return nil, err
	}
	return w, nil
}

// OpenAtIndex opens the WAL at the given index.
// The returned WAL is ready to read and the first record will be the given
// index. The WAL cannot be appended to before reading out all of its
// previous records.
func OpenAtIndex(dirpath string, index int64) (*WAL, error) {
	log.Printf("path=%s wal.load index=%d", dirpath, index)
	names, err := readDir(dirpath)
	if err != nil {
		return nil, err
	}
	names = checkWalNames(names)
	if len(names) == 0 {
		return nil, ErrNotFound
	}

	sort.Sort(sort.StringSlice(names))

	nameIndex, ok := searchIndex(names, index)
	if !ok || !isValidSeq(names[nameIndex:]) {
		return nil, ErrNotFound
	}

	// open the wal files for reading
	rcs := make([]io.ReadCloser, 0)
	for _, name := range names[nameIndex:] {
		f, err := os.Open(path.Join(dirpath, name))
		if err != nil {
			return nil, err
		}
		rcs = append(rcs, f)
	}
	rc := MultiReadCloser(rcs...)

	// open the lastest wal file for appending
	seq, _, err := parseWalName(names[len(names)-1])
	if err != nil {
		rc.Close()
		return nil, err
	}
	last := path.Join(dirpath, names[len(names)-1])
	f, err := os.OpenFile(last, os.O_WRONLY|os.O_APPEND, 0)
	if err != nil {
		rc.Close()
		return nil, err
	}

	// create a WAL ready for reading
	w := &WAL{
		ri:      index,
		decoder: newDecoder(rc),

		f:   f,
		seq: seq,
	}
	return w, nil
}

// ReadAll reads out all records of the current WAL.
// After ReadAll, the WAL will be ready for appending new records.
func (w *WAL) ReadAll() (id int64, state raftpb.State, ents []raftpb.Entry, err error) {
	rec := &walpb.Record{}
	decoder := w.decoder

	for err = decoder.decode(rec); err == nil; err = decoder.decode(rec) {
		switch rec.Type {
		case entryType:
			e := mustUnmarshalEntry(rec.Data)
			if e.Index >= w.ri {
				ents = append(ents[:e.Index-w.ri], e)
			}
			w.enti = e.Index
		case stateType:
			state = mustUnmarshalState(rec.Data)
		case infoType:
			i := mustUnmarshalInfo(rec.Data)
			if id != 0 && id != i.Id {
				state.Reset()
				return 0, state, nil, ErrIDMismatch
			}
			id = i.Id
		case crcType:
			crc := decoder.crc.Sum32()
			// current crc of decoder must match the crc of the record.
			// do no need to match 0 crc, since the decoder is a new one at this case.
			if crc != 0 && rec.Validate(crc) != nil {
				state.Reset()
				return 0, state, nil, ErrCRCMismatch
			}
			decoder.updateCRC(rec.Crc)
		default:
			state.Reset()
			return 0, state, nil, fmt.Errorf("unexpected block type %d", rec.Type)
		}
	}
	if err != io.EOF {
		state.Reset()
		return 0, state, nil, err
	}

	// close decoder, disable reading
	w.decoder.close()
	w.ri = 0

	// create encoder (chain crc with the decoder), enable appending
	w.encoder = newEncoder(w.f, w.decoder.lastCRC())
	w.decoder = nil
	return id, state, ents, nil
}

// Cut closes current file written and creates a new one ready to append.
func (w *WAL) Cut() error {
	log.Printf("wal.cut index=%d", w.enti+1)

	// create a new wal file with name sequence + 1
	fpath := path.Join(w.dir, fmt.Sprintf("%016x-%016x.wal", w.seq+1, w.enti+1))
	f, err := os.OpenFile(fpath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
	if err != nil {
		return err
	}

	w.Sync()
	w.f.Close()

	// update writer and save the previous crc
	w.f = f
	w.seq++
	prevCrc := w.encoder.crc.Sum32()
	w.encoder = newEncoder(w.f, prevCrc)
	return w.saveCrc(prevCrc)
}

func (w *WAL) Sync() error {
	if w.encoder != nil {
		if err := w.encoder.flush(); err != nil {
			return err
		}
	}
	return w.f.Sync()
}

func (w *WAL) Close() {
	log.Printf("path=%s wal.close", w.f.Name())
	if w.f != nil {
		w.Sync()
		w.f.Close()
	}
}

func (w *WAL) SaveInfo(i *raftpb.Info) error {
	log.Printf("path=%s wal.saveInfo id=%d", w.f.Name(), i.Id)
	b, err := i.Marshal()
	if err != nil {
		panic(err)
	}
	rec := &walpb.Record{Type: infoType, Data: b}
	return w.encoder.encode(rec)
}

func (w *WAL) SaveEntry(e *raftpb.Entry) error {
	b, err := e.Marshal()
	if err != nil {
		panic(err)
	}
	rec := &walpb.Record{Type: entryType, Data: b}
	if err := w.encoder.encode(rec); err != nil {
		return err
	}
	w.enti = e.Index
	return nil
}

func (w *WAL) SaveState(s *raftpb.State) error {
	if raft.IsEmptyState(*s) {
		return nil
	}
	log.Printf("path=%s wal.saveState state=\"%+v\"", w.f.Name(), s)
	b, err := s.Marshal()
	if err != nil {
		panic(err)
	}
	rec := &walpb.Record{Type: stateType, Data: b}
	return w.encoder.encode(rec)
}

func (w *WAL) Save(st raftpb.State, ents []raftpb.Entry) {
	// TODO(xiangli): no more reference operator
	w.SaveState(&st)
	for i := range ents {
		w.SaveEntry(&ents[i])
	}
	w.Sync()
}

func (w *WAL) saveCrc(prevCrc uint32) error {
	return w.encoder.encode(&walpb.Record{Type: crcType, Crc: prevCrc})
}
<|endoftext|>


<commit_before>package watcher

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

var (
	// ErrNothingAdded is an error that occurs when a Watcher's Start() method is
	// called and no files or folders have been added to the Watcher's watchlist.
	ErrNothingAdded = errors.New("error: no files added to the watchlist")

	// ErrWatchedFileDeleted is an error that occurs when a file or folder that was
	// being watched has been deleted.
	ErrWatchedFileDeleted = errors.New("error: watched file or folder deleted")
)

// An EventType is a type that is used to describe what type
// of event has occured during the watching process.
type EventType int

const (
	EventFileAdded EventType = 1 << iota
	EventFileDeleted
	EventFileModified
)

// An Option is a type that is used to set options for a Watcher.
type Option int

const (
	// NonRecursive sets the watcher to not watch directories recursively.
	NonRecursive Option = 1 << iota

	// IgnoreDotFiles sets the watcher to ignore dot files.
	IgnoreDotFiles
)

// An Event desribes an event that is received when files or directory
// changes occur. It includes the os.FileInfo os the changes file or
// directory and the type of event that's occured.
type Event struct {
	EventType
	os.FileInfo
}

// String returns a string depending on what type of event occured and the
// file name associated with the event.
func (e Event) String() string {
	var fileType string
	if e.IsDir() {
		fileType = "DIRECTORY"
	} else {
		fileType = "FILE"
	}

	switch e.EventType {
	case EventFileAdded:
		return fmt.Sprintf("%s %q ADDED", fileType, e.Name())
	case EventFileDeleted:
		return fmt.Sprintf("%s %q DELETED", fileType, e.Name())
	case EventFileModified:
		return fmt.Sprintf("%s %q MODIFIED", fileType, e.Name())
	default:
		return "UNRECOGNIZED EVENT"
	}
}

// A Watcher describes a file watcher.
type Watcher struct {
	Event chan Event
	Error chan error

	options []Option

	maxEventsPerCycle int

	// mu protects Files and Names.
	mu    *sync.Mutex
	Files map[string]os.FileInfo
	Names []string
}

// New returns a new initialized *Watcher.
func New(options ...Option) *Watcher {
	return &Watcher{
		Event:   make(chan Event),
		Error:   make(chan error),
		options: options,
		mu:      new(sync.Mutex),
		Files:   make(map[string]os.FileInfo),
		Names:   []string{},
	}
}

// SetMaxEvents controls the maximum amount of events that are sent on
// the Event channel per watching cycle. If max events is less than 1, there is
// no limit, which is the default.
func (w *Watcher) SetMaxEvents(amount int) {
	w.mu.Lock()
	w.maxEventsPerCycle = amount
	w.mu.Unlock()
}

// fileInfo is an implementation of os.FileInfo that can be used
// as a mocked os.FileInfo when triggering an event when the specified
// os.FileInfo is nil.
type fileInfo struct {
	name    string
	size    int64
	mode    os.FileMode
	modTime time.Time
	sys     interface{}
}

func (fs *fileInfo) IsDir() bool {
	return false
}
func (fs *fileInfo) ModTime() time.Time {
	return fs.modTime
}
func (fs *fileInfo) Mode() os.FileMode {
	return fs.mode
}
func (fs *fileInfo) Name() string {
	return fs.name
}
func (fs *fileInfo) Size() int64 {
	return fs.size
}
func (fs *fileInfo) Sys() interface{} {
	return fs.sys
}

// Add adds either a single file or recursed directory to
// the Watcher's file list.
func (w *Watcher) Add(name string) error {
	// Add the name from w's Names list.
	w.mu.Lock()
	w.Names = append(w.Names, name)
	w.mu.Unlock()

	// Make sure name exists.
	fInfo, err := os.Stat(name)
	if err != nil {
		return err
	}

	// If watching a single file, add it and return.
	if !fInfo.IsDir() {
		w.mu.Lock()
		w.Files[fInfo.Name()] = fInfo
		w.mu.Unlock()
		return nil
	}

	// Retrieve a list of all of the os.FileInfo's to add to w.Files.
	fInfoList, err := ListFiles(name, w.options...)
	if err != nil {
		return err
	}
	w.mu.Lock()
	for k, v := range fInfoList {
		w.Files[k] = v
	}
	w.mu.Unlock()
	return nil
}

// Remove removes either a single file or recursed directory from
// the Watcher's file list.
func (w *Watcher) Remove(name string) error {
	// Remove the name from w's Names list.
	w.mu.Lock()
	for i := range w.Names {
		if w.Names[i] == name {
			w.Names = append(w.Names[:i], w.Names[i+1:]...)
		}
	}
	w.mu.Unlock()

	// Make sure name exists.
	fInfo, err := os.Stat(name)
	if err != nil {
		return err
	}

	// If name is a single file, remove it and return.
	if !fInfo.IsDir() {
		w.mu.Lock()
		delete(w.Files, fInfo.Name())
		w.mu.Unlock()
		return nil
	}

	// Retrieve a list of all of the os.FileInfo's to delete from w.Files.
	fInfoList, err := ListFiles(name, w.options...)
	if err != nil {
		return err
	}

	// Remove the appropriate os.FileInfo's from w's os.FileInfo list.
	w.mu.Lock()
	for path := range fInfoList {
		delete(w.Files, path)
	}
	w.mu.Unlock()
	return nil
}

// TriggerEvent is a method that can be used to trigger an event, separate to
// the file watching process.
func (w *Watcher) TriggerEvent(eventType EventType, file os.FileInfo) {
	if file == nil {
		file = &fileInfo{name: "triggered event", modTime: time.Now()}
	}
	w.Event <- Event{eventType, file}
}

// Start starts the watching process and checks for changes every `pollInterval` duration.
// If pollInterval is 0, the default is 100ms.
func (w *Watcher) Start(pollInterval time.Duration) error {
	if pollInterval <= 0 {
		pollInterval = time.Millisecond * 100
	}

	if len(w.Names) < 1 {
		return ErrNothingAdded
	}

	for {
		fileList := make(map[string]os.FileInfo)
		for _, name := range w.Names {
			// Retrieve the list of os.FileInfo's from w.Name.
			list, err := ListFiles(name, w.options...)
			if err != nil {
				if os.IsNotExist(err) {
					w.Error <- ErrWatchedFileDeleted
				} else {
					w.Error <- err
				}
			}
			for k, v := range list {
				fileList[k] = v
			}
		}

		numEvents := 0
		addedAndDeleted := make(map[string]struct{})

		// Check for added files.
		for path, file := range fileList {
			if w.maxEventsPerCycle > 0 && numEvents >= w.maxEventsPerCycle {
				goto SLEEP
			}
			if _, found := w.Files[path]; !found {
				addedAndDeleted[path] = struct{}{}
				w.Event <- Event{EventType: EventFileAdded, FileInfo: file}
				numEvents++
			}
		}

		// Check for deleted files.
		for path, file := range w.Files {
			if w.maxEventsPerCycle > 0 && numEvents >= w.maxEventsPerCycle {
				goto SLEEP
			}
			if _, found := fileList[path]; !found {
				addedAndDeleted[path] = struct{}{}
				w.Event <- Event{EventType: EventFileDeleted, FileInfo: file}
				numEvents++
			}
		}

		// Check for modified files.
		for path, file := range w.Files {
			if w.maxEventsPerCycle > 0 && numEvents >= w.maxEventsPerCycle {
				goto SLEEP
			}
			if _, found := addedAndDeleted[path]; !found {
				if fileList[path].ModTime() != file.ModTime() {
					w.Event <- Event{EventType: EventFileModified, FileInfo: file}
					numEvents++
				}
			}
		}

	SLEEP:
		// Update w.Files and then sleep for a little bit.
		w.Files = fileList
		time.Sleep(pollInterval)
	}

	return nil
}

// hasOption returns true or false based on whether or not
// an Option exists in an Option slice.
func hasOption(options []Option, option Option) bool {
	for _, o := range options {
		if option&o != 0 {
			return true
		}
	}
	return false
}

// ListFiles returns a map of all os.FileInfo's recursively
// contained in a directory. If name is a single file, it returns
// an os.FileInfo map containing a single os.FileInfo.
func ListFiles(name string, options ...Option) (map[string]os.FileInfo, error) {
	fileList := make(map[string]os.FileInfo)

	name = filepath.Clean(name)

	nonRecursive := hasOption(options, NonRecursive)
	ignoreDotFiles := hasOption(options, IgnoreDotFiles)

	if nonRecursive {
		f, err := os.Open(name)
		if err != nil {
			return nil, err
		}
		defer f.Close()
		info, err := os.Stat(name)
		if err != nil {
			return nil, err
		}
		// Add the name to fileList.
		if !info.IsDir() && ignoreDotFiles && strings.HasPrefix(name, ".") {
			return fileList, nil
		}
		fileList[name] = info
		if !info.IsDir() {
			return fileList, nil
		}
		// It's a directory, read it's contents.
		fInfoList, err := f.Readdir(-1)
		if err != nil {
			return nil, err
		}
		// Add all of the FileInfo's returned from f.ReadDir to fileList.
		for _, fInfo := range fInfoList {
			if ignoreDotFiles && strings.HasPrefix(fInfo.Name(), ".") {
				continue
			}
			fileList[filepath.Join(name, fInfo.Name())] = fInfo
		}
		return fileList, nil
	}

	if err := filepath.Walk(name, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if ignoreDotFiles && strings.HasPrefix(info.Name(), ".") {
			if info.IsDir() && info.Name() != "." && info.Name() != ".." {
				return filepath.SkipDir
			}
			return nil
		}

		fileList[path] = info

		return nil
	}); err != nil {
		return nil, err
	}

	return fileList, nil
}
<commit_msg>removed redundant line<commit_after>package watcher

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

var (
	// ErrNothingAdded is an error that occurs when a Watcher's Start() method is
	// called and no files or folders have been added to the Watcher's watchlist.
	ErrNothingAdded = errors.New("error: no files added to the watchlist")

	// ErrWatchedFileDeleted is an error that occurs when a file or folder that was
	// being watched has been deleted.
	ErrWatchedFileDeleted = errors.New("error: watched file or folder deleted")
)

// An EventType is a type that is used to describe what type
// of event has occured during the watching process.
type EventType int

const (
	EventFileAdded EventType = 1 << iota
	EventFileDeleted
	EventFileModified
)

// An Option is a type that is used to set options for a Watcher.
type Option int

const (
	// NonRecursive sets the watcher to not watch directories recursively.
	NonRecursive Option = 1 << iota

	// IgnoreDotFiles sets the watcher to ignore dot files.
	IgnoreDotFiles
)

// An Event desribes an event that is received when files or directory
// changes occur. It includes the os.FileInfo os the changes file or
// directory and the type of event that's occured.
type Event struct {
	EventType
	os.FileInfo
}

// String returns a string depending on what type of event occured and the
// file name associated with the event.
func (e Event) String() string {
	var fileType string
	if e.IsDir() {
		fileType = "DIRECTORY"
	} else {
		fileType = "FILE"
	}

	switch e.EventType {
	case EventFileAdded:
		return fmt.Sprintf("%s %q ADDED", fileType, e.Name())
	case EventFileDeleted:
		return fmt.Sprintf("%s %q DELETED", fileType, e.Name())
	case EventFileModified:
		return fmt.Sprintf("%s %q MODIFIED", fileType, e.Name())
	default:
		return "UNRECOGNIZED EVENT"
	}
}

// A Watcher describes a file watcher.
type Watcher struct {
	Event chan Event
	Error chan error

	options []Option

	maxEventsPerCycle int

	// mu protects Files and Names.
	mu    *sync.Mutex
	Files map[string]os.FileInfo
	Names []string
}

// New returns a new initialized *Watcher.
func New(options ...Option) *Watcher {
	return &Watcher{
		Event:   make(chan Event),
		Error:   make(chan error),
		options: options,
		mu:      new(sync.Mutex),
		Files:   make(map[string]os.FileInfo),
		Names:   []string{},
	}
}

// SetMaxEvents controls the maximum amount of events that are sent on
// the Event channel per watching cycle. If max events is less than 1, there is
// no limit, which is the default.
func (w *Watcher) SetMaxEvents(amount int) {
	w.mu.Lock()
	w.maxEventsPerCycle = amount
	w.mu.Unlock()
}

// fileInfo is an implementation of os.FileInfo that can be used
// as a mocked os.FileInfo when triggering an event when the specified
// os.FileInfo is nil.
type fileInfo struct {
	name    string
	size    int64
	mode    os.FileMode
	modTime time.Time
	sys     interface{}
}

func (fs *fileInfo) IsDir() bool {
	return false
}
func (fs *fileInfo) ModTime() time.Time {
	return fs.modTime
}
func (fs *fileInfo) Mode() os.FileMode {
	return fs.mode
}
func (fs *fileInfo) Name() string {
	return fs.name
}
func (fs *fileInfo) Size() int64 {
	return fs.size
}
func (fs *fileInfo) Sys() interface{} {
	return fs.sys
}

// Add adds either a single file or recursed directory to
// the Watcher's file list.
func (w *Watcher) Add(name string) error {
	// Add the name from w's Names list.
	w.mu.Lock()
	w.Names = append(w.Names, name)
	w.mu.Unlock()

	// Make sure name exists.
	fInfo, err := os.Stat(name)
	if err != nil {
		return err
	}

	// If watching a single file, add it and return.
	if !fInfo.IsDir() {
		w.mu.Lock()
		w.Files[fInfo.Name()] = fInfo
		w.mu.Unlock()
		return nil
	}

	// Retrieve a list of all of the os.FileInfo's to add to w.Files.
	fInfoList, err := ListFiles(name, w.options...)
	if err != nil {
		return err
	}
	w.mu.Lock()
	for k, v := range fInfoList {
		w.Files[k] = v
	}
	w.mu.Unlock()
	return nil
}

// Remove removes either a single file or recursed directory from
// the Watcher's file list.
func (w *Watcher) Remove(name string) error {
	// Remove the name from w's Names list.
	w.mu.Lock()
	for i := range w.Names {
		if w.Names[i] == name {
			w.Names = append(w.Names[:i], w.Names[i+1:]...)
		}
	}
	w.mu.Unlock()

	// Make sure name exists.
	fInfo, err := os.Stat(name)
	if err != nil {
		return err
	}

	// If name is a single file, remove it and return.
	if !fInfo.IsDir() {
		w.mu.Lock()
		delete(w.Files, fInfo.Name())
		w.mu.Unlock()
		return nil
	}

	// Retrieve a list of all of the os.FileInfo's to delete from w.Files.
	fInfoList, err := ListFiles(name, w.options...)
	if err != nil {
		return err
	}

	// Remove the appropriate os.FileInfo's from w's os.FileInfo list.
	w.mu.Lock()
	for path := range fInfoList {
		delete(w.Files, path)
	}
	w.mu.Unlock()
	return nil
}

// TriggerEvent is a method that can be used to trigger an event, separate to
// the file watching process.
func (w *Watcher) TriggerEvent(eventType EventType, file os.FileInfo) {
	if file == nil {
		file = &fileInfo{name: "triggered event", modTime: time.Now()}
	}
	w.Event <- Event{eventType, file}
}

// Start starts the watching process and checks for changes every `pollInterval` duration.
// If pollInterval is 0, the default is 100ms.
func (w *Watcher) Start(pollInterval time.Duration) error {
	if pollInterval <= 0 {
		pollInterval = time.Millisecond * 100
	}

	if len(w.Names) < 1 {
		return ErrNothingAdded
	}

	for {
		fileList := make(map[string]os.FileInfo)
		for _, name := range w.Names {
			// Retrieve the list of os.FileInfo's from w.Name.
			list, err := ListFiles(name, w.options...)
			if err != nil {
				if os.IsNotExist(err) {
					w.Error <- ErrWatchedFileDeleted
				} else {
					w.Error <- err
				}
			}
			for k, v := range list {
				fileList[k] = v
			}
		}

		numEvents := 0
		addedAndDeleted := make(map[string]struct{})

		// Check for added files.
		for path, file := range fileList {
			if w.maxEventsPerCycle > 0 && numEvents >= w.maxEventsPerCycle {
				goto SLEEP
			}
			if _, found := w.Files[path]; !found {
				addedAndDeleted[path] = struct{}{}
				w.Event <- Event{EventType: EventFileAdded, FileInfo: file}
				numEvents++
			}
		}

		// Check for deleted files.
		for path, file := range w.Files {
			if w.maxEventsPerCycle > 0 && numEvents >= w.maxEventsPerCycle {
				goto SLEEP
			}
			if _, found := fileList[path]; !found {
				addedAndDeleted[path] = struct{}{}
				w.Event <- Event{EventType: EventFileDeleted, FileInfo: file}
				numEvents++
			}
		}

		// Check for modified files.
		for path, file := range w.Files {
			if w.maxEventsPerCycle > 0 && numEvents >= w.maxEventsPerCycle {
				goto SLEEP
			}
			if _, found := addedAndDeleted[path]; !found {
				if fileList[path].ModTime() != file.ModTime() {
					w.Event <- Event{EventType: EventFileModified, FileInfo: file}
					numEvents++
				}
			}
		}

	SLEEP:
		// Update w.Files and then sleep for a little bit.
		w.Files = fileList
		time.Sleep(pollInterval)
	}

	return nil
}

// hasOption returns true or false based on whether or not
// an Option exists in an Option slice.
func hasOption(options []Option, option Option) bool {
	for _, o := range options {
		if option&o != 0 {
			return true
		}
	}
	return false
}

// ListFiles returns a map of all os.FileInfo's recursively
// contained in a directory. If name is a single file, it returns
// an os.FileInfo map containing a single os.FileInfo.
func ListFiles(name string, options ...Option) (map[string]os.FileInfo, error) {
	fileList := make(map[string]os.FileInfo)

	name = filepath.Clean(name)

	nonRecursive := hasOption(options, NonRecursive)
	ignoreDotFiles := hasOption(options, IgnoreDotFiles)

	if nonRecursive {
		f, err := os.Open(name)
		if err != nil {
			return nil, err
		}
		defer f.Close()
		info, err := os.Stat(name)
		if err != nil {
			return nil, err
		}
		// Add the name to fileList.
		if !info.IsDir() && ignoreDotFiles && strings.HasPrefix(name, ".") {
			return fileList, nil
		}
		fileList[name] = info
		if !info.IsDir() {
			return fileList, nil
		}
		// It's a directory, read it's contents.
		fInfoList, err := f.Readdir(-1)
		if err != nil {
			return nil, err
		}
		// Add all of the FileInfo's returned from f.ReadDir to fileList.
		for _, fInfo := range fInfoList {
			if ignoreDotFiles && strings.HasPrefix(fInfo.Name(), ".") {
				continue
			}
			fileList[filepath.Join(name, fInfo.Name())] = fInfo
		}
		return fileList, nil
	}

	if err := filepath.Walk(name, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if ignoreDotFiles && strings.HasPrefix(info.Name(), ".") {
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		fileList[path] = info

		return nil
	}); err != nil {
		return nil, err
	}

	return fileList, nil
}
<|endoftext|>


<commit_before>package revel

import (
	"gopkg.in/fsnotify.v1"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
)

// Listener is an interface for receivers of filesystem events.
type Listener interface {
	// Refresh is invoked by the watcher on relevant filesystem events.
	// If the listener returns an error, it is served to the user on the current request.
	Refresh() *Error
}

// DiscerningListener allows the receiver to selectively watch files.
type DiscerningListener interface {
	Listener
	WatchDir(info os.FileInfo) bool
	WatchFile(basename string) bool
}

// Watcher allows listeners to register to be notified of changes under a given
// directory.
type Watcher struct {
	// Parallel arrays of watcher/listener pairs.
	watchers     []*fsnotify.Watcher
	listeners    []Listener
	forceRefresh bool
	lastError    int
	notifyMutex  sync.Mutex
}

func NewWatcher() *Watcher {
	return &Watcher{
		forceRefresh: true,
		lastError:    -1,
	}
}

// Listen registers for events within the given root directories (recursively).
func (w *Watcher) Listen(listener Listener, roots ...string) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		ERROR.Fatal(err)
	}

	// Replace the unbuffered Event channel with a buffered one.
	// Otherwise multiple change events only come out one at a time, across
	// multiple page views.  (There appears no way to "pump" the events out of
	// the watcher)
	watcher.Events = make(chan fsnotify.Event, 100)
	watcher.Errors = make(chan error, 10)

	// Walk through all files / directories under the root, adding each to watcher.
	for _, p := range roots {
		// is the directory / file a symlink?
		f, err := os.Lstat(p)
		if err == nil && f.Mode()&os.ModeSymlink == os.ModeSymlink {
			realPath, err := filepath.EvalSymlinks(p)
			if err != nil {
				panic(err)
			}
			p = realPath
		}

		fi, err := os.Stat(p)
		if err != nil {
			ERROR.Println("Failed to stat watched path", p, ":", err)
			continue
		}

		// If it is a file, watch that specific file.
		if !fi.IsDir() {
			err = watcher.Add(p)
			if err != nil {
				ERROR.Println("Failed to watch", p, ":", err)
			}
			continue
		}

		var watcherWalker func(path string, info os.FileInfo, err error) error

		watcherWalker = func(path string, info os.FileInfo, err error) error {
			if err != nil {
				ERROR.Println("Error walking path:", err)
				return nil
			}

			// is it a symlinked template?
			link, err := os.Lstat(path)
			if err == nil && link.Mode()&os.ModeSymlink == os.ModeSymlink {
				TRACE.Println("Watcher symlink: ", path)
				// lookup the actual target & check for goodness
				targetPath, err := filepath.EvalSymlinks(path)
				if err != nil {
					ERROR.Println("Failed to read symlink", err)
					return err
				}
				targetInfo, err := os.Stat(targetPath)
				if err != nil {
					ERROR.Println("Failed to stat symlink target", err)
					return err
				}

				// set the template path to the target of the symlink
				path = targetPath
				info = targetInfo
				filepath.Walk(path, watcherWalker)
			}

			if info.IsDir() {
				if dl, ok := listener.(DiscerningListener); ok {
					if !dl.WatchDir(info) {
						return filepath.SkipDir
					}
				}

				err = watcher.Add(path)
				if err != nil {
					ERROR.Println("Failed to watch", path, ":", err)
				}
			}
			return nil
		}

		// Else, walk the directory tree.
		filepath.Walk(p, watcherWalker)
	}

	if w.eagerRebuildEnabled() {
		// Create goroutine to notify file changes in real time
		go w.NotifyWhenUpdated(listener, watcher)
	}

	w.watchers = append(w.watchers, watcher)
	w.listeners = append(w.listeners, listener)
}

// Wait until receiving a file change event.
// When a watcher receives an event, the watcher notifies it.
func (w *Watcher) NotifyWhenUpdated(listener Listener, watcher *fsnotify.Watcher) {
	for {
		// Do not catch default for performance.
		select {
		case ev := <-watcher.Event:
			// Ignore changes to dotfiles.
			if !strings.HasPrefix(path.Base(ev.Name), ".") {
				if dl, ok := listener.(DiscerningListener); ok {
					if !dl.WatchFile(ev.Name) || ev.IsAttrib() {
						continue
					}
				}

				// Use w.Notify() instead of listener.Refresh() to serialize w.Notify() calling
				w.forceRefresh = true
				go w.Notify()
			}
		case <-watcher.Error:
			continue
		}
	}
}

// Notify causes the watcher to forward any change events to listeners.
// It returns the first (if any) error returned.
func (w *Watcher) Notify() *Error {
	// Serialize Notify() calls.
	w.notifyMutex.Lock()
	defer w.notifyMutex.Unlock()

	for i, watcher := range w.watchers {
		listener := w.listeners[i]

		// Pull all pending events / errors from the watcher.
		// When rebuild.eager is true, NotifyWhenUpdated goroutine receives the events.
		refresh := false
		if !w.eagerRebuildEnabled() {
			for {
				select {
				case ev := <-watcher.Event:
					// Ignore changes to dotfiles.
					if !strings.HasPrefix(path.Base(ev.Name), ".") {
						if dl, ok := listener.(DiscerningListener); ok {
							if !dl.WatchFile(ev.Name) || ev.IsAttrib() {
								continue
							}
						}

						refresh = true
					}
					continue
				case <-watcher.Error:
					continue
				default:
					// No events left to pull
				}
				break
			}
		}

		if w.forceRefresh || refresh || w.lastError == i {
			err := listener.Refresh()
			if err != nil {
				w.lastError = i
				return err
			}
		}
	}

	w.forceRefresh = false
	w.lastError = -1
	return nil
}

// If the rebuild.eager config is set true, watcher rebuilds an application
// every time a source file is changed.
// This feature is available only in dev mode.
func (w *Watcher) eagerRebuildEnabled() bool {
	return Config.BoolDefault("mode.dev", true) &&
		Config.BoolDefault("watch", true) &&
		Config.BoolDefault("rebuild.eager", false)
}

var WatchFilter = func(c *Controller, fc []Filter) {
	if MainWatcher != nil {
		err := MainWatcher.Notify()
		if err != nil {
			c.Result = c.RenderError(err)
			return
		}
	}
	fc[0](c, fc[1:])
}
<commit_msg>Refactor about Notify flow<commit_after>package revel

import (
	"gopkg.in/fsnotify.v1"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
)

// Listener is an interface for receivers of filesystem events.
type Listener interface {
	// Refresh is invoked by the watcher on relevant filesystem events.
	// If the listener returns an error, it is served to the user on the current request.
	Refresh() *Error
}

// DiscerningListener allows the receiver to selectively watch files.
type DiscerningListener interface {
	Listener
	WatchDir(info os.FileInfo) bool
	WatchFile(basename string) bool
}

// Watcher allows listeners to register to be notified of changes under a given
// directory.
type Watcher struct {
	// Parallel arrays of watcher/listener pairs.
	watchers     []*fsnotify.Watcher
	listeners    []Listener
	forceRefresh bool
	lastError    int
	notifyMutex  sync.Mutex
}

func NewWatcher() *Watcher {
	return &Watcher{
		forceRefresh: true,
		lastError:    -1,
	}
}

// Listen registers for events within the given root directories (recursively).
func (w *Watcher) Listen(listener Listener, roots ...string) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		ERROR.Fatal(err)
	}

	// Replace the unbuffered Event channel with a buffered one.
	// Otherwise multiple change events only come out one at a time, across
	// multiple page views.  (There appears no way to "pump" the events out of
	// the watcher)
	watcher.Events = make(chan fsnotify.Event, 100)
	watcher.Errors = make(chan error, 10)

	// Walk through all files / directories under the root, adding each to watcher.
	for _, p := range roots {
		// is the directory / file a symlink?
		f, err := os.Lstat(p)
		if err == nil && f.Mode()&os.ModeSymlink == os.ModeSymlink {
			realPath, err := filepath.EvalSymlinks(p)
			if err != nil {
				panic(err)
			}
			p = realPath
		}

		fi, err := os.Stat(p)
		if err != nil {
			ERROR.Println("Failed to stat watched path", p, ":", err)
			continue
		}

		// If it is a file, watch that specific file.
		if !fi.IsDir() {
			err = watcher.Add(p)
			if err != nil {
				ERROR.Println("Failed to watch", p, ":", err)
			}
			continue
		}

		var watcherWalker func(path string, info os.FileInfo, err error) error

		watcherWalker = func(path string, info os.FileInfo, err error) error {
			if err != nil {
				ERROR.Println("Error walking path:", err)
				return nil
			}

			// is it a symlinked template?
			link, err := os.Lstat(path)
			if err == nil && link.Mode()&os.ModeSymlink == os.ModeSymlink {
				TRACE.Println("Watcher symlink: ", path)
				// lookup the actual target & check for goodness
				targetPath, err := filepath.EvalSymlinks(path)
				if err != nil {
					ERROR.Println("Failed to read symlink", err)
					return err
				}
				targetInfo, err := os.Stat(targetPath)
				if err != nil {
					ERROR.Println("Failed to stat symlink target", err)
					return err
				}

				// set the template path to the target of the symlink
				path = targetPath
				info = targetInfo
				filepath.Walk(path, watcherWalker)
			}

			if info.IsDir() {
				if dl, ok := listener.(DiscerningListener); ok {
					if !dl.WatchDir(info) {
						return filepath.SkipDir
					}
				}

				err = watcher.Add(path)
				if err != nil {
					ERROR.Println("Failed to watch", path, ":", err)
				}
			}
			return nil
		}

		// Else, walk the directory tree.
		filepath.Walk(p, watcherWalker)
	}

	if w.eagerRebuildEnabled() {
		// Create goroutine to notify file changes in real time
		go w.NotifyWhenUpdated(listener, watcher)
	}

	w.watchers = append(w.watchers, watcher)
	w.listeners = append(w.listeners, listener)
}

// Wait until receiving a file change event.
// When a watcher receives an event, the watcher notifies it.
func (w *Watcher) NotifyWhenUpdated(listener Listener, watcher *fsnotify.Watcher) {
	for {
		// Do not catch default for performance.
		select {
		case ev := <-watcher.Event:
			if w.rebuildRequired(ev, listener) {
				// Seriarize listener.Refresh() calls.
				w.notifyMutex.Lock()
				listener.Refresh()
				w.notifyMutex.Unlock()
			}
		case <-watcher.Error:
			continue
		}
	}
}

// Notify causes the watcher to forward any change events to listeners.
// It returns the first (if any) error returned.
func (w *Watcher) Notify() *Error {
	// Serialize Notify() calls.
	w.notifyMutex.Lock()
	defer w.notifyMutex.Unlock()

	for i, watcher := range w.watchers {
		listener := w.listeners[i]

		// Pull all pending events / errors from the watcher.
		refresh := false
		for {
			select {
			case ev := <-watcher.Event:
				if w.rebuildRequired(ev, listener) {
					refresh = true
				}
				continue
			case <-watcher.Error:
				continue
			default:
				// No events left to pull
			}
			break
		}

		if w.forceRefresh || refresh || w.lastError == i {
			err := listener.Refresh()
			if err != nil {
				w.lastError = i
				return err
			}
		}
	}

	w.forceRefresh = false
	w.lastError = -1
	return nil
}

// If the rebuild.eager config is set true, watcher rebuilds an application
// every time a source file is changed.
// This feature is available only in dev mode.
func (w *Watcher) eagerRebuildEnabled() bool {
	return Config.BoolDefault("mode.dev", true) &&
		Config.BoolDefault("watch", true) &&
		Config.BoolDefault("rebuild.eager", false)
}

func (w *Watcher) rebuildRequired(ev *fsnotify.FileEvent, listener Listener) bool {
	// Ignore changes to dotfiles.
	if strings.HasPrefix(path.Base(ev.Name), ".") {
		return false
	}

	if dl, ok := listener.(DiscerningListener); ok {
		if !dl.WatchFile(ev.Name) || ev.IsAttrib() {
			return false
		}
	}
	return true
}

var WatchFilter = func(c *Controller, fc []Filter) {
	if MainWatcher != nil {
		err := MainWatcher.Notify()
		if err != nil {
			c.Result = c.RenderError(err)
			return
		}
	}
	fc[0](c, fc[1:])
}
<|endoftext|>


<commit_before>// watches the current directory for changes and runs the specificed program on change
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
	"github.com/howeyc/fsnotify"
)

var (
	verbose = flag.Bool("v", false, "verbose")
	depth   = flag.Int("depth", 1, "recursion depth")
	dir     = flag.String("dir", ".", "directory root to use for watching")
	quiet   = flag.Duration("quiet", 800*time.Millisecond, "quiet period after command execution")
)

func usage() {
	fmt.Fprintf(os.Stderr, "usage: %s [flags] [command to execute and args]\n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	flag.Usage = usage
	flag.Parse()

	watcher, err := newWatcher()
	if err != nil {
		log.Fatal(err)
	}
	if len(flag.Args()) == 0 {
		flag.Usage()
		os.Exit(1)
	}
	cmd, args := flag.Args()[0], flag.Args()[1:]

	fileEvents := make(chan interface{}, 100)

	// pipe all events to fileEvents (for buffering and draining)
	go watcher.pipeEvents(fileEvents)

	go watchAndExecute(fileEvents, cmd, args)

	dir, err := filepath.Abs(*dir)
	if err != nil {
		log.Fatal(err)
	}
	err = watcher.watchDirAndChildren(dir, *depth)
	if err != nil {
		log.Fatal(err)
	}
	<-make(chan struct{})
	watcher.Close()
}

type watcher struct {
	*fsnotify.Watcher
}

func newWatcher() (watcher, error) {
	fsnw, err := fsnotify.NewWatcher()
	return watcher{fsnw}, err
}

// Execute cmd with args when a file event occurs
func watchAndExecute(fileEvents chan interface{}, cmd string, args []string) {
	for {
		// execute command
		c := exec.Command(cmd, args...)
		c.Stdout = os.Stdout
		c.Stderr = os.Stderr
		c.Stdin = os.Stdin

		fmt.Fprintln(os.Stderr, "running", cmd, args)
		if err := c.Run(); err != nil {
			fmt.Fprintln(os.Stderr, "error running:", err)
		}
		if *verbose {
			fmt.Fprintln(os.Stderr, "done.")
		}
		// drain until quiet period is over
		drainFor(*quiet, fileEvents)
		ev := <-fileEvents
		if *verbose {
			fmt.Fprintln(os.Stderr, "File changed:", ev)
		}
	}
}

// Add dir and children (recursively) to watcher
func (w watcher) watchDirAndChildren(path string, depth int) error {
	if err := w.Watch(path); err != nil {
		return err
	}
	baseNumSeps := strings.Count(path, string(os.PathSeparator))
	return filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			pathDepth := strings.Count(path, string(os.PathSeparator)) - baseNumSeps
			if pathDepth > depth {
				return filepath.SkipDir
			}
			if *verbose {
				fmt.Fprintln(os.Stderr, "Watching", path)
			}
			if err := w.Watch(path); err != nil {
				return err
			}
		}
		return nil
	})
}

// pipeEvents sends valid events to `events` and errors to stderr
func (w watcher) pipeEvents(events chan interface{}) {
	for {
		select {
		case ev := <-w.Event:
			events <- ev
			// @todo handle created/renamed/deleted dirs
		case err := <-w.Error:
			log.Println("fsnotify error:", err)
		}
	}
}

// drainFor drains events from channel with a until a period in ms has elapsed timeout
func drainFor(drainUntil time.Duration, c chan interface{}) {
	timeout := time.After(drainUntil)
	for {
		select {
		case <-c:
		case <-timeout:
			return
		}
	}
}
<commit_msg>Add basic ignore pattern<commit_after>// watches the current directory for changes and runs the specificed program on change
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/howeyc/fsnotify"
)

var (
	verbose = flag.Bool("v", false, "verbose")
	depth   = flag.Int("depth", 1, "recursion depth")
	dir     = flag.String("dir", ".", "directory root to use for watching")
	quiet   = flag.Duration("quiet", 800*time.Millisecond, "quiet period after command execution")
	ignore  = flag.String("ignore", "", "path ignore pattern")
)

func usage() {
	fmt.Fprintf(os.Stderr, "usage: %s [flags] [command to execute and args]\n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	flag.Usage = usage
	flag.Parse()

	watcher, err := newWatcher()
	if err != nil {
		log.Fatal(err)
	}
	if len(flag.Args()) == 0 {
		flag.Usage()
		os.Exit(1)
	}
	cmd, args := flag.Args()[0], flag.Args()[1:]

	fileEvents := make(chan interface{}, 100)

	// pipe all events to fileEvents (for buffering and draining)
	go watcher.pipeEvents(fileEvents)

	// if we have an ignore pattern, set up predicate and replace fileEvents
	if *ignore != "" {
		fileEvents = filter(fileEvents, func(e interface{}) bool {
			fe := e.(*fsnotify.FileEvent)
			ignored, err := filepath.Match(*ignore, filepath.Base(fe.Name))
			if err != nil {
				fmt.Fprintln(os.Stderr, "error performing match:", err)
			}
			return !ignored
		})
	}

	go watchAndExecute(fileEvents, cmd, args)

	dir, err := filepath.Abs(*dir)
	if err != nil {
		log.Fatal(err)
	}
	err = watcher.watchDirAndChildren(dir, *depth)
	if err != nil {
		log.Fatal(err)
	}
	<-make(chan struct{})
	watcher.Close()
}

type watcher struct {
	*fsnotify.Watcher
}

func newWatcher() (watcher, error) {
	fsnw, err := fsnotify.NewWatcher()
	return watcher{fsnw}, err
}

// Execute cmd with args when a file event occurs
func watchAndExecute(fileEvents chan interface{}, cmd string, args []string) {
	for {
		// execute command
		c := exec.Command(cmd, args...)
		c.Stdout = os.Stdout
		c.Stderr = os.Stderr
		c.Stdin = os.Stdin

		fmt.Fprintln(os.Stderr, "running", cmd, args)
		if err := c.Run(); err != nil {
			fmt.Fprintln(os.Stderr, "error running:", err)
		}
		if *verbose {
			fmt.Fprintln(os.Stderr, "done.")
		}
		// drain until quiet period is over
		drainFor(*quiet, fileEvents)
		ev := <-fileEvents
		if *verbose {
			fmt.Fprintln(os.Stderr, "File changed:", ev)
		}
	}
}

// Add dir and children (recursively) to watcher
func (w watcher) watchDirAndChildren(path string, depth int) error {
	if err := w.Watch(path); err != nil {
		return err
	}
	baseNumSeps := strings.Count(path, string(os.PathSeparator))
	return filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			pathDepth := strings.Count(path, string(os.PathSeparator)) - baseNumSeps
			if pathDepth > depth {
				return filepath.SkipDir
			}
			if *verbose {
				fmt.Fprintln(os.Stderr, "Watching", path)
			}
			if err := w.Watch(path); err != nil {
				return err
			}
		}
		return nil
	})
}

// pipeEvents sends valid events to `events` and errors to stderr
func (w watcher) pipeEvents(events chan interface{}) {
	for {
		select {
		case ev := <-w.Event:
			events <- ev
			// @todo handle created/renamed/deleted dirs
		case err := <-w.Error:
			log.Println("fsnotify error:", err)
		}
	}
}

func filter(items chan interface{}, predicate func(interface{}) bool) chan interface{} {
	results := make(chan interface{})
	go func() {
		for {
			item := <-items
			if predicate(item) {
				results <- item
			}
		}
	}()
	return results
}

// drainFor drains events from channel with a until a period in ms has elapsed timeout
func drainFor(drainUntil time.Duration, c chan interface{}) {
	timeout := time.After(drainUntil)
	for {
		select {
		case <-c:
		case <-timeout:
			return
		}
	}
}
<|endoftext|>


<commit_before>package web

import (
	"github.com/varunamachi/orek/data"
	"time"
	"github.com/gocraft/web"
)

type Context struct{
	SessionUser 	data.User
	CreationTime	time.Duration
}

func (c *Context) Login(rw web.ResponseWriter, req *web.Request, next web.NextMiddlewareFunc) {

}

func Setup() error {

	return  nil
}<commit_msg>Added API stubs<commit_after>package web

import (
	"github.com/gocraft/web"
	"github.com/varunamachi/orek/data"
	"net/http"
	"time"
)

type Context struct {
	SessionUser  data.User
	CreationTime time.Duration
}

func (c *Context) SessionChecker(rw web.ResponseWriter,
	req *web.Request,
	next web.NextMiddlewareFunc) {
	//get the session ID
	//see if it is still valid
	//put the user and session objects into the context object
	next(req, req)
}

func (c *Context) Login(resp http.ResponseWriter, req *http.Request) {
	//Get the user name and password from the request
	//Authenticate the user and create the user object, put the user into
	// context
	//Check if the current session already registered to some other login
	//if so expire this session
	//Create a new session in the session table
}

func (c *Context) Logout(resp http.ResponseWriter, req *http.Request) {

}

func (c *Context) GetAllUsers(resp http.ResponseWriter, req *http.Request) {

}

func (c *Context) GetUser(resp http.ResponseWriter, req *http.Request) {

}

func (c *Context) CreateUser(resp http.ResponseWriter, req *http.Request) {

}

func (c *Context) DeleteUser(resp http.ResponseWriter, req *http.Request) {

}

func (c *Context) GetAllSources(resp http.ResponseWriter, req *http.Request) {

}

func (c *Context) GetSource(resp http.ResponseWriter, req *http.Request) {

}

func (c *Context) GetSourceWithId(resp http.ResponseWriter, req *http.Request) {

}

func (c *Context) CreateOrUpdateSource(resp http.ResponseWriter,
	req *http.Request) {

}

func (c *Context) DeleteSource(resp http.ResponseWriter, req *http.Request) {

}

func (c *Context) GetAllVariables(resp http.ResponseWriter, req *http.Request) {

}

func (c *Context) GetVariable(resp http.ResponseWriter, req *http.Request) {

}

func (c *Context) GetVariableWithId(resp http.ResponseWriter,
	req *http.Request) {

}

func (c *Context) CreateOrUpdateVariable(resp http.ResponseWriter,
	req *http.Request) {

}

func (c *Context) DeleteVariable(resp http.ResponseWriter, req *http.Request) {

}

func (c *Context) GetAllUserGroups(resp http.ResponseWriter,
	req *http.Request) {

}

func (c *Context) GetUserGroup(resp http.ResponseWriter, req *http.Request) {

}

func (c *Context) CreateOrUpdateUserGroup(resp http.ResponseWriter,
	req *http.Request) {

}

func (c *Context) DeleteUserGroup(resp http.ResponseWriter, req *http.Request) {

}

func (c *Context) GetAllVariableGroups(resp http.ResponseWriter,
	req *http.Request) {

}

func (c *Context) GetVariableGroup(resp http.ResponseWriter,
	req *http.Request) {

}

func (c *Context) GetVariableGroupWithId(resp http.ResponseWriter,
	req *http.Request) {

}

func (c *Context) CreateOrUpdateVariableGroup(resp http.ResponseWriter,
	req *http.Request) {

}

func (c *Context) DeleteVariableGroup(resp http.ResponseWriter,
	req *http.Request) {

}

func (c *Context) AddUserToGroup(resp http.ResponseWriter, req *http.Request) {

}

func (c *Context) RemoveUserFromGroup(resp http.ResponseWriter,
	req *http.Request) {

}

func (c *Context) GetUsersInGroup(resp http.ResponseWriter, req *http.Request) {

}

func (c *Context) GetGroupsForUser(resp http.ResponseWriter,
	req *http.Request) {

}

func (c *Context) AddVariableToGroup(resp http.ResponseWriter,
	req *http.Request) {

}

func (c *Context) RemoveVariableFromGroup(resp http.ResponseWriter,
	req *http.Request) {

}

func (c *Context) GetVariablesInGroup(resp http.ResponseWriter,
	req *http.Request) {

}

func (c *Context) GetGroupsForVariable(resp http.ResponseWriter,
    req *http.Request) {

}

func (c *Context) AddVariableValue(resp http.ResponseWriter,
    req *http.Request) {

}

func (c *Context) ClearValuesForVariable(resp http.ResponseWriter,
    req *http.Request) {

}

func (c *Context) GetValuesForVariable(resp http.ResponseWriter,
    req *http.Request) {

}

func Setup() error {

	return nil
}
<|endoftext|>


<commit_before>package web

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/gocraft/web"
	"github.com/varunamachi/orek/data"
)

type Context struct {
	SessionUser  data.User
	CreationTime time.Duration
}

func checkSession(context *Context) bool {
	return true
}

func (c *Context) SessionChecker(rw web.ResponseWriter,
	req *web.Request,
	next web.NextMiddlewareFunc) {
	//get the session ID
	//see if it is still valid
	//put the user and session objects into the context object
	next(rw, req)
}

func (c *Context) Login(resp web.ResponseWriter, req *web.Request) {
	//Get the user name and password from the request
	//Authenticate the user and create the user object, put the user into
	// context
	//Check if the current session already registered to some other login
	//if so expire this session
	//Create a new session in the session table
	//	userName := req.FormValue("userName")
	//	password := req.FormValue("password")
	//	sessionId := req.FormValue("sessionId")
	//	data.DataSource().GetUser("varun").FirstName

	//	data.DataSource().GetUser(
}

func (c *Context) Logout(resp web.ResponseWriter, req *web.Request) {
	//resp.Write
}

func (c *Context) GetAllUsers(resp web.ResponseWriter, req *web.Request) {
	users, err := data.DataSource().GetAllUsers()
	if err == nil {
		marshalled, err := json.Marshal(users)
		if err == nil {
			fmt.Fprintf(resp, string(marshalled))
		} else {
			fmt.Fprintf(resp, "!Error:MarshalError")
		}
	} else {
		fmt.Fprintf(resp, "!Error:DataSourceError")
	}
}

func (c *Context) GetUser(resp web.ResponseWriter, req *web.Request) {
	userName := req.PathParams["userName"]
	user, err := data.DataSource().GetUser(userName)
	if err == nil {
		mrsh, err := json.Marshal(user)
		if err == nil {
			fmt.Fprintf(resp, string(mrsh))
		} else {
			fmt.Fprintf(resp, "!Error:Marshal Error")
		}
	} else {
		fmt.Fprintf(resp, "!Error:DataSource Error")
	}
}

func (c *Context) CreateUser(resp web.ResponseWriter, req *web.Request) {
	decoder := json.NewDecoder(req.Body)
	var user data.User
	if err := decoder.Decode(&user); err == nil {
		err := data.DataSource().CreateOrUpdateUser(&user)
		if err != nil {
			fmt.Fprintf(resp, "!Error:DataSource Error")
		}
	} else {
		//TODO: Create a structure call OrekError and serialize it
		fmt.Fprintf(resp, "!Error:JSON Decode Error")
	}

}

func (c *Context) DeleteUser(resp web.ResponseWriter, req *web.Request) {
	userName := req.PathParams["userName"]
	if len(userName) > 0 {
		err := data.DataSource().DeleteUser(userName)
		if err != nil {
			//data source error
		}
	} else {
		//error invalid user name
	}
}

func (c *Context) GetAllSources(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) GetSource(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) GetSourceWithId(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) CreateOrUpdateSource(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) DeleteSource(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) GetAllVariables(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) GetVariable(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) GetVariableWithId(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) CreateOrUpdateVariable(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) DeleteVariable(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) GetAllUserGroups(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) GetUserGroup(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) CreateOrUpdateUserGroup(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) DeleteUserGroup(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) GetAllVariableGroups(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) GetVariableGroup(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) GetVariableGroupWithId(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) CreateOrUpdateVariableGroup(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) DeleteVariableGroup(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) AddUserToGroup(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) RemoveUserFromGroup(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) GetUsersInGroup(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) GetGroupsForUser(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) AddVariableToGroup(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) RemoveVariableFromGroup(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) GetVariablesInGroup(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) GetGroupsForVariable(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) AddVariableValue(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) ClearValuesForVariable(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) GetValuesForVariable(resp web.ResponseWriter,
	req *web.Request) {

}

func Setup() error {

	return nil
}
<commit_msg>More APIs<commit_after>package web

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/gocraft/web"
	"github.com/varunamachi/orek/data"
)

type Context struct {
	SessionUser  data.User
	CreationTime time.Duration
}

func checkSession(context *Context) bool {
	return true
}

func (c *Context) SessionChecker(rw web.ResponseWriter,
	req *web.Request,
	next web.NextMiddlewareFunc) {
	//get the session ID
	//see if it is still valid
	//put the user and session objects into the context object
	next(rw, req)
}

func (c *Context) Login(resp web.ResponseWriter, req *web.Request) {
	//Get the user name and password from the request
	//Authenticate the user and create the user object, put the user into
	// context
	//Check if the current session already registered to some other login
	//if so expire this session
	//Create a new session in the session table
	//	userName := req.FormValue("userName")
	//	password := req.FormValue("password")
	//	sessionId := req.FormValue("sessionId")
	//	data.DataSource().GetUser("varun").FirstName

	//	data.DataSource().GetUser(
}

func (c *Context) Logout(resp web.ResponseWriter, req *web.Request) {
	//resp.Write
}

func (c *Context) GetAllUsers(resp web.ResponseWriter, req *web.Request) {
	users, err := data.DataSource().GetAllUsers()
	if err == nil {
		marshalled, err := json.Marshal(users)
		if err == nil {
			fmt.Fprintf(resp, string(marshalled))
		} else {
			emsg, _ := json.Marshal(OrekError{
				"MarshalError",
				"Failed to marshal user list"})
			fmt.Fprintf(resp, string(emsg))
		}
	} else {
		emsg, _ := json.Marshal(OrekError{
			"DataSourceError",
			"Failed retrieve user list from datasource"})
		fmt.Fprintf(resp, string(emsg))
	}
}

func (c *Context) GetUser(resp web.ResponseWriter, req *web.Request) {
	userName := req.PathParams["userName"]
	user, err := data.DataSource().GetUser(userName)
	if err == nil {
		mrsh, err := json.Marshal(user)
		if err == nil {
			fmt.Fprintf(resp, string(mrsh))
		} else {
			emsg, _ := json.Marshal(OrekError{
				"MarshalError",
				"Failed to marshal user details"})
			fmt.Fprintf(resp, string(emsg))
		}
	} else {
		emsg, _ := json.Marshal(OrekError{
			"DataSourceError",
			"Failed to fetch user detail from data source"})
		fmt.Fprintf(resp, string(emsg))
	}
}

func (c *Context) CreateUser(resp web.ResponseWriter, req *web.Request) {
	decoder := json.NewDecoder(req.Body)
	var user data.User
	if err := decoder.Decode(&user); err == nil {
		err := data.DataSource().CreateOrUpdateUser(&user)
		if err != nil {
			emsg, _ := json.Marshal(OrekError{
				"MarshalError",
				"Failed to marshal user creation result"})
			fmt.Fprintf(resp, string(emsg))
		}
	} else {
		//TODO: Create a structure call OrekError and serialize it
		fmt.Fprintf(resp, "!Error:JSON Decode Error")
	}

}

func (c *Context) DeleteUser(resp web.ResponseWriter, req *web.Request) {
	userName := req.PathParams["userName"]
	if len(userName) > 0 {
		err := data.DataSource().DeleteUser(userName)
		if err != nil {
			//data source error
		}
	} else {
		//error invalid user name
	}
}

func (c *Context) GetAllSources(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) GetSource(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) GetSourceWithId(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) CreateOrUpdateSource(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) DeleteSource(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) GetAllVariables(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) GetVariable(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) GetVariableWithId(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) CreateOrUpdateVariable(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) DeleteVariable(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) GetAllUserGroups(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) GetUserGroup(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) CreateOrUpdateUserGroup(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) DeleteUserGroup(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) GetAllVariableGroups(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) GetVariableGroup(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) GetVariableGroupWithId(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) CreateOrUpdateVariableGroup(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) DeleteVariableGroup(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) AddUserToGroup(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) RemoveUserFromGroup(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) GetUsersInGroup(resp web.ResponseWriter, req *web.Request) {

}

func (c *Context) GetGroupsForUser(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) AddVariableToGroup(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) RemoveVariableFromGroup(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) GetVariablesInGroup(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) GetGroupsForVariable(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) AddVariableValue(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) ClearValuesForVariable(resp web.ResponseWriter,
	req *web.Request) {

}

func (c *Context) GetValuesForVariable(resp web.ResponseWriter,
	req *web.Request) {

}

func Setup() error {

	return nil
}
<|endoftext|>


<commit_before>package crawler

import (
	"context"
	"fmt"
	"log"
	"time"

	"go.opentelemetry.io/otel/label"

	index_types "github.com/ipfs-search/ipfs-search/components/index/types"
	t "github.com/ipfs-search/ipfs-search/types"
)

func appendReference(refs index_types.References, r *t.Reference) (index_types.References, bool) {
	if r.Parent == nil {
		// No new reference, not updating
		return refs, false
	}

	for _, indexedRef := range refs {
		if indexedRef.ParentHash == r.Parent.ID && indexedRef.Name == r.Name {
			// Existing reference, not updating
			return refs, false
		}
	}

	return append(refs, index_types.Reference{
		ParentHash: r.Parent.ID,
		Name:       r.Name,
	}), true
}

// updateExisting updates known existing items.
func (c *Crawler) updateExisting(ctx context.Context, i *existingItem) error {
	ctx, span := c.Tracer.Start(ctx, "crawler.updateExisting")
	defer span.End()

	switch i.Source {
	case t.DirectorySource:
		// Item referenced from a directory, consider updating references (but not last-seen).
		refs, refsUpdated := appendReference(i.References, &i.AnnotatedResource.Reference)

		if refsUpdated {
			span.AddEvent(ctx, "Updating",
				label.String("reason", "reference-added"),
				label.Any("new-reference", i.AnnotatedResource.Reference),
			)

			return i.Index.Update(ctx, i.AnnotatedResource.ID, &index_types.Update{
				References: refs,
			})
		}

	case t.SnifferSource, t.UnknownSource:
		// TODO: Remove UnknownSource after sniffer is updated and queue is flushed.
		// Item sniffed, conditionally update last-seen.
		now := time.Now()

		// Strip milliseconds to cater to legacy ES index format.
		// This can be safely removed after the next reindex with _nomillis removed from time format.
		now = now.Truncate(time.Second)

		isRecent := now.Sub(*i.LastSeen) > c.config.MinUpdateAge

		if isRecent {
			span.AddEvent(ctx, "Updating",
				label.String("reason", "is-recent"),
				label.Stringer("last-seen", i.LastSeen),
			)

			return i.Index.Update(ctx, i.AnnotatedResource.ID, &index_types.Update{
				LastSeen: &now,
			})
		}

	case t.ManualSource, t.UserSource:
		// Do not update based on manual or user input.

	default:
		// Panic for unexpected Source values, instead of hard failing.
		panic(fmt.Sprintf("Unexpected source %s for item %+v", i.Source, i))
	}

	span.AddEvent(ctx, "Not updating")

	return nil
}

// deletePartial deletes partial items.
func (c *Crawler) deletePartial(ctx context.Context, i *existingItem) error {
	return c.indexes.Partials.Delete(ctx, i.ID)
}

// processPartial processes partials found in index; previously recognized as an unreferenced partial
func (c *Crawler) processPartial(ctx context.Context, i *existingItem) (bool, error) {
	if i.Reference.Parent == nil {
		log.Printf("Quick-skipping unreferenced partial %v", i)

		// Skip unreferenced partial
		return true, nil
	}

	// Referenced partial; delete as partial
	if err := c.deletePartial(ctx, i); err != nil {
		return true, err
	}

	// Index item as new
	return false, nil
}

func (c *Crawler) processExisting(ctx context.Context, i *existingItem) (bool, error) {
	switch i.Index {
	case c.indexes.Invalids:
		// Already indexed as invalid; we're done
		return true, nil
	case c.indexes.Partials:
		return c.processPartial(ctx, i)
	}

	// Update item and we're done.
	if err := c.updateExisting(ctx, i); err != nil {
		return true, err
	}

	return true, nil
}

// updateMaybeExisting updates an item when it exists and returnes true when item exists.
func (c *Crawler) updateMaybeExisting(ctx context.Context, r *t.AnnotatedResource) (bool, error) {
	ctx, span := c.Tracer.Start(ctx, "crawler.updateMaybeExisting")
	defer span.End()

	existing, err := c.getExistingItem(ctx, r)
	if err != nil {
		return false, err
	}

	// Process existing item
	if existing != nil {
		if span.IsRecording() {
			span.AddEvent(ctx, "existing", label.Any("index", existing.Index))
		}

		return c.processExisting(ctx, existing)
	}

	return false, nil
}
<commit_msg>Check for nil LastSeen.<commit_after>package crawler

import (
	"context"
	"fmt"
	"log"
	"time"

	"go.opentelemetry.io/otel/label"

	index_types "github.com/ipfs-search/ipfs-search/components/index/types"
	t "github.com/ipfs-search/ipfs-search/types"
)

func appendReference(refs index_types.References, r *t.Reference) (index_types.References, bool) {
	if r.Parent == nil {
		// No new reference, not updating
		return refs, false
	}

	for _, indexedRef := range refs {
		if indexedRef.ParentHash == r.Parent.ID && indexedRef.Name == r.Name {
			// Existing reference, not updating
			return refs, false
		}
	}

	return append(refs, index_types.Reference{
		ParentHash: r.Parent.ID,
		Name:       r.Name,
	}), true
}

// updateExisting updates known existing items.
func (c *Crawler) updateExisting(ctx context.Context, i *existingItem) error {
	ctx, span := c.Tracer.Start(ctx, "crawler.updateExisting")
	defer span.End()

	switch i.Source {
	case t.DirectorySource:
		// Item referenced from a directory, consider updating references (but not last-seen).
		refs, refsUpdated := appendReference(i.References, &i.AnnotatedResource.Reference)

		if refsUpdated {
			span.AddEvent(ctx, "Updating",
				label.String("reason", "reference-added"),
				label.Any("new-reference", i.AnnotatedResource.Reference),
			)

			return i.Index.Update(ctx, i.AnnotatedResource.ID, &index_types.Update{
				References: refs,
			})
		}

	case t.SnifferSource, t.UnknownSource:
		// TODO: Remove UnknownSource after sniffer is updated and queue is flushed.
		// Item sniffed, conditionally update last-seen.
		now := time.Now()

		// Strip milliseconds to cater to legacy ES index format.
		// This can be safely removed after the next reindex with _nomillis removed from time format.
		now = now.Truncate(time.Second)

		if i.LastSeen == nil {
			panic(fmt.Sprintf("nil LastSeen on %+v", i))
		}
		isRecent := now.Sub(*i.LastSeen) > c.config.MinUpdateAge

		if isRecent {
			span.AddEvent(ctx, "Updating",
				label.String("reason", "is-recent"),
				label.Stringer("last-seen", i.LastSeen),
			)

			return i.Index.Update(ctx, i.AnnotatedResource.ID, &index_types.Update{
				LastSeen: &now,
			})
		}

	case t.ManualSource, t.UserSource:
		// Do not update based on manual or user input.

	default:
		// Panic for unexpected Source values, instead of hard failing.
		panic(fmt.Sprintf("Unexpected source %s for item %+v", i.Source, i))
	}

	span.AddEvent(ctx, "Not updating")

	return nil
}

// deletePartial deletes partial items.
func (c *Crawler) deletePartial(ctx context.Context, i *existingItem) error {
	return c.indexes.Partials.Delete(ctx, i.ID)
}

// processPartial processes partials found in index; previously recognized as an unreferenced partial
func (c *Crawler) processPartial(ctx context.Context, i *existingItem) (bool, error) {
	if i.Reference.Parent == nil {
		log.Printf("Quick-skipping unreferenced partial %v", i)

		// Skip unreferenced partial
		return true, nil
	}

	// Referenced partial; delete as partial
	if err := c.deletePartial(ctx, i); err != nil {
		return true, err
	}

	// Index item as new
	return false, nil
}

func (c *Crawler) processExisting(ctx context.Context, i *existingItem) (bool, error) {
	switch i.Index {
	case c.indexes.Invalids:
		// Already indexed as invalid; we're done
		return true, nil
	case c.indexes.Partials:
		return c.processPartial(ctx, i)
	}

	// Update item and we're done.
	if err := c.updateExisting(ctx, i); err != nil {
		return true, err
	}

	return true, nil
}

// updateMaybeExisting updates an item when it exists and returnes true when item exists.
func (c *Crawler) updateMaybeExisting(ctx context.Context, r *t.AnnotatedResource) (bool, error) {
	ctx, span := c.Tracer.Start(ctx, "crawler.updateMaybeExisting")
	defer span.End()

	existing, err := c.getExistingItem(ctx, r)
	if err != nil {
		return false, err
	}

	// Process existing item
	if existing != nil {
		if span.IsRecording() {
			span.AddEvent(ctx, "existing", label.Any("index", existing.Index))
		}

		return c.processExisting(ctx, existing)
	}

	return false, nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"fmt"
	"github.com/gorilla/schema" //For populating struct with multiple url values
	"html/template"
	"io"
	"net/http"
)

// Hasher type contains the multiple url values, Query and Format.
// Currently the formats supported are only md5, sha1, sha256.
//
// IMPORTANT: Make sure that fields are exported, i.e. Capitalized first letters.
type Hasher struct {
	Query  string `schema:"q"`      //query
	Format string `schema:"format"` //format of hash (md5, sha1, sha256)
}

var hasher = new(Hasher) //Returns a pointer to a new Hasher type
var decoder = schema.NewDecoder()
var templates = template.Must(template.ParseFiles("index.html"))

// HashHandler parses through the url values and determines the query string and
// format type parameters. It checks hasher.Format and writes the corresponding hash
// to the http.ResponseWriter.
// Currently, It suports only MD5, SHA1, SHA256.
func HashHandler(w http.ResponseWriter, r *http.Request) {
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	err = decoder.Decode(hasher, r.Form)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	if hasher.Query == "" || hasher.Format == "" {
		http.Redirect(w, r, "/", http.StatusFound) //If the query or the format is empty, redirect to the home page.
		return
	}

	switch hasher.Format {
	case "md5":
		h := md5.New()
		io.WriteString(h, hasher.Query)
		fmt.Fprintf(w, "%x", h.Sum(nil))
		return
	case "sha1":
		h := sha1.New()
		io.WriteString(h, hasher.Query)
		fmt.Fprintf(w, "%x", h.Sum(nil))
		return
	case "sha256":
		h := sha256.New()
		io.WriteString(h, hasher.Query)
		fmt.Fprintf(w, "%x", h.Sum(nil))
		return
	default:
		http.Redirect(w, r, "/", http.StatusFound) //If the format is not supported, redirect to the home page.
		return
	}
}

// InputHandler returns a HTML form for query string input and selecting hash type
func InputHandler(w http.ResponseWriter, r *http.Request) {
	renderTemplate(w, "index")
}

// renderTemplate renders the template and handles errors.
// It takes http.Response Writer and the template filename as inputs.
func renderTemplate(w http.ResponseWriter, tmpl string) {
	err := templates.ExecuteTemplate(w, tmpl+".html", "")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func main() {
	http.HandleFunc("/", InputHandler)
	http.HandleFunc("/hash", HashHandler)
	http.ListenAndServe(":8080", nil)
}
<commit_msg>Fix empty queries<commit_after>package main

import (
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"fmt"
	"github.com/gorilla/schema" //For populating struct with multiple url values
	"html/template"
	"io"
	"net/http"
)

// Hasher type contains the multiple url values, Query and Format.
// Currently the formats supported are only md5, sha1, sha256.
//
// IMPORTANT: Make sure that fields are exported, i.e. Capitalized first letters.
type Hasher struct {
	Query  string `schema:"q"`      //query
	Format string `schema:"format"` //format of hash (md5, sha1, sha256)
}

var hasher = new(Hasher) //Returns a pointer to a new Hasher type
var decoder = schema.NewDecoder()
var templates = template.Must(template.ParseFiles("index.html"))

// HashHandler parses through the url values and determines the query string and
// format type parameters. It checks hasher.Format and writes the corresponding hash
// to the http.ResponseWriter.
// Currently, It suports only MD5, SHA1, SHA256.
func HashHandler(w http.ResponseWriter, r *http.Request) {
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	err = decoder.Decode(hasher, r.Form)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	if hasher.Query == "" || hasher.Format == "" {
		http.Redirect(w, r, "/", http.StatusFound) //If the query or the format is empty, redirect to the home page.
	}

	switch hasher.Format {
	case "md5":
		h := md5.New()
		io.WriteString(h, hasher.Query)
		fmt.Fprintf(w, "%x", h.Sum(nil))
		return
	case "sha1":
		h := sha1.New()
		io.WriteString(h, hasher.Query)
		fmt.Fprintf(w, "%x", h.Sum(nil))
		return
	case "sha256":
		h := sha256.New()
		io.WriteString(h, hasher.Query)
		fmt.Fprintf(w, "%x", h.Sum(nil))
		return
	default:
		http.Redirect(w, r, "/", http.StatusFound) //If the format is not supported, redirect to the home page.
		return
	}
}

// InputHandler returns a HTML form for query string input and selecting hash type
func InputHandler(w http.ResponseWriter, r *http.Request) {
	renderTemplate(w, "index")
}

// renderTemplate renders the template and handles errors.
// It takes http.Response Writer and the template filename as inputs.
func renderTemplate(w http.ResponseWriter, tmpl string) {
	err := templates.ExecuteTemplate(w, tmpl+".html", "")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func main() {
	http.HandleFunc("/", InputHandler)
	http.HandleFunc("/hash", HashHandler)
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

type Repository struct {
	Name        string `json:"name"`
	URL         string `json:"url"`
	Description string `json:"description"`
	Homepage    string `json:"homepage"`
}

type Commit struct {
	ID        string `json:"id"`
	Message   string `json:"message"`
	Timestamp string `json:"timestamp"`
	URL       string `json:"url"`
	Author    Author `json:"author"`
}

type Author struct {
	Name  string `json:"name"`
	Email string `json:"email"`
}

type PushEvent struct {
	Before            string     `json:"before"`
	After             string     `json:"after"`
	Ref               string     `json:"ref"`
	UserID            int        `json:"user_id"`
	UserName          string     `json:"user_name"`
	ProjectID         int        `json:"project_id"`
	Repository        Repository `json:"repository"`
	Commits           []Commit   `json:"commits"`
	TotalCommitsCount int        `json:"total_commits_count"`
}

func main() {
	mux := http.NewServeMux()

	mux.HandleFunc("/push", pushEventHandler)
	mux.HandleFunc("/tag", tagEventHandler)

	http.ListenAndServe(":12138", mux)
}

func pushEventHandler(w http.ResponseWriter, req *http.Request) {
	data, err := ioutil.ReadAll(req.Body)
	if err != nil {
		fmt.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	event := &PushEvent{}
	err = json.Unmarshal(data, event)
	if err != nil {
		fmt.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	fmt.Printf("%v\n", event)
}

func tagEventHandler(w http.ResponseWriter, req *http.Request) {
	data, err := ioutil.ReadAll(req.Body)
	if err != nil {
		fmt.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	fmt.Printf("%v\n", string(data))
}
<commit_msg>unmarshal pushtag event<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

type Repository struct {
	Name        string `json:"name"`
	URL         string `json:"url"`
	Description string `json:"description"`
	Homepage    string `json:"homepage"`
}

type Commit struct {
	ID        string `json:"id"`
	Message   string `json:"message"`
	Timestamp string `json:"timestamp"`
	URL       string `json:"url"`
	Author    Author `json:"author"`
}

type Author struct {
	Name  string `json:"name"`
	Email string `json:"email"`
}

type PushEvent struct {
	Before            string     `json:"before"`
	After             string     `json:"after"`
	Ref               string     `json:"ref"`
	UserID            int        `json:"user_id"`
	UserName          string     `json:"user_name"`
	ProjectID         int        `json:"project_id"`
	Repository        Repository `json:"repository"`
	Commits           []Commit   `json:"commits"`
	TotalCommitsCount int        `json:"total_commits_count"`
}

func main() {
	mux := http.NewServeMux()

	mux.HandleFunc("/push", pushEventHandler)
	mux.HandleFunc("/push/tag", pushTagEventHandler)

	http.ListenAndServe(":12138", mux)
}

func pushEventHandler(w http.ResponseWriter, req *http.Request) {
	data, err := ioutil.ReadAll(req.Body)
	if err != nil {
		fmt.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	event := &PushEvent{}
	err = json.Unmarshal(data, event)
	if err != nil {
		fmt.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	fmt.Printf("%v\n", event)
}

func pushTagEventHandler(w http.ResponseWriter, req *http.Request) {
	data, err := ioutil.ReadAll(req.Body)
	if err != nil {
		fmt.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	event := &PushEvent{}
	err = json.Unmarshal(data, event)
	if err != nil {
		fmt.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	fmt.Printf("%v\n", event)
}
<|endoftext|>


<commit_before>// The consolidator program subscribes to a topic on Google Cloud Pubsub, and reads bundles of
// composite ADSB messages from it. These are deduped, and unique ones are published to a
// different topic. A snapshot is written to memcache, for other apps to access.
// The ideal deployment model for consolidator is as a user-managed AppEngine VM.
package main

// go run consolidator.go

// https://godoc.org/google.golang.org/cloud/pubsub#Publish

import (
	"flag"
	"log"
	"os"
	"time"

	"github.com/skypies/pi/airspace"
	"github.com/skypies/pi/pubsub"
)

var (
	Log                   *log.Logger

	fJsonAuthFile          string
	fProjectName           string
	fPubsubSubscription    string
	fPubsubOutputTopic     string
)

func init() {
	flag.StringVar(&fJsonAuthFile, "auth", "serfr0-fdb-48c34ecd36c9.json",
		"The JSON auth file for a Google service worker account")
	flag.StringVar(&fProjectName, "project", "serfr0-fdb",
		"Name of the Google cloud project hosting the pubsub")
	flag.StringVar(&fPubsubSubscription, "sub", "consolidator",
		"Name of the pubsub subscription on the adsb-inbound topic")
	flag.StringVar(&fPubsubOutputTopic, "output", "adsb-consolidated",
		"Name of the pubsub topic to post deduped output on (short name, not projects/blah...)")

	flag.Parse()
	Log = log.New(os.Stdout,"", log.Ldate|log.Ltime)//|log.Lshortfile)
}

func main() {
	airspace := airspace.Airspace{}
	c := pubsub.GetContext(fJsonAuthFile, fProjectName)

	// While developing - purge things
	pubsub.PurgeSub(c, fPubsubSubscription, "adsb-inbound")

	for {
		msgs,err := pubsub.Pull(c, fPubsubSubscription, 10)
		if err != nil {
			Log.Printf("Pull/sub=%s: err: %s", fPubsubSubscription, err)
			continue
		} else if msgs == nil || len(*msgs) == 0 {
			//Log.Printf("-- nothing found, sleeping")
			time.Sleep(time.Second * 1)
			continue
		}
		Log.Printf("Pull: found %d ADSB", len(*msgs))

		newMsgs := airspace.MaybeUpdate(msgs)
		Log.Printf("Pull: %d were new (%d already seen)", len(newMsgs), len(*msgs)-len(newMsgs))

		for i,msg := range newMsgs {
			Log.Printf(" new: [%2d] %s\n", i, msg)
		}

		// Update the memcache thing

		if fPubsubOutputTopic != "" {
			// Publish newMsgs
		}
	}

	Log.Print("Final clean shutdown")
}
<commit_msg>Pre-merge<commit_after>// The consolidator program subscribes to a topic on Google Cloud Pubsub, and reads bundles of
// composite ADSB messages from it. These are deduped, and unique ones are published to a
// different topic. A snapshot is written to memcache, for other apps to access.
// The ideal deployment model for consolidator is as a user-managed AppEngine VM.
package main

// go run consolidator.go

// https://godoc.org/google.golang.org/cloud/pubsub#Publish

import (
	"flag"
	"log"
	"os"
	"time"

	"github.com/skypies/pi/airspace"
	"github.com/skypies/pi/pubsub"
)

var (
	Log                   *log.Logger

	fJsonAuthFile          string
	fProjectName           string
	fPubsubSubscription    string
	fPubsubOutputTopic     string
)

func init() {
	flag.StringVar(&fJsonAuthFile, "auth", "serfr0-fdb-48c34ecd36c9.json",
		"The JSON auth file for a Google service worker account")
	flag.StringVar(&fProjectName, "project", "serfr0-fdb",
		"Name of the Google cloud project hosting the pubsub")
	flag.StringVar(&fPubsubSubscription, "sub", "consolidator",
		"Name of the pubsub subscription on the adsb-inbound topic")
	flag.StringVar(&fPubsubOutputTopic, "output", "adsb-consolidated",
		"Name of the pubsub topic to post deduped output on (short name, not projects/blah...)")

	flag.Parse()
	Log = log.New(os.Stdout,"", log.Ldate|log.Ltime)//|log.Lshortfile)
}

func main() {
	airspace := airspace.Airspace{}
	c := pubsub.GetContext(fJsonAuthFile, fProjectName)

	// While developing - purge things
	pubsub.PurgeSub(c, fPubsubSubscription, "adsb-inbound")

	for {
		msgs,err := pubsub.Pull(c, fPubsubSubscription, 10)
		if err != nil {
			Log.Printf("Pull/sub=%s: err: %s", fPubsubSubscription, err)
			continue
		} else if msgs == nil || len(*msgs) == 0 {
			//Log.Printf("-- nothing found, sleeping")
			time.Sleep(time.Second * 1)
			continue
		}
		//Log.Printf("Pull: found %d ADSB", len(*msgs))

		newMsgs := airspace.MaybeUpdate(msgs)
		//Log.Printf(" %d were new (%d already seen)", len(newMsgs), len(*msgs)-len(newMsgs))

		for i,msg := range newMsgs {
			Log.Printf(" new: [%2d] %s\n", i, msg)
		}

		// Update the memcache thing

		if fPubsubOutputTopic != "" {
			// Publish newMsgs
		}
	}

	Log.Print("Final clean shutdown")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/inominate/apicache"
)

var apiClient apicache.Client

// worker tracking
var activeWorkerCount, workerCount int32
var workCount []int32

type apiReq struct {
	apiReq  *apicache.Request
	apiResp *apicache.Response

	apiErr apicache.APIError

	expires time.Time

	worker   int
	httpCode int
	err      error
	respChan chan apiReq
}

// Channel for sending jobs to workers
var workChan chan apiReq

func APIReq(url string, params map[string]string) (*apicache.Response, error) {
	var errorStr string

	if atomic.LoadInt32(&workerCount) <= 0 {
		panic("No workers!")
	}

	// Build the request
	apireq := apicache.NewRequest(url)
	for k, v := range params {
		switch k {
		case "force":
			if v != "" {
				apireq.Force = true
			}
		default:
			apireq.Set(k, v)
		}
	}

	workerID := "C"
	// Don't send it to a worker if we can just yank it fromm the cache
	apiResp, err := apireq.GetCached()
	if err != nil || apireq.Force {
		for i := 0; i <= conf.Retries; i++ {
			respChan := make(chan apiReq)
			req := apiReq{apiReq: apireq, respChan: respChan}
			workChan <- req

			resp := <-respChan
			close(respChan)

			apiResp = resp.apiResp
			err = resp.err
			workerID = fmt.Sprintf("%d", resp.worker)

			// Attempt to recover from 221s
			if apiResp.Error.ErrorCode != 221 {
				break
			}
			time.Sleep(3 * time.Second)
			log.Printf("Got 221 from API, retrying...")
		}
	}
	if apiResp.Error.ErrorCode == 221 {
		log.Printf("Failed to recover from error 221.")
	}

	// This is similar to the request log, but knows more about where it came from.
	if debug {
		if apiResp.Error.ErrorCode != 0 {
			errorStr = fmt.Sprintf(" Error %d: %s", apiResp.Error.ErrorCode, apiResp.Error.ErrorText)
		}
		logParams := ""
		var paramVal string
		for k, _ := range params {
			if conf.Logging.CensorLog && strings.ToLower(k) == "vcode" && len(params[k]) > 8 {
				paramVal = params[k][0:8] + "..."
			} else {
				paramVal = params[k]
			}
			logParams = fmt.Sprintf("%s&%s=%s", logParams, k, paramVal)
		}
		if logParams != "" {
			logParams = "?" + logParams[1:]
		}

		debugLog.Printf("w%s: %s%s HTTP: %d Expires: %s%s", workerID, url, logParams, apiResp.HTTPCode, apiResp.Expires.Format("2006-01-02 15:04:05"), errorStr)
	}
	return apiResp, err
}

func worker(reqChan chan apiReq, workerID int) {
	atomic.AddInt32(&workerCount, 1)
	var err error

	var eErr error
	var rErr error

	var errStr string
	for req := range reqChan {
		atomic.AddInt32(&activeWorkerCount, 1)

		// Run both of the error limiters simultaneously rather than in
		// sequence. Still need both before we continue.
		errorLimiter := make(chan error)
		rpsLimiter := make(chan error)
		go func() {
			err := errorRateLimiter.Start(60 * time.Second)
			errorLimiter <- err
		}()
		go func() {
			err := rateLimiter.Start(60 * time.Second)
			rpsLimiter <- err
		}()
		eErr = <-errorLimiter
		rErr = <-rpsLimiter

		// Check the error limiter for timeouts
		if eErr != nil {
			err = eErr
			errStr = "error throttling"

			// If the rate limiter didn't timeout be sure to signal it that we
			// didn't do anything.
			if rErr == nil {
				rateLimiter.Finish(true)
			}
		}
		if rErr != nil {
			err = rErr
			errStr = "rate limiting"

			// If the error limiter didn't also timeout be sure to signal it that we
			// didn't do anything.
			if eErr == nil {
				errorRateLimiter.Finish(true)
			}
		}
		// We're left with a single err and errStr for returning an error to the client.
		if err != nil {
			req.apiResp = &apicache.Response{
				Data: apicache.SynthesizeAPIError(500,
					fmt.Sprintf("APIProxy Error: Proxy timeout due to %s.", errStr),
					5*time.Minute),
				Expires: time.Now().Add(5 * time.Minute),
				Error: apicache.APIError{500,
					fmt.Sprintf("APIProxy Error: Proxy timeout due to %s.", errStr)},
				HTTPCode: 504,
			}
			req.err = err
		} else {
			resp, err := req.apiReq.Do()
			req.apiResp = resp
			req.err = err
			if resp.Error.ErrorCode == 0 {
				// Finish, but skip recording the event in the rate limiter
				// when there is no error.
				errorRateLimiter.Finish(true)
			} else {
				errorRateLimiter.Finish(false)
			}
			rateLimiter.Finish(false)
		}

		req.worker = workerID
		req.respChan <- req
		atomic.AddInt32(&workCount[workerID], 1)
		atomic.AddInt32(&activeWorkerCount, -1)
	}
	atomic.AddInt32(&workerCount, -1)
}

var startWorkersOnce = &sync.Once{}

func startWorkers() {
	startWorkersOnce.Do(realStartWorkers)
}

func realStartWorkers() {
	log.Printf("Starting %d Workers...", conf.Workers)
	workChan = make(chan apiReq)
	workCount = make([]int32, conf.Workers+1)

	for i := 1; i <= conf.Workers; i++ {
		debugLog.Printf("Starting worker #%d.", i)
		go worker(workChan, i)
	}
}

func stopWorkers() {
	close(workChan)
	for atomic.LoadInt32(&workerCount) > 0 {
		time.Sleep(10 * time.Millisecond)
	}
	startWorkersOnce = &sync.Once{}
}

func PrintWorkerStats(w io.Writer) {
	active, loaded := GetWorkerStats()
	fmt.Fprintf(w, "%d workers idle, %d workers active.\n", loaded-active, active)

	rateCount := rateLimiter.Count()
	errorCount := errorRateLimiter.Count()
	fmt.Fprintf(w, "%d requests in the last second.\n", rateCount)
	fmt.Fprintf(w, "%d errors over last %d seconds.\n", errorCount, conf.ErrorPeriod)

	for i := int32(1); i <= atomic.LoadInt32(&workerCount); i++ {
		count := atomic.LoadInt32(&workCount[i])
		fmt.Fprintf(w, "   %d: %d\n", i, count)
	}
}

func GetWorkerStats() (int32, int32) {
	return atomic.LoadInt32(&activeWorkerCount), atomic.LoadInt32(&workerCount)
}
<commit_msg>can't recover from errors if we pull them back from the cache!<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/inominate/apicache"
)

var apiClient apicache.Client

// worker tracking
var activeWorkerCount, workerCount int32
var workCount []int32

type apiReq struct {
	apiReq  *apicache.Request
	apiResp *apicache.Response

	apiErr apicache.APIError

	expires time.Time

	worker   int
	httpCode int
	err      error
	respChan chan apiReq
}

// Channel for sending jobs to workers
var workChan chan apiReq

func APIReq(url string, params map[string]string) (*apicache.Response, error) {
	var errorStr string

	if atomic.LoadInt32(&workerCount) <= 0 {
		panic("No workers!")
	}

	// Build the request
	apireq := apicache.NewRequest(url)
	for k, v := range params {
		switch k {
		case "force":
			if v != "" {
				apireq.Force = true
			}
		default:
			apireq.Set(k, v)
		}
	}

	workerID := "C"
	// Don't send it to a worker if we can just yank it fromm the cache
	apiResp, err := apireq.GetCached()
	if err != nil || apireq.Force {
		for i := 0; i <= conf.Retries; i++ {
			respChan := make(chan apiReq)
			req := apiReq{apiReq: apireq, respChan: respChan}
			workChan <- req

			resp := <-respChan
			close(respChan)

			apiResp = resp.apiResp
			err = resp.err
			workerID = fmt.Sprintf("%d", resp.worker)

			// Attempt to recover from 221s
			if apiResp.Error.ErrorCode != 221 {
				break
			}
			time.Sleep(3 * time.Second)
			log.Printf("Got 221 from API, retrying...")
			apireq.Force = true
		}
	}
	if apiResp.Error.ErrorCode == 221 {
		log.Printf("Failed to recover from error 221.")
	}

	// This is similar to the request log, but knows more about where it came from.
	if debug {
		if apiResp.Error.ErrorCode != 0 {
			errorStr = fmt.Sprintf(" Error %d: %s", apiResp.Error.ErrorCode, apiResp.Error.ErrorText)
		}
		logParams := ""
		var paramVal string
		for k, _ := range params {
			if conf.Logging.CensorLog && strings.ToLower(k) == "vcode" && len(params[k]) > 8 {
				paramVal = params[k][0:8] + "..."
			} else {
				paramVal = params[k]
			}
			logParams = fmt.Sprintf("%s&%s=%s", logParams, k, paramVal)
		}
		if logParams != "" {
			logParams = "?" + logParams[1:]
		}

		debugLog.Printf("w%s: %s%s HTTP: %d Expires: %s%s", workerID, url, logParams, apiResp.HTTPCode, apiResp.Expires.Format("2006-01-02 15:04:05"), errorStr)
	}
	return apiResp, err
}

func worker(reqChan chan apiReq, workerID int) {
	atomic.AddInt32(&workerCount, 1)
	var err error

	var eErr error
	var rErr error

	var errStr string
	for req := range reqChan {
		atomic.AddInt32(&activeWorkerCount, 1)

		// Run both of the error limiters simultaneously rather than in
		// sequence. Still need both before we continue.
		errorLimiter := make(chan error)
		rpsLimiter := make(chan error)
		go func() {
			err := errorRateLimiter.Start(60 * time.Second)
			errorLimiter <- err
		}()
		go func() {
			err := rateLimiter.Start(60 * time.Second)
			rpsLimiter <- err
		}()
		eErr = <-errorLimiter
		rErr = <-rpsLimiter

		// Check the error limiter for timeouts
		if eErr != nil {
			err = eErr
			errStr = "error throttling"

			// If the rate limiter didn't timeout be sure to signal it that we
			// didn't do anything.
			if rErr == nil {
				rateLimiter.Finish(true)
			}
		}
		if rErr != nil {
			err = rErr
			errStr = "rate limiting"

			// If the error limiter didn't also timeout be sure to signal it that we
			// didn't do anything.
			if eErr == nil {
				errorRateLimiter.Finish(true)
			}
		}
		// We're left with a single err and errStr for returning an error to the client.
		if err != nil {
			req.apiResp = &apicache.Response{
				Data: apicache.SynthesizeAPIError(500,
					fmt.Sprintf("APIProxy Error: Proxy timeout due to %s.", errStr),
					5*time.Minute),
				Expires: time.Now().Add(5 * time.Minute),
				Error: apicache.APIError{500,
					fmt.Sprintf("APIProxy Error: Proxy timeout due to %s.", errStr)},
				HTTPCode: 504,
			}
			req.err = err
		} else {
			resp, err := req.apiReq.Do()
			req.apiResp = resp
			req.err = err
			if resp.Error.ErrorCode == 0 {
				// Finish, but skip recording the event in the rate limiter
				// when there is no error.
				errorRateLimiter.Finish(true)
			} else {
				errorRateLimiter.Finish(false)
			}
			rateLimiter.Finish(false)
		}

		req.worker = workerID
		req.respChan <- req
		atomic.AddInt32(&workCount[workerID], 1)
		atomic.AddInt32(&activeWorkerCount, -1)
	}
	atomic.AddInt32(&workerCount, -1)
}

var startWorkersOnce = &sync.Once{}

func startWorkers() {
	startWorkersOnce.Do(realStartWorkers)
}

func realStartWorkers() {
	log.Printf("Starting %d Workers...", conf.Workers)
	workChan = make(chan apiReq)
	workCount = make([]int32, conf.Workers+1)

	for i := 1; i <= conf.Workers; i++ {
		debugLog.Printf("Starting worker #%d.", i)
		go worker(workChan, i)
	}
}

func stopWorkers() {
	close(workChan)
	for atomic.LoadInt32(&workerCount) > 0 {
		time.Sleep(10 * time.Millisecond)
	}
	startWorkersOnce = &sync.Once{}
}

func PrintWorkerStats(w io.Writer) {
	active, loaded := GetWorkerStats()
	fmt.Fprintf(w, "%d workers idle, %d workers active.\n", loaded-active, active)

	rateCount := rateLimiter.Count()
	errorCount := errorRateLimiter.Count()
	fmt.Fprintf(w, "%d requests in the last second.\n", rateCount)
	fmt.Fprintf(w, "%d errors over last %d seconds.\n", errorCount, conf.ErrorPeriod)

	for i := int32(1); i <= atomic.LoadInt32(&workerCount); i++ {
		count := atomic.LoadInt32(&workCount[i])
		fmt.Fprintf(w, "   %d: %d\n", i, count)
	}
}

func GetWorkerStats() (int32, int32) {
	return atomic.LoadInt32(&activeWorkerCount), atomic.LoadInt32(&workerCount)
}
<|endoftext|>