<commit_before>package goleg

/*
#include "oleg.h"
*/
import "C"
import (
	"errors"
	"sync"
	"time"
)

type Database struct {
	db          *C.ol_database
	RecordCount *C.int
	mutex       *sync.Mutex
}

func Open(path, name string, features int) (Database, error) {
	var database Database
	database.db = COpen(path, name, features)
	if database.db == nil {
		return Database{}, errors.New("Can't open database (NULL returned)")
	}
	database.RecordCount = &database.db.rcrd_cnt
	database.mutex = &sync.Mutex{}
	return database, nil
}

func (d Database) Close() int {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	return CClose(d.db)
}

func (d Database) Unjar(key string) []byte {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	var dsize uintptr
	return CUnjar(d.db, key, uintptr(len(key)), &dsize)
}

func (d Database) Jar(key string, value []byte) int {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	return CJar(d.db, key, uintptr(len(key)), value, uintptr(len(value)))
}

func (d Database) Scoop(key string) int {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	return CScoop(d.db, key, uintptr(len(key)))
}

func (d Database) Uptime() int {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	return CUptime(d.db)
}

func (d Database) Expiration(key string) (time.Time, bool) {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	return CExpirationTime(d.db, key, uintptr(len(key)))
}

func (d Database) Spoil(key string, expiration time.Time) int {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	return CSpoil(d.db, key, uintptr(len(key)), expiration)
}

func (d Database) Exists(key string) bool {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	return CExists(d.db, key, uintptr(len(key))) == 0
}

func (d Database) Squish() bool {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	return CSquish(d.db) == 1
}

func (d Database) PrefixMatch(prefix string) (bool, []string) {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	len, out := CPrefixMatch(d.db, prefix, uintptr(len(prefix)))
	return len >= 0, out
}

func (d Database) DumpKeys() (bool, []string) {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	len, out := CDumpKeys(d.db)
	return len >= 0, out
}

func (d Database) First() (bool, string, []byte) {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	node := CNodeFirst(d.db)
	if node == nil {
		return false, "", nil
	}
	return CNodeGet(d.db, node)
}

func (d Database) Last() (bool, string, []byte) {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	node := CNodeLast(d.db)
	if node == nil {
		return false, "", nil
	}
	return CNodeGet(d.db, node)
}

func (d Database) Next(key string) (bool, string, []byte) {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	node := CNodeNext(d.db, key, uintptr(len(key)))
	if node == nil {
		return false, "", nil
	}
	return CNodeGet(d.db, node)
}

func (d Database) Prev(key string) (bool, string, []byte) {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	node := CNodePrev(d.db, key, uintptr(len(key)))
	if node == nil {
		return false, "", nil
	}
	return CNodeGet(d.db, node)
}
<commit_msg>Avoid using defer statements on time-sensitive function calls.<commit_after>package goleg

/*
#include "oleg.h"
*/
import "C"
import (
	"errors"
	"sync"
	"time"
)

type Database struct {
	db          *C.ol_database
	RecordCount *C.int
	mutex       *sync.Mutex
}

func Open(path, name string, features int) (Database, error) {
	var database Database
	database.db = COpen(path, name, features)
	if database.db == nil {
		return Database{}, errors.New("Can't open database (NULL returned)")
	}
	database.RecordCount = &database.db.rcrd_cnt
	database.mutex = &sync.Mutex{}
	return database, nil
}

func (d Database) Close() int {
	d.mutex.Lock()
	defer d.mutex.Unlock()
	return CClose(d.db)
}

func (d Database) Unjar(key string) []byte {
	d.mutex.Lock()
	var dsize uintptr
	var ret = CUnjar(d.db, key, uintptr(len(key)), &dsize)
	d.mutex.Unlock()
	return ret
}

func (d Database) Jar(key string, value []byte) int {
	d.mutex.Lock()
	var ret = CJar(d.db, key, uintptr(len(key)), value, uintptr(len(value)))
	d.mutex.Unlock()
	return ret
}

func (d Database) Scoop(key string) int {
	d.mutex.Lock()
	var ret = CScoop(d.db, key, uintptr(len(key)))
	d.mutex.Unlock()
	return ret
}

func (d Database) Uptime() int {
	d.mutex.Lock()
	var ret = CUptime(d.db)
	d.mutex.Unlock()
	return ret
}

func (d Database) Expiration(key string) (time.Time, bool) {
	d.mutex.Lock()
	var time, exists = CExpirationTime(d.db, key, uintptr(len(key)))
	d.mutex.Unlock()
	return time, exists
}

func (d Database) Spoil(key string, expiration time.Time) int {
	d.mutex.Lock()
	var ret = CSpoil(d.db, key, uintptr(len(key)), expiration)
	d.mutex.Unlock()
	return ret
}

func (d Database) Exists(key string) bool {
	d.mutex.Lock()
	var ret = CExists(d.db, key, uintptr(len(key))) == 0
	d.mutex.Unlock()
	return ret
}

func (d Database) Squish() bool {
	d.mutex.Lock()
	var ret = CSquish(d.db) == 1
	d.mutex.Unlock()
	return ret
}

func (d Database) PrefixMatch(prefix string) (bool, []string) {
	d.mutex.Lock()
	len, out := CPrefixMatch(d.db, prefix, uintptr(len(prefix)))
	d.mutex.Unlock()
	return len >= 0, out
}

func (d Database) DumpKeys() (bool, []string) {
	d.mutex.Lock()
	len, out := CDumpKeys(d.db)
	d.mutex.Unlock()
	return len >= 0, out
}

func (d Database) First() (bool, string, []byte) {
	d.mutex.Lock()
	node := CNodeFirst(d.db)
	if node == nil {
		d.mutex.Unlock()
		return false, "", nil
	}
	var exists, key, data = CNodeGet(d.db, node)
	d.mutex.Unlock()
	return exists, key, data
}

func (d Database) Last() (bool, string, []byte) {
	d.mutex.Lock()
	node := CNodeLast(d.db)
	if node == nil {
		d.mutex.Unlock()
		return false, "", nil
	}
	var exists, key, data = CNodeGet(d.db, node)
	d.mutex.Unlock()
	return exists, key, data
}

func (d Database) Next(key string) (bool, string, []byte) {
	d.mutex.Lock()
	node := CNodeNext(d.db, key, uintptr(len(key)))
	if node == nil {
		d.mutex.Unlock()
		return false, "", nil
	}
	var exists, next_key, data = CNodeGet(d.db, node)
	d.mutex.Unlock()
	return exists, next_key, data
}

func (d Database) Prev(key string) (bool, string, []byte) {
	d.mutex.Lock()
	node := CNodePrev(d.db, key, uintptr(len(key)))
	if node == nil {
		d.mutex.Unlock()
		return false, "", nil
	}
	var exists, prev_key, data = CNodeGet(d.db, node)
	d.mutex.Unlock()
	return exists, prev_key, data
}
<|endoftext|>


<commit_before><commit_msg>expose diffhunk<commit_after><|endoftext|>


<commit_before>package slogger

import (
	"errors"
	"fmt"
	"runtime"
	"strings"
	"time"
)

type Log struct {
	Prefix     string
	Level      Level
	Filename   string
	Line       int
	Timestamp  time.Time
	MessageFmt string
	Args       []interface{}
}

func (self *Log) Message() string {
	return fmt.Sprintf(self.MessageFmt, self.Args...)
}

type Logger struct {
	Prefix    string
	Appenders []Appender
	StripDirs int
}

// Log a message and a level to a logger instance. This returns a
// pointer to a Log and a slice of errors that were gathered from every
// Appender (nil errors included).
func (self *Logger) Logf(level Level, messageFmt string, args ...interface{}) (*Log, []error) {
	return self.logf(level, messageFmt, args...)
}

// Log and return a formatted error string.
// Example:
//
// if whatIsExpected != whatIsReturned {
//     return slogger.Errorf(slogger.WARN, "Unexpected return value. Expected: %v Received: %v",
//         whatIsExpected, whatIsReturned)
// }
//
func (self *Logger) Errorf(level Level, messageFmt string, args ...interface{}) error {
	log, _ := self.logf(level, messageFmt, args...)
	return errors.New(log.Message())
}

// Stackf is designed to work in tandem with `NewStackError`. This
// function is similar to `Logf`, but takes a `stackErr`
// parameter. `stackErr` is expected to be of type StackError, but does
// not have to be.
func (self *Logger) Stackf(level Level, stackErr error, messageFmt string, args ...interface{}) (*Log, []error) {
	messageFmt = fmt.Sprintf("%v\n%v", messageFmt, stackErr.Error())
	return self.logf(level, messageFmt, args...)
}

var ignoredFileNames = []string{"logger.go"}

// Add a file to the list of file names that slogger will skip when it identifies the source
// of a message.  This is useful if you have a logging library built on top of slogger.
// If you IgnoreThisFilenameToo(...) on the files of that library, logging messages
// will be marked as coming from your code that calls your library, rather than from your library.
func IgnoreThisFilenameToo(fn string) {
	ignoredFileNames = append(ignoredFileNames, fn)
}

func containsAnyIgnoredFilename(s string) bool {
	for _, ign := range ignoredFileNames {
		if strings.Contains(s, ign)  {
			return true
		}
	}
	return false
}

func nonSloggerCaller() (pc uintptr, file string, line int, ok bool) {
	for skip := 0; skip < 100; skip++ {
		pc,file,line,ok := runtime.Caller(skip)
		if !ok || !containsAnyIgnoredFilename(file)  {
			return pc,file,line,ok
		}
	}
	return 0, "", 0, false
}

func (self *Logger) logf(level Level, messageFmt string, args ...interface{}) (*Log, []error) {
	var errors []error

	_, file, line, ok := nonSloggerCaller()
//	_, file, line, ok := runtime.Caller(2+offset)
	if ok == false {
		return nil, []error{fmt.Errorf("Failed to find the calling method.")}
	}

	file = stripDirectories(file, self.StripDirs)

	log := &Log{
		Prefix:     self.Prefix,
		Level:      level,
		Filename:   file,
		Line:       line,
		Timestamp:  time.Now(),
		MessageFmt: messageFmt,
		Args:       args,
	}

	// cache disabled for now as not yet used -Tim
	//	Cache.Add(log)

	for _, appender := range self.Appenders {
		if err := appender.Append(log); err != nil {
			error := fmt.Errorf("Error appending. Appender: %T Error: %v", appender, err)
			errors = append(errors, error)
		}
	}

	return log, errors
}

type Level uint8

// The level is in an order such that the expressions
// `level < WARN`, `level >= INFO` have intuitive meaning.
const (
	OFF Level = iota
	DEBUG
	ROUTINE
	INFO
	WARN
	ERROR
	DOOM
)

func (self Level) Type() string {
	switch self {
	case ERROR:
		return "error"
	case WARN:
		return "warn"
	case INFO:
		return "info"
	case ROUTINE:
		return "routine"
	case DEBUG:
		return "debug"
	case DOOM:
		return "doom"
	}

	return "off?"
}

func stacktrace() []string {
	ret := make([]string, 0, 2)
	for skip := 2; true; skip++ {
		_, file, line, ok := runtime.Caller(skip)
		if ok == false {
			break
		}

		ret = append(ret, fmt.Sprintf("at %s:%d", stripDirectories(file, 2), line))
	}

	return ret
}

type StackError struct {
	Message    string
	Stacktrace []string
}

func NewStackError(messageFmt string, args ...interface{}) *StackError {
	return &StackError{
		Message:    fmt.Sprintf(messageFmt, args...),
		Stacktrace: stacktrace(),
	}
}

func (self *StackError) Error() string {
	return fmt.Sprintf("%s\n\t%s", self.Message, strings.Join(self.Stacktrace, "\n\t"))
}

func stripDirectories(filepath string, toKeep int) string {
	var idxCutoff int
	if idxCutoff = strings.LastIndex(filepath, "/"); idxCutoff == -1 {
		return filepath
	}

	for dirToKeep := 0; dirToKeep < toKeep; dirToKeep++ {
		switch idx := strings.LastIndex(filepath[:idxCutoff], "/"); idx {
		case -1:
			break
		default:
			idxCutoff = idx
		}
	}

	return filepath[idxCutoff+1:]
}
<commit_msg>Add newLevel(string)Level function.<commit_after>package slogger

import (
	"errors"
	"fmt"
	"runtime"
	"strings"
	"time"
)

type Log struct {
	Prefix     string
	Level      Level
	Filename   string
	Line       int
	Timestamp  time.Time
	MessageFmt string
	Args       []interface{}
}

func (self *Log) Message() string {
	return fmt.Sprintf(self.MessageFmt, self.Args...)
}

type Logger struct {
	Prefix    string
	Appenders []Appender
	StripDirs int
}

// Log a message and a level to a logger instance. This returns a
// pointer to a Log and a slice of errors that were gathered from every
// Appender (nil errors included).
func (self *Logger) Logf(level Level, messageFmt string, args ...interface{}) (*Log, []error) {
	return self.logf(level, messageFmt, args...)
}

// Log and return a formatted error string.
// Example:
//
// if whatIsExpected != whatIsReturned {
//     return slogger.Errorf(slogger.WARN, "Unexpected return value. Expected: %v Received: %v",
//         whatIsExpected, whatIsReturned)
// }
//
func (self *Logger) Errorf(level Level, messageFmt string, args ...interface{}) error {
	log, _ := self.logf(level, messageFmt, args...)
	return errors.New(log.Message())
}

// Stackf is designed to work in tandem with `NewStackError`. This
// function is similar to `Logf`, but takes a `stackErr`
// parameter. `stackErr` is expected to be of type StackError, but does
// not have to be.
func (self *Logger) Stackf(level Level, stackErr error, messageFmt string, args ...interface{}) (*Log, []error) {
	messageFmt = fmt.Sprintf("%v\n%v", messageFmt, stackErr.Error())
	return self.logf(level, messageFmt, args...)
}

var ignoredFileNames = []string{"logger.go"}

// Add a file to the list of file names that slogger will skip when it identifies the source
// of a message.  This is useful if you have a logging library built on top of slogger.
// If you IgnoreThisFilenameToo(...) on the files of that library, logging messages
// will be marked as coming from your code that calls your library, rather than from your library.
func IgnoreThisFilenameToo(fn string) {
	ignoredFileNames = append(ignoredFileNames, fn)
}

func containsAnyIgnoredFilename(s string) bool {
	for _, ign := range ignoredFileNames {
		if strings.Contains(s, ign)  {
			return true
		}
	}
	return false
}

func nonSloggerCaller() (pc uintptr, file string, line int, ok bool) {
	for skip := 0; skip < 100; skip++ {
		pc,file,line,ok := runtime.Caller(skip)
		if !ok || !containsAnyIgnoredFilename(file)  {
			return pc,file,line,ok
		}
	}
	return 0, "", 0, false
}

func (self *Logger) logf(level Level, messageFmt string, args ...interface{}) (*Log, []error) {
	var errors []error

	_, file, line, ok := nonSloggerCaller()
//	_, file, line, ok := runtime.Caller(2+offset)
	if ok == false {
		return nil, []error{fmt.Errorf("Failed to find the calling method.")}
	}

	file = stripDirectories(file, self.StripDirs)

	log := &Log{
		Prefix:     self.Prefix,
		Level:      level,
		Filename:   file,
		Line:       line,
		Timestamp:  time.Now(),
		MessageFmt: messageFmt,
		Args:       args,
	}

	// cache disabled for now as not yet used -Tim
	//	Cache.Add(log)

	for _, appender := range self.Appenders {
		if err := appender.Append(log); err != nil {
			error := fmt.Errorf("Error appending. Appender: %T Error: %v", appender, err)
			errors = append(errors, error)
		}
	}

	return log, errors
}

type Level uint8

// The level is in an order such that the expressions
// `level < WARN`, `level >= INFO` have intuitive meaning.
const (
	OFF Level = iota
	DEBUG
	ROUTINE
	INFO
	WARN
	ERROR
	DOOM
	topLevel
)

var strToLevel map[string]Level

var levelToStr []string

func init() {
	strToLevel = map[string]Level {
		"off"     : OFF,
		"debug"   : DEBUG,
		"routine" : ROUTINE,
		"info"    : INFO,
		"warn"    : WARN,
		"error"   : ERROR,
		"doom"    : DOOM,
	}

	levelToStr = make([]string, len(strToLevel))
	for str, level := range strToLevel {
		levelToStr[uint8(level)] = str
	}
}

func newLevel(levelStr string) (Level, error) {
	level, ok := strToLevel[strings.ToLower(levelStr)]

	if !ok {
		err := UnknownLevelError{levelStr}
		return OFF, err
	}

	return level, nil
}

func (self Level) Type() string {
	if self >= topLevel {
		return "off?"
	}

	return levelToStr[uint8(self)]
}

func stacktrace() []string {
	ret := make([]string, 0, 2)
	for skip := 2; true; skip++ {
		_, file, line, ok := runtime.Caller(skip)
		if ok == false {
			break
		}

		ret = append(ret, fmt.Sprintf("at %s:%d", stripDirectories(file, 2), line))
	}

	return ret
}

type StackError struct {
	Message    string
	Stacktrace []string
}

func NewStackError(messageFmt string, args ...interface{}) *StackError {
	return &StackError{
		Message:    fmt.Sprintf(messageFmt, args...),
		Stacktrace: stacktrace(),
	}
}

func (self *StackError) Error() string {
	return fmt.Sprintf("%s\n\t%s", self.Message, strings.Join(self.Stacktrace, "\n\t"))
}

func stripDirectories(filepath string, toKeep int) string {
	var idxCutoff int
	if idxCutoff = strings.LastIndex(filepath, "/"); idxCutoff == -1 {
		return filepath
	}

	for dirToKeep := 0; dirToKeep < toKeep; dirToKeep++ {
		switch idx := strings.LastIndex(filepath[:idxCutoff], "/"); idx {
		case -1:
			break
		default:
			idxCutoff = idx
		}
	}

	return filepath[idxCutoff+1:]
}

type UnknownLevelError struct {
	levelStr string
}

func (self UnknownLevelError) Error() string {
	return fmt.Sprintf("Unknown level: %s", self.levelStr)
}

<|endoftext|>


<commit_before>package fun

import (
	cryptoRand "crypto/rand"
	"encoding/base64"
	"errors"
	"io"
	"math"
	mathRand "math/rand"
	"strconv"
	"time"
)

func init() {
	mathRand.Seed(time.Now().UnixNano())
}

// Returns a random int in the range [min, lessThan)
func RandomBetween(min, lessThan int) int {
	return mathRand.Intn(lessThan-min) + min
}

// Returns a random number with numDigits digits
func RandomDigits(numDigits int) int {
	min := int(math.Pow10(numDigits - 1))
	lessThan := int(math.Pow10(numDigits))
	return RandomBetween(min, lessThan)
}

func RandomDigitString(numDigits int) string {
	return strconv.Itoa(RandomDigits(numDigits))
}

func MakeUid(numChars int) (uid string, err error) {
	if numChars%4 != 0 {
		err = errors.New("uid length must be a multiple of 4")
		return
	}

	buf := make([]byte, numChars)
	_, err = io.ReadFull(cryptoRand.Reader, buf)
	if err != nil {
		return
	}

	uid = base64.URLEncoding.EncodeToString(buf)

	return
}
<commit_msg>Fix RandomDigits<commit_after>package fun

import (
	cryptoRand "crypto/rand"
	"encoding/base64"
	"errors"
	"io"
	"math"
	mathRand "math/rand"
	"strconv"
	"time"
)

func init() {
	mathRand.Seed(time.Now().UnixNano())
}

// Returns a random int in the range [min, lessThan)
func RandomBetween(min, lessThan int) int {
	return mathRand.Intn(lessThan-min) + min
}

// Returns a random number with numDigits digits
func RandomDigits(numDigits int) string {
	min := int(math.Pow10(numDigits - 1))
	lessThan := int(math.Pow10(numDigits))
	return strconv.Itoa(RandomBetween(min, lessThan))
}

func MakeUid(numChars int) (uid string, err error) {
	if numChars%4 != 0 {
		err = errors.New("uid length must be a multiple of 4")
		return
	}

	buf := make([]byte, numChars)
	_, err = io.ReadFull(cryptoRand.Reader, buf)
	if err != nil {
		return
	}

	uid = base64.URLEncoding.EncodeToString(buf)

	return
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Pagoda Box Inc
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v.
// 2.0. If a copy of the MPL was not distributed with this file, You can obtain one
// at http://mozilla.org/MPL/2.0/.
//

package mist

import (
	set "github.com/deckarep/golang-set"
	"sync"
	"sync/atomic"
)

//
type (

	//
	Mist struct {
		clients map[uint32]MistClient
		next    uint32
	}

	//
	MistClient struct {
		sync.Mutex

		check chan Message
		done  chan bool
		pipe  chan Message

		subscriptions []set.Set
		mist          *Mist
		id            uint32
	}

	// A Message contains the tags used when subscribing, and the data that is being
	// published through mist
	Message struct {
		tags set.Set
		Tags []string    `json:"tags"`
		Data interface{} `json:"data"`
	}
)

// creates a new mist
func New() *Mist {

	return &Mist{
		clients: make(map[uint32]MistClient),
	}
}

func makeSet(tags []string) set.Set {
	set := set.NewThreadUnsafeSet()
	for _, i := range tags {
		set.Add(i)
	}

	return set
}

// Publish takes a list of tags and iterates through mist's list of clients,
// sending to each if they are available.
func (mist *Mist) Publish(tags []string, data interface{}) {

	message := Message{
		Tags: tags,
		tags: makeSet(tags),
		Data: data}

	for _, client := range mist.clients {
		// currently a slow reader will cause all readers to be slow.
		select {
		case <-client.done:
		case client.check <- message:
			// default:
			// 	// do we drop the message? enqueue it? pull one off the front and then add this one?
		}
	}
}

func (mist *Mist) NewClient(buffer int) *MistClient {
	client := MistClient{
		check: make(chan Message, buffer),
		done:  make(chan bool),
		pipe:  make(chan Message),
		mist:  mist,
		id:    atomic.AddUint32(&mist.next, 1)}

	// this gofunc handles matching messages to subscriptions for the client
	go func(client *MistClient) {

		defer func() {
			close(client.check)
			close(client.pipe)
		}()

		for {
			select {
			case msg := <-client.check:
				// we do this so that we don't need a mutex
				subscriptions := client.subscriptions
				for _, subscription := range subscriptions {
					if subscription.IsSubset(msg.tags) {
						client.pipe <- msg
					}
				}
			case <-client.done:
				return
			}
		}
	}(&client)

	mist.clients[client.id] = client

	return &client
}

func (client *MistClient) Subscribe(tags []string) {
	subscription := makeSet(tags)

	client.Lock()
	client.subscriptions = append(client.subscriptions, subscription)
	client.Unlock()
}

// Unsubscribe iterates through each of mist clients subscriptions keeping all subscriptions
// that aren't the specified subscription
func (client *MistClient) Unsubscribe(tags []string) {
	client.Lock()

	//create a set for quick comparison
	test := makeSet(tags)

	// create a slice of subscriptions that are going to be kept
	keep := []set.Set{}

	// iterate over all of mist clients subscriptions looking for ones that match the
	// subscription to unsubscribe
	for _, subscription := range client.subscriptions {

		// if they are not the same set (meaning they are a different subscription) then add them
		// to the keep set
		if !test.Equal(subscription) {
			keep = append(keep, subscription)
		}
	}

	client.subscriptions = keep

	client.Unlock()
}

func (client *MistClient) List() [][]string {
	subscriptions := make([][]string, len(client.subscriptions))
	for i, subscription := range client.subscriptions {
		sub := make([]string, subscription.Cardinality())
		for j, tag := range subscription.ToSlice() {
			sub[j] = tag.(string)
		}
		subscriptions[i] = sub
	}
	return subscriptions
}

func (client *MistClient) Close() {
	// this closes the goroutine that is matching messages to subscriptions
	close(client.done)

	// remove this client from mist
	delete(client.mist.clients, client.id)
}

// Returns all messages that have sucessfully matched the list of subscriptions that this
// client has subscribed to
func (client *MistClient) Messages() <-chan Message {
	return client.pipe
}

// Sends a message across mist
func (client *MistClient) Publish(tags []string, data interface{}) {
	client.mist.Publish(tags, data)
}
<commit_msg>enabled lossy forwarding<commit_after>// Copyright (c) 2015 Pagoda Box Inc
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v.
// 2.0. If a copy of the MPL was not distributed with this file, You can obtain one
// at http://mozilla.org/MPL/2.0/.
//

package mist

import (
	set "github.com/deckarep/golang-set"
	"sync"
	"sync/atomic"
)

//
type (

	//
	Mist struct {
		clients map[uint32]MistClient
		next    uint32
	}

	//
	MistClient struct {
		sync.Mutex

		check chan Message
		done  chan bool
		pipe  chan Message

		subscriptions []set.Set
		mist          *Mist
		id            uint32
	}

	// A Message contains the tags used when subscribing, and the data that is being
	// published through mist
	Message struct {
		tags set.Set
		Tags []string    `json:"tags"`
		Data interface{} `json:"data"`
	}
)

// creates a new mist
func New() *Mist {

	return &Mist{
		clients: make(map[uint32]MistClient),
	}
}

func makeSet(tags []string) set.Set {
	set := set.NewThreadUnsafeSet()
	for _, i := range tags {
		set.Add(i)
	}

	return set
}

// Publish takes a list of tags and iterates through mist's list of clients,
// sending to each if they are available.
func (mist *Mist) Publish(tags []string, data interface{}) {

	message := Message{
		Tags: tags,
		tags: makeSet(tags),
		Data: data}

	for _, client := range mist.clients {
		// currently a slow reader will cause all readers to be slow.
		select {
		case <-client.done:
		case client.check <- message:
		default:
			// do we drop the message? enqueue it? pull one off the front and then add this one?
		}
	}
}

func (mist *Mist) NewClient(buffer int) *MistClient {
	client := MistClient{
		check: make(chan Message, buffer),
		done:  make(chan bool),
		pipe:  make(chan Message),
		mist:  mist,
		id:    atomic.AddUint32(&mist.next, 1)}

	// this gofunc handles matching messages to subscriptions for the client
	go func(client *MistClient) {

		defer func() {
			close(client.check)
			close(client.pipe)
		}()

		for {
			select {
			case msg := <-client.check:
				// we do this so that we don't need a mutex
				subscriptions := client.subscriptions
				for _, subscription := range subscriptions {
					if subscription.IsSubset(msg.tags) {
						client.pipe <- msg
					}
				}
			case <-client.done:
				return
			}
		}
	}(&client)

	mist.clients[client.id] = client

	return &client
}

func (client *MistClient) Subscribe(tags []string) {
	subscription := makeSet(tags)

	client.Lock()
	client.subscriptions = append(client.subscriptions, subscription)
	client.Unlock()
}

// Unsubscribe iterates through each of mist clients subscriptions keeping all subscriptions
// that aren't the specified subscription
func (client *MistClient) Unsubscribe(tags []string) {
	client.Lock()

	//create a set for quick comparison
	test := makeSet(tags)

	// create a slice of subscriptions that are going to be kept
	keep := []set.Set{}

	// iterate over all of mist clients subscriptions looking for ones that match the
	// subscription to unsubscribe
	for _, subscription := range client.subscriptions {

		// if they are not the same set (meaning they are a different subscription) then add them
		// to the keep set
		if !test.Equal(subscription) {
			keep = append(keep, subscription)
		}
	}

	client.subscriptions = keep

	client.Unlock()
}

func (client *MistClient) List() [][]string {
	subscriptions := make([][]string, len(client.subscriptions))
	for i, subscription := range client.subscriptions {
		sub := make([]string, subscription.Cardinality())
		for j, tag := range subscription.ToSlice() {
			sub[j] = tag.(string)
		}
		subscriptions[i] = sub
	}
	return subscriptions
}

func (client *MistClient) Close() {
	// this closes the goroutine that is matching messages to subscriptions
	close(client.done)

	// remove this client from mist
	delete(client.mist.clients, client.id)
}

// Returns all messages that have sucessfully matched the list of subscriptions that this
// client has subscribed to
func (client *MistClient) Messages() <-chan Message {
	return client.pipe
}

// Sends a message across mist
func (client *MistClient) Publish(tags []string, data interface{}) {
	client.mist.Publish(tags, data)
}
<|endoftext|>


<commit_before>package msgpack

import (
	"fmt"
	"reflect"

	"github.com/vmihailenco/msgpack/v5/msgpcode"
)

var (
	mapStringStringPtrType = reflect.TypeOf((*map[string]string)(nil))
	mapStringStringType    = mapStringStringPtrType.Elem()
)

var (
	mapStringInterfacePtrType = reflect.TypeOf((*map[string]interface{})(nil))
	mapStringInterfaceType    = mapStringInterfacePtrType.Elem()
)

func decodeMapValue(d *Decoder, v reflect.Value) error {
	n, err := d.DecodeMapLen()
	if err != nil {
		return err
	}

	typ := v.Type()
	if n == -1 {
		v.Set(reflect.Zero(typ))
		return nil
	}

	if v.IsNil() {
		v.Set(reflect.MakeMap(typ))
	}
	if n == 0 {
		return nil
	}

	return d.decodeTypedMapValue(v, n)
}

func (d *Decoder) decodeMapDefault() (interface{}, error) {
	if d.decodeMapFunc != nil {
		return d.decodeMapFunc(d)
	}

	n, err := d.DecodeMapLen()
	if err != nil {
		return nil, err
	}
	if n == -1 {
		return nil, nil
	}

	code, err := d.PeekCode()
	if err != nil {
		return nil, err
	}

	if msgpcode.IsString(code) || msgpcode.IsBin(code) || msgpcode.IsExt(code) {
		return d.decodeMap(n)
	}
	return d.decodeUntypedMap(n)
}

// DecodeMapLen decodes map length. Length is -1 when map is nil.
func (d *Decoder) DecodeMapLen() (int, error) {
	c, err := d.readCode()
	if err != nil {
		return 0, err
	}

	if msgpcode.IsExt(c) {
		if err = d.skipExtHeader(c); err != nil {
			return 0, err
		}

		c, err = d.readCode()
		if err != nil {
			return 0, err
		}
	}
	return d.mapLen(c)
}

func (d *Decoder) mapLen(c byte) (int, error) {
	if c == msgpcode.Nil {
		return -1, nil
	}
	if c >= msgpcode.FixedMapLow && c <= msgpcode.FixedMapHigh {
		return int(c & msgpcode.FixedMapMask), nil
	}
	if c == msgpcode.Map16 {
		size, err := d.uint16()
		return int(size), err
	}
	if c == msgpcode.Map32 {
		size, err := d.uint32()
		return int(size), err
	}
	return 0, unexpectedCodeError{code: c, hint: "map length"}
}

func decodeMapStringStringValue(d *Decoder, v reflect.Value) error {
	mptr := v.Addr().Convert(mapStringStringPtrType).Interface().(*map[string]string)
	return d.decodeMapStringStringPtr(mptr)
}

func (d *Decoder) decodeMapStringStringPtr(ptr *map[string]string) error {
	size, err := d.DecodeMapLen()
	if err != nil {
		return err
	}
	if size == -1 {
		*ptr = nil
		return nil
	}

	m := *ptr
	if m == nil {
		*ptr = make(map[string]string, min(size, maxMapSize))
		m = *ptr
	}

	for i := 0; i < size; i++ {
		mk, err := d.DecodeString()
		if err != nil {
			return err
		}
		mv, err := d.DecodeString()
		if err != nil {
			return err
		}
		m[mk] = mv
	}

	return nil
}

func decodeMapStringInterfaceValue(d *Decoder, v reflect.Value) error {
	ptr := v.Addr().Convert(mapStringInterfacePtrType).Interface().(*map[string]interface{})
	return d.decodeMapStringInterfacePtr(ptr)
}

func (d *Decoder) decodeMapStringInterfacePtr(ptr *map[string]interface{}) error {
	n, err := d.DecodeMapLen()
	if err != nil {
		return err
	}
	if n == -1 {
		*ptr = nil
		return nil
	}

	m := *ptr
	if m == nil {
		*ptr = make(map[string]interface{}, min(n, maxMapSize))
		m = *ptr
	}

	for i := 0; i < n; i++ {
		mk, err := d.DecodeString()
		if err != nil {
			return err
		}
		mv, err := d.decodeInterfaceCond()
		if err != nil {
			return err
		}
		m[mk] = mv
	}

	return nil
}

func (d *Decoder) DecodeMap() (map[string]interface{}, error) {
	n, err := d.DecodeMapLen()
	if err != nil {
		return nil, err
	}
	if n == -1 {
		return nil, nil
	}
	return d.decodeMap(n)
}

func (d *Decoder) decodeMap(n int) (map[string]interface{}, error) {
	m := make(map[string]interface{}, min(n, maxMapSize))
	for i := 0; i < n; i++ {
		mk, err := d.DecodeString()
		if err != nil {
			return nil, err
		}
		mv, err := d.decodeInterfaceCond()
		if err != nil {
			return nil, err
		}
		m[mk] = mv
	}
	return m, nil
}

func (d *Decoder) DecodeUntypedMap() (interface{}, error) {
	n, err := d.DecodeMapLen()
	if err != nil {
		return nil, err
	}
	if n == -1 {
		return nil, nil
	}
	return d.decodeUntypedMap(n)
}

func (d *Decoder) decodeUntypedMap(n int) (interface{}, error) {
	m := make(map[interface{}]interface{}, min(n, maxMapSize))

	for i := 0; i < n; i++ {
		mk, err := d.decodeInterfaceCond()
		if err != nil {
			return nil, err
		}

		mv, err := d.decodeInterfaceCond()
		if err != nil {
			return nil, err
		}

		m[mk] = mv
	}

	return m, nil
}

func (d *Decoder) DecodeTypedMap() (interface{}, error) {
	n, err := d.DecodeMapLen()
	if err != nil {
		return nil, err
	}
	if n == -1 {
		return nil, nil
	}

	key, err := d.decodeInterfaceCond()
	if err != nil {
		return nil, err
	}

	value, err := d.decodeInterfaceCond()
	if err != nil {
		return nil, err
	}

	keyType := reflect.TypeOf(key)
	valueType := reflect.TypeOf(value)

	if !keyType.Comparable() {
		return nil, fmt.Errorf("msgpack: unsupported map key: %s", keyType.String())
	}

	mapType := reflect.MapOf(keyType, valueType)
	mapValue := reflect.MakeMap(mapType)
	mapValue.SetMapIndex(reflect.ValueOf(key), reflect.ValueOf(value))

	n--
	if err := d.decodeTypedMapValue(mapValue, n); err != nil {
		return nil, err
	}

	return mapValue.Interface(), nil
}

func (d *Decoder) decodeTypedMapValue(v reflect.Value, n int) error {
	typ := v.Type()
	keyType := typ.Key()
	valueType := typ.Elem()

	for i := 0; i < n; i++ {
		mk := reflect.New(keyType).Elem()
		if err := d.DecodeValue(mk); err != nil {
			return err
		}

		mv := reflect.New(valueType).Elem()
		if err := d.DecodeValue(mv); err != nil {
			return err
		}

		v.SetMapIndex(mk, mv)
	}

	return nil
}

func (d *Decoder) skipMap(c byte) error {
	n, err := d.mapLen(c)
	if err != nil {
		return err
	}
	for i := 0; i < n; i++ {
		if err := d.Skip(); err != nil {
			return err
		}
		if err := d.Skip(); err != nil {
			return err
		}
	}
	return nil
}

func decodeStructValue(d *Decoder, v reflect.Value) error {
	c, err := d.readCode()
	if err != nil {
		return err
	}

	var isArray bool

	n, err := d.mapLen(c)
	if err != nil {
		var err2 error
		n, err2 = d.arrayLen(c)
		if err2 != nil {
			return err
		}
		isArray = true
	}
	if n == -1 {
		v.Set(reflect.Zero(v.Type()))
		return nil
	}

	fields := structs.Fields(v.Type(), d.structTag)
	if isArray {
		for i, f := range fields.List {
			if i >= n {
				break
			}
			if err := f.DecodeValue(d, v); err != nil {
				return err
			}
		}

		// Skip extra values.
		for i := len(fields.List); i < n; i++ {
			if err := d.Skip(); err != nil {
				return err
			}
		}

		return nil
	}

	for i := 0; i < n; i++ {
		name, err := d.bytesTemp()
		if err != nil {
			return err
		}

		if f := fields.Map[string(name)]; f != nil {
			if err := f.DecodeValue(d, v); err != nil {
				return err
			}
		} else if d.flags&disallowUnknownFieldsFlag != 0 {
			return fmt.Errorf("msgpack: unknown field %q", name)
		} else if err := d.Skip(); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Cleanup<commit_after>package msgpack

import (
	"fmt"
	"reflect"

	"github.com/vmihailenco/msgpack/v5/msgpcode"
)

var (
	mapStringStringPtrType = reflect.TypeOf((*map[string]string)(nil))
	mapStringStringType    = mapStringStringPtrType.Elem()
)

var (
	mapStringInterfacePtrType = reflect.TypeOf((*map[string]interface{})(nil))
	mapStringInterfaceType    = mapStringInterfacePtrType.Elem()
)

func decodeMapValue(d *Decoder, v reflect.Value) error {
	n, err := d.DecodeMapLen()
	if err != nil {
		return err
	}

	typ := v.Type()
	if n == -1 {
		v.Set(reflect.Zero(typ))
		return nil
	}

	if v.IsNil() {
		v.Set(reflect.MakeMap(typ))
	}
	if n == 0 {
		return nil
	}

	return d.decodeTypedMapValue(v, n)
}

func (d *Decoder) decodeMapDefault() (interface{}, error) {
	if d.decodeMapFunc != nil {
		return d.decodeMapFunc(d)
	}

	n, err := d.DecodeMapLen()
	if err != nil {
		return nil, err
	}
	if n == -1 {
		return nil, nil
	}

	code, err := d.PeekCode()
	if err != nil {
		return nil, err
	}

	if msgpcode.IsString(code) || msgpcode.IsBin(code) || msgpcode.IsExt(code) {
		return d.decodeMap(n)
	}
	return d.decodeUntypedMap(n)
}

// DecodeMapLen decodes map length. Length is -1 when map is nil.
func (d *Decoder) DecodeMapLen() (int, error) {
	c, err := d.readCode()
	if err != nil {
		return 0, err
	}

	if msgpcode.IsExt(c) {
		if err = d.skipExtHeader(c); err != nil {
			return 0, err
		}

		c, err = d.readCode()
		if err != nil {
			return 0, err
		}
	}
	return d.mapLen(c)
}

func (d *Decoder) mapLen(c byte) (int, error) {
	if c == msgpcode.Nil {
		return -1, nil
	}
	if c >= msgpcode.FixedMapLow && c <= msgpcode.FixedMapHigh {
		return int(c & msgpcode.FixedMapMask), nil
	}
	if c == msgpcode.Map16 {
		size, err := d.uint16()
		return int(size), err
	}
	if c == msgpcode.Map32 {
		size, err := d.uint32()
		return int(size), err
	}
	return 0, unexpectedCodeError{code: c, hint: "map length"}
}

func decodeMapStringStringValue(d *Decoder, v reflect.Value) error {
	mptr := v.Addr().Convert(mapStringStringPtrType).Interface().(*map[string]string)
	return d.decodeMapStringStringPtr(mptr)
}

func (d *Decoder) decodeMapStringStringPtr(ptr *map[string]string) error {
	size, err := d.DecodeMapLen()
	if err != nil {
		return err
	}
	if size == -1 {
		*ptr = nil
		return nil
	}

	m := *ptr
	if m == nil {
		*ptr = make(map[string]string, min(size, maxMapSize))
		m = *ptr
	}

	for i := 0; i < size; i++ {
		mk, err := d.DecodeString()
		if err != nil {
			return err
		}
		mv, err := d.DecodeString()
		if err != nil {
			return err
		}
		m[mk] = mv
	}

	return nil
}

func decodeMapStringInterfaceValue(d *Decoder, v reflect.Value) error {
	ptr := v.Addr().Convert(mapStringInterfacePtrType).Interface().(*map[string]interface{})
	return d.decodeMapStringInterfacePtr(ptr)
}

func (d *Decoder) decodeMapStringInterfacePtr(ptr *map[string]interface{}) error {
	m, err := d.DecodeMap()
	if err != nil {
		return err
	}
	*ptr = m
	return nil
}

func (d *Decoder) DecodeMap() (map[string]interface{}, error) {
	n, err := d.DecodeMapLen()
	if err != nil {
		return nil, err
	}
	if n == -1 {
		return nil, nil
	}
	return d.decodeMap(n)
}

func (d *Decoder) decodeMap(n int) (map[string]interface{}, error) {
	m := make(map[string]interface{}, min(n, maxMapSize))
	for i := 0; i < n; i++ {
		mk, err := d.DecodeString()
		if err != nil {
			return nil, err
		}
		mv, err := d.decodeInterfaceCond()
		if err != nil {
			return nil, err
		}
		m[mk] = mv
	}
	return m, nil
}

func (d *Decoder) DecodeUntypedMap() (interface{}, error) {
	n, err := d.DecodeMapLen()
	if err != nil {
		return nil, err
	}
	if n == -1 {
		return nil, nil
	}
	return d.decodeUntypedMap(n)
}

func (d *Decoder) decodeUntypedMap(n int) (interface{}, error) {
	m := make(map[interface{}]interface{}, min(n, maxMapSize))

	for i := 0; i < n; i++ {
		mk, err := d.decodeInterfaceCond()
		if err != nil {
			return nil, err
		}

		mv, err := d.decodeInterfaceCond()
		if err != nil {
			return nil, err
		}

		m[mk] = mv
	}

	return m, nil
}

func (d *Decoder) DecodeTypedMap() (interface{}, error) {
	n, err := d.DecodeMapLen()
	if err != nil {
		return nil, err
	}
	if n == -1 {
		return nil, nil
	}

	key, err := d.decodeInterfaceCond()
	if err != nil {
		return nil, err
	}

	value, err := d.decodeInterfaceCond()
	if err != nil {
		return nil, err
	}

	keyType := reflect.TypeOf(key)
	valueType := reflect.TypeOf(value)

	if !keyType.Comparable() {
		return nil, fmt.Errorf("msgpack: unsupported map key: %s", keyType.String())
	}

	mapType := reflect.MapOf(keyType, valueType)
	mapValue := reflect.MakeMap(mapType)
	mapValue.SetMapIndex(reflect.ValueOf(key), reflect.ValueOf(value))

	n--
	if err := d.decodeTypedMapValue(mapValue, n); err != nil {
		return nil, err
	}

	return mapValue.Interface(), nil
}

func (d *Decoder) decodeTypedMapValue(v reflect.Value, n int) error {
	typ := v.Type()
	keyType := typ.Key()
	valueType := typ.Elem()

	for i := 0; i < n; i++ {
		mk := reflect.New(keyType).Elem()
		if err := d.DecodeValue(mk); err != nil {
			return err
		}

		mv := reflect.New(valueType).Elem()
		if err := d.DecodeValue(mv); err != nil {
			return err
		}

		v.SetMapIndex(mk, mv)
	}

	return nil
}

func (d *Decoder) skipMap(c byte) error {
	n, err := d.mapLen(c)
	if err != nil {
		return err
	}
	for i := 0; i < n; i++ {
		if err := d.Skip(); err != nil {
			return err
		}
		if err := d.Skip(); err != nil {
			return err
		}
	}
	return nil
}

func decodeStructValue(d *Decoder, v reflect.Value) error {
	c, err := d.readCode()
	if err != nil {
		return err
	}

	var isArray bool

	n, err := d.mapLen(c)
	if err != nil {
		var err2 error
		n, err2 = d.arrayLen(c)
		if err2 != nil {
			return err
		}
		isArray = true
	}
	if n == -1 {
		v.Set(reflect.Zero(v.Type()))
		return nil
	}

	fields := structs.Fields(v.Type(), d.structTag)
	if isArray {
		for i, f := range fields.List {
			if i >= n {
				break
			}
			if err := f.DecodeValue(d, v); err != nil {
				return err
			}
		}

		// Skip extra values.
		for i := len(fields.List); i < n; i++ {
			if err := d.Skip(); err != nil {
				return err
			}
		}

		return nil
	}

	for i := 0; i < n; i++ {
		name, err := d.bytesTemp()
		if err != nil {
			return err
		}

		if f := fields.Map[string(name)]; f != nil {
			if err := f.DecodeValue(d, v); err != nil {
				return err
			}
		} else if d.flags&disallowUnknownFieldsFlag != 0 {
			return fmt.Errorf("msgpack: unknown field %q", name)
		} else if err := d.Skip(); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package roundRegister
import (
  "time"
  "fmt"
)

type roundRegisterResponseType int

const (
    nackREADType roundRegisterResponseType  = iota
    ackREADType
    nackWRITEType
    ackWriteType
)
type roundRegisterCommandType int
const (
    readType roundRegisterCommandType = iota
    writeType
)
type roundRegisterResponse struct {
    responseType roundRegisterResponseType
    k time.Time
    write time.Time
    v interface{}
}

type roundRegisterCommand struct {
    CommandType roundRegisterCommandType
    K time.Time
    V interface{}
}

type RoundRegister struct {
    bootStrap time.Duration
	read time.Time
    write time.Time
    value interface{}
    //The number of peers must be known in advance and must not increase during runtime,
    peers []string
    transport Transport
}

func NewRoundRegister(tr Transport, peersList []string, bootStrap time.Duration) (*RoundRegister) {
	r := &RoundRegister{
		read: time.Now(),
        write: time.Now(),
        value: nil,
        transport: tr,
		peers:      peersList,
        bootStrap: bootStrap,
	}
    go r.HandleRequests()
	return r
}

func (r *RoundRegister) SendRead(target string, k time.Time) (error, *roundRegisterResponse) {
    cmd := &roundRegisterCommand{}
    cmd.CommandType = readType
    cmd.K = k
    resp, err := r.transport.makeRPC(target,cmd)
    if (err != nil){
        return err, nil
    }
    roundRegisterResp := resp.Response.(*roundRegisterResponse)
    return nil, roundRegisterResp
}

func (r *RoundRegister) SendWrite(target string, k time.Time, v interface{}) (error, *roundRegisterResponse) {
    cmd := &roundRegisterCommand{}
    cmd.CommandType = writeType
    cmd.K = k
    cmd.V = v
    resp, err := r.transport.makeRPC(target,cmd)
    if (err != nil){
        return err, nil
    }
    roundRegisterResp := resp.Response.(*roundRegisterResponse)
    return nil, roundRegisterResp
}

// Read value using Round k
func (r *RoundRegister) Read(k time.Time) (error, interface{}) {
    respArray := []*roundRegisterResponse  {}
    var maxWriteTime time.Time
    var maxVal interface{}
    // wait for (n+1)/2 response
    for _ , element := range r.peers {
        //TODO handle RPC timeout
        err, resp := r.SendRead(element, k)
        if (err != nil){
            return err, nil
        }
        respArray = append(respArray, resp)
    }

    // Return the element with the highest write version number
    // from a majority of peers
    for _ , element := range respArray {
        if (element.responseType== nackREADType){
            return fmt.Errorf("abort"), nil
        }
        if (element.write.After(maxWriteTime)) {
                maxWriteTime = element.write
                maxVal = element.v
        }
    }
    return nil, maxVal
}

// Write value using round k
func (r *RoundRegister) Write(k time.Time, value interface{}) error {
    // wait for (n+1)/2 response
    for _ , element := range r.peers {
        //TODO handle RPC timeout
        err, resp  := r.SendWrite(element, k, value)
        if (err !=nil){
           return err
        }
        if (resp.responseType == nackWRITEType){
            // if received at least one nack abort
            return fmt.Errorf("abort")
        }

    }
    // Received ack from (n+1)/2 node, Commit
    return nil

}

func (r *RoundRegister) HandleRequest(req *roundRegisterCommand) (*roundRegisterResponse, error){
 resp := &roundRegisterResponse {}
 // Handle read request
 if (req.CommandType == readType) {
        if (r.write.After(req.K)  || r.read.After(req.K)) {
            resp.responseType = nackREADType
            resp.k = req.K
        } else {
            r.read = req.K
            resp.responseType = ackREADType
            resp.k = req.K
            resp.write = r.write
            resp.v = r.value
        }
    return resp, nil
 }
 // Handle write request
 if (req.CommandType == writeType) {
    if (r.write.After(req.K) || r.read.After(req.K)) {
         resp.responseType = nackWRITEType
         resp.k = req.K
    } else {
        r.write = req.K
        r.value = req.V
        resp.responseType = ackWriteType
        resp.k = req.K
    }
    return resp, nil
 }
 return resp, fmt.Errorf("unknow command")
}

// Listen for request in loop
func (r *RoundRegister) HandleRequests() {
        time.Sleep(r.bootStrap)
		for {
			select {
			case rpc := <-r.transport.Consumer():
				// Verify the command
				req := rpc.Command.(*roundRegisterCommand)
                resp, err := r.HandleRequest(req)
                rpc.RespChan <- RPCResponse {
                    Response: resp,
                    Error: err,
                }
			}
		}
}<commit_msg>update roundRegister<commit_after>package roundRegister
import (
  "time"
  "fmt"
)

type roundRegisterResponseType int

const (
    nackREADType roundRegisterResponseType  = iota
    ackREADType
    nackWRITEType
    ackWriteType
    bootStraping
)
type roundRegisterCommandType int
const (
    readType roundRegisterCommandType = iota
    writeType
)
type roundRegisterResponse struct {
    responseType roundRegisterResponseType
    k time.Time
    write time.Time
    v interface{}
}

type roundRegisterCommand struct {
    CommandType roundRegisterCommandType
    K time.Time
    V interface{}
}

type RoundRegister struct {
    bootStrap time.Duration
	read time.Time
    write time.Time
    value interface{}
    //The number of peers must be known in advance and must not increase during runtime,
    peers []string
    transport Transport
}

func NewRoundRegister(tr Transport, peersList []string, bootStrapDuration time.Duration) (*RoundRegister) {
	r := &RoundRegister{
		read: time.Now(),
        write: time.Now(),
        value: nil,
        transport: tr,
		peers:      peersList,
        bootStrap: bootStrapDuration,
	}
    go r.HandleRequests()
	return r
}

func (r *RoundRegister) SendRead(target string, k time.Time) (error, *roundRegisterResponse) {
    cmd := &roundRegisterCommand{}
    cmd.CommandType = readType
    cmd.K = k
    resp, err := r.transport.makeRPC(target,cmd)
    if (err != nil){
        return err, nil
    }
    roundRegisterResp := resp.Response.(*roundRegisterResponse)
    return nil, roundRegisterResp
}

func (r *RoundRegister) SendWrite(target string, k time.Time, v interface{}) (error, *roundRegisterResponse) {
    cmd := &roundRegisterCommand{}
    cmd.CommandType = writeType
    cmd.K = k
    cmd.V = v
    resp, err := r.transport.makeRPC(target,cmd)
    if (err != nil){
        return err, nil
    }
    roundRegisterResp := resp.Response.(*roundRegisterResponse)
    return nil, roundRegisterResp
}

// Read value using Round k
func (r *RoundRegister) Read(k time.Time) (error, interface{}) {
    // Create a response channel
    respCh := make(chan *roundRegisterResponse, len(r.peers))
    peerRead := func(peer string, k time.Time) {
            err, resp := r.SendRead(peer, k)
            if (err == nil){
                respCh <- resp
            }
    }
    var maxWriteTime time.Time
    var maxVal interface{}

    for _ , element := range r.peers {
        go peerRead(element, k)
    }

    responseReceived := 0
    responseNeeded := r.quorumSize()
    // wait for (n+1)/2 response
    for responseReceived < responseNeeded {
        select {
        case response := <-respCh:
        if (response.responseType == nackREADType){
               return fmt.Errorf("abort"), nil
        }
        if (response.responseType != bootStraping) {
          // Return the element with the highest write version number
          if (response.write.After(maxWriteTime)) {
                maxWriteTime = response.write
                maxVal = response.v
          }
          responseReceived++
        }
        case <-time.After(50 * time.Millisecond):
            return fmt.Errorf("timeout"), nil
        }
    }

    return nil, maxVal
}

func (r *RoundRegister) quorumSize() int {
    return (len(r.peers) / 2) +1;
}

// Write value using round k
func (r *RoundRegister) Write(k time.Time, value interface{}) error {
    // Create a response channel
    respCh := make(chan *roundRegisterResponse, len(r.peers))
    peerWrite := func(peer string) {
        err, resp  := r.SendWrite(peer, k, value)
        if (err ==nil){
           respCh <- resp
        }
    }
    for _ , element := range r.peers {
        go peerWrite(element)
    }
    responseReceived := 0
    responseNeeded := r.quorumSize()
    // wait for (n+1)/2 response
    for responseReceived < responseNeeded {
        select {
        case resp := <-respCh:
          if (resp.responseType != bootStraping) {
            if (resp.responseType == nackWRITEType){
              // if received at least one nack abort
              return fmt.Errorf("abort")
            }
            responseReceived++
          }
        case <-time.After(50 * time.Millisecond):
          return fmt.Errorf("timeout")
       }
    }
    // Received ack from (n+1)/2 node, Commit
    return nil

}

func (r *RoundRegister) handleRead(req *roundRegisterCommand) (*roundRegisterResponse, error){
  resp := &roundRegisterResponse {}
   if (r.write.After(req.K)  || r.read.After(req.K)) {
            resp.responseType = nackREADType
            resp.k = req.K
   } else {
            r.read = req.K
            resp.responseType = ackREADType
            resp.k = req.K
            resp.write = r.write
            resp.v = r.value
   }
   return resp, nil
}

func (r *RoundRegister) handleWrite(req *roundRegisterCommand) (*roundRegisterResponse, error){
  resp := &roundRegisterResponse {}
  if (r.write.After(req.K) || r.read.After(req.K)) {
    resp.responseType = nackWRITEType
    resp.k = req.K
  } else {
    r.write = req.K
    r.value = req.V
    resp.responseType = ackWriteType
    resp.k = req.K
  }
  return resp, nil
}

func (r *RoundRegister) HandleRequest(req *roundRegisterCommand) (*roundRegisterResponse, error){
 resp := &roundRegisterResponse {}
 switch req.CommandType {
 case readType:
     return r.handleRead(req)
 case writeType:
    return r.handleWrite(req)
 default:
    return resp, fmt.Errorf("unknow command")
 }

}

// Listen for request in loop
func (r *RoundRegister) HandleRequests() {
        time.Sleep(r.bootStrap)
		for {
			select {
			case rpc := <-r.transport.Consumer():
				// Verify the command
				req := rpc.Command.(*roundRegisterCommand)
                resp, err := r.HandleRequest(req)
                rpc.RespChan <- RPCResponse {
                    Response: resp,
                    Error: err,
                }
			}
		}
}<|endoftext|>


<commit_before>package routes_test

import (
	"bytes"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"os"
	"reflect"
	"strings"
	"testing"

	"github.com/gorilla/mux"
	"github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/gomega"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gstruct"
	"github.com/onsi/gomega/types"
	"github.com/petergtz/bitsgo/httputil"
	. "github.com/petergtz/bitsgo/routes"
	"github.com/petergtz/pegomock"
	. "github.com/petergtz/pegomock"
)

//go:generate pegomock generate --use-experimental-model-gen --package routes_test Blobstore

func TestRoutes(t *testing.T) {
	gomega.RegisterFailHandler(ginkgo.Fail)
	pegomock.RegisterMockFailHandler(ginkgo.Fail)
	ginkgo.RunSpecs(t, "Routes")
}

var _ = Describe("routes", func() {
	BeforeSuite(func() {
		log.SetFlags(log.LstdFlags | log.Lshortfile | log.LUTC)
	})

	var (
		blobstore      *MockBlobstore
		router         *mux.Router
		responseWriter *httptest.ResponseRecorder
	)

	BeforeEach(func() {
		blobstore = NewMockBlobstore()
		router = mux.NewRouter()
		responseWriter = httptest.NewRecorder()
	})

	ItSupportsMethodsGetPutDeleteFor := func(routeName string, resourceType string) {
		Context("Method GET", func() {
			It("returns StatusNotFound when blobstore returns NotFoundError", func() {
				When(func() { blobstore.Get(AnyString()) }).
					ThenReturn(nil, "", NewNotFoundError())

				router.ServeHTTP(responseWriter, httptest.NewRequest("GET", "/"+routeName+"/theguid", nil))

				Expect(responseWriter.Code).To(Equal(http.StatusNotFound))
			})

			It("returns StatusOK and fills body with contents from file located at the partitioned path", func() {
				When(func() { blobstore.Get("/th/eg/theguid") }).
					ThenReturn(ioutil.NopCloser(strings.NewReader("thecontent")), "", nil)

				router.ServeHTTP(responseWriter, httptest.NewRequest("GET", "/"+routeName+"/theguid", nil))

				Expect(*responseWriter).To(haveStatusCodeAndBody(
					Equal(http.StatusOK),
					Equal("thecontent")))
			})
		})

		Context("Method PUT", func() {
			It("returns StatusBadRequest when "+resourceType+" form file field is missing in request body", func() {
				router.ServeHTTP(responseWriter, httptest.NewRequest("PUT", "/"+routeName+"/theguid", nil))

				Expect(responseWriter.Code).To(Equal(http.StatusBadRequest))
			})

			It("returns StatusOK and an empty body, and forwards the file reader to the blobstore", func() {
				router.ServeHTTP(responseWriter, newHttpTestPutRequest("/"+routeName+"/theguid", map[string]map[string]io.Reader{
					resourceType: map[string]io.Reader{"somefilename": strings.NewReader("My test string")},
				}))

				Expect(*responseWriter).To(haveStatusCodeAndBody(
					Equal(http.StatusCreated),
					BeEmpty()))

				_, fileContent := blobstore.VerifyWasCalledOnce().Put(EqString("/th/eg/theguid"), AnyReadSeeker()).GetCapturedArguments()
				Expect(ioutil.ReadAll(fileContent)).To(MatchRegexp("My test string"))
			})
		})
	}

	Describe("/packages/{guid}", func() {
		BeforeEach(func() { SetUpPackageRoutes(router, blobstore) })
		ItSupportsMethodsGetPutDeleteFor("packages", "package")
	})

	Describe("/droplets/{guid}", func() {
		BeforeEach(func() { SetUpDropletRoutes(router, blobstore) })
		ItSupportsMethodsGetPutDeleteFor("droplets", "droplet")
	})

	Describe("/buildpacks/{guid}", func() {
		BeforeEach(func() { SetUpBuildpackRoutes(router, blobstore) })
		ItSupportsMethodsGetPutDeleteFor("buildpacks", "buildpack")
	})

	Describe("/app_stash", func() {
		BeforeEach(func() {
			SetUpAppStashRoutes(router, blobstore)
		})

		Describe("/app_stash/entries", func() {
			It("Unzips file and copies bits to blobstore", func() {
				zipFile, e := os.Open("test_data/test_archive.zip")
				Expect(e).NotTo(HaveOccurred())
				defer zipFile.Close()

				entries := make(map[string]string)

				When(blobstore.Put(AnyString(), AnyReadSeeker())).Then(func(params []pegomock.Param) pegomock.ReturnValues {
					content, e := ioutil.ReadAll(params[1].(io.ReadSeeker))
					Expect(e).NotTo(HaveOccurred())
					entries[params[0].(string)] = string(content)
					return nil
				})

				router.ServeHTTP(responseWriter, newHttpTestPostRequest("/app_stash/entries", map[string]map[string]io.Reader{
					"application": map[string]io.Reader{"application": zipFile},
				}))

				Expect(responseWriter.Code).To(Equal(http.StatusOK), responseWriter.Body.String())
				Expect(entries).To(HaveKeyWithValue("971555ab39d1dfe8dff8b78c2b20e85e01c06595", "1\n\n"))
				Expect(entries).To(HaveKeyWithValue("bbd33de01c17b165b4ce00308e8a19a942023ab8", "2\n\n"))
				Expect(entries).To(HaveKeyWithValue("27cc6f77ee63df90ab3285f9d5fc4ebcb2448c12", "3\n\n"))
			})
		})

		Describe("/app_stash/matches", func() {
			It("returns StatusUnprocessableEntity when body is invalid", func() {
				router.ServeHTTP(responseWriter, httptest.NewRequest(
					"POST", "/app_stash/matches", strings.NewReader("some invalid format")))

				Expect(responseWriter.Code).To(Equal(http.StatusUnprocessableEntity), responseWriter.Body.String())
			})

			It("returns StatusOK and missing fingerprints when body is valid", func() {
				When(blobstore.Exists("abc")).ThenReturn(true, nil)

				router.ServeHTTP(responseWriter, httptest.NewRequest(
					"POST", "/app_stash/matches", strings.NewReader("[{\"sha1\":\"abc\"}, {\"sha1\":\"def\"}]")))

				Expect(*responseWriter).To(haveStatusCodeAndBody(
					Equal(http.StatusOK),
					Equal("[{\"sha1\":\"def\"}]")))
			})
		})
	})
})

func haveStatusCodeAndBody(statusCode types.GomegaMatcher, body types.GomegaMatcher) types.GomegaMatcher {
	return MatchFields(IgnoreExtras, Fields{
		"Code": statusCode,
		"Body": WithTransform(func(body *bytes.Buffer) string { return body.String() }, body),
	})
}

func writeStatusCodeAndBody(statusCode int, body string) func([]Param) ReturnValues {
	return func(params []Param) ReturnValues {
		for _, param := range params {
			if responseWriter, ok := param.(http.ResponseWriter); ok {
				responseWriter.WriteHeader(statusCode)
				responseWriter.Write([]byte(body))
				return nil
			}
		}
		panic("Unexpected: no ResponseWriter in parameter list.")
	}
}

func newHttpTestPutRequest(path string, formFiles map[string]map[string]io.Reader) *http.Request {
	return newHttpTestRequest("PUT", path, formFiles)
}

func newHttpTestPostRequest(path string, formFiles map[string]map[string]io.Reader) *http.Request {
	return newHttpTestRequest("POST", path, formFiles)
}

func newHttpTestRequest(method string, path string, formFiles map[string]map[string]io.Reader) *http.Request {
	bodyBuf := &bytes.Buffer{}
	header, e := httputil.AddFormFileTo(bodyBuf, formFiles)
	Expect(e).NotTo(HaveOccurred())
	request := httptest.NewRequest(method, path, bodyBuf)
	httputil.AddHeaderTo(request, header)
	return request
}

func AnyReadSeeker() io.ReadSeeker {
	RegisterMatcher(NewAnyMatcher(reflect.TypeOf((*io.ReadSeeker)(nil)).Elem()))
	return nil
}
<commit_msg>Use InMemoryBlobstore in tests<commit_after>package routes_test

import (
	"bytes"
	"io"
	"log"
	"net/http"
	"net/http/httptest"
	"os"
	"reflect"
	"strings"
	"testing"

	"github.com/gorilla/mux"
	"github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/gomega"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gstruct"
	"github.com/onsi/gomega/types"
	"github.com/petergtz/bitsgo/httputil"
	"github.com/petergtz/bitsgo/inmemory_blobstore"
	. "github.com/petergtz/bitsgo/routes"
	"github.com/petergtz/pegomock"
	. "github.com/petergtz/pegomock"
)

//go:generate pegomock generate --use-experimental-model-gen --package routes_test Blobstore

func TestRoutes(t *testing.T) {
	gomega.RegisterFailHandler(ginkgo.Fail)
	pegomock.RegisterMockFailHandler(ginkgo.Fail)
	ginkgo.RunSpecs(t, "Routes")
}

var _ = Describe("routes", func() {
	BeforeSuite(func() {
		log.SetFlags(log.LstdFlags | log.Lshortfile | log.LUTC)
	})

	var (
		entries        map[string][]byte
		blobstore      *inmemory_blobstore.InMemoryBlobstore
		router         *mux.Router
		responseWriter *httptest.ResponseRecorder
	)

	BeforeEach(func() {
		entries = make(map[string][]byte)
		blobstore = inmemory_blobstore.NewInMemoryBlobstoreWithEntries(entries)
		router = mux.NewRouter()
		responseWriter = httptest.NewRecorder()
	})

	ItSupportsMethodsGetPutDeleteFor := func(routeName string, resourceType string) {
		Context("Method GET", func() {
			It("returns StatusNotFound when blobstore returns NotFoundError", func() {
				router.ServeHTTP(responseWriter, httptest.NewRequest("GET", "/"+routeName+"/theguid", nil))

				Expect(responseWriter.Code).To(Equal(http.StatusNotFound))
			})

			It("returns StatusOK and fills body with contents from file located at the partitioned path", func() {
				entries["/th/eg/theguid"] = []byte("thecontent")

				router.ServeHTTP(responseWriter, httptest.NewRequest("GET", "/"+routeName+"/theguid", nil))

				Expect(*responseWriter).To(haveStatusCodeAndBody(
					Equal(http.StatusOK),
					Equal("thecontent")))
			})
		})

		Context("Method PUT", func() {
			It("returns StatusBadRequest when "+resourceType+" form file field is missing in request body", func() {
				router.ServeHTTP(responseWriter, httptest.NewRequest("PUT", "/"+routeName+"/theguid", nil))

				Expect(responseWriter.Code).To(Equal(http.StatusBadRequest))
			})

			It("returns StatusOK and an empty body, and forwards the file reader to the blobstore", func() {
				router.ServeHTTP(responseWriter, newHttpTestPutRequest("/"+routeName+"/theguid", map[string]map[string]io.Reader{
					resourceType: map[string]io.Reader{"somefilename": strings.NewReader("My test string")},
				}))

				Expect(*responseWriter).To(haveStatusCodeAndBody(
					Equal(http.StatusCreated),
					BeEmpty()))

				Expect(entries).To(HaveKeyWithValue("/th/eg/theguid", []byte("My test string")))
			})
		})
	}

	Describe("/packages/{guid}", func() {
		BeforeEach(func() { SetUpPackageRoutes(router, blobstore) })
		ItSupportsMethodsGetPutDeleteFor("packages", "package")
	})

	Describe("/droplets/{guid}", func() {
		BeforeEach(func() { SetUpDropletRoutes(router, blobstore) })
		ItSupportsMethodsGetPutDeleteFor("droplets", "droplet")
	})

	Describe("/buildpacks/{guid}", func() {
		BeforeEach(func() { SetUpBuildpackRoutes(router, blobstore) })
		ItSupportsMethodsGetPutDeleteFor("buildpacks", "buildpack")
	})

	Describe("/app_stash", func() {
		BeforeEach(func() {
			SetUpAppStashRoutes(router, blobstore)
		})

		Describe("/app_stash/entries", func() {
			It("Unzips file and copies bits to blobstore", func() {
				zipFile, e := os.Open("test_data/test_archive.zip")
				Expect(e).NotTo(HaveOccurred())
				defer zipFile.Close()

				router.ServeHTTP(responseWriter, newHttpTestPostRequest("/app_stash/entries", map[string]map[string]io.Reader{
					"application": map[string]io.Reader{"application": zipFile},
				}))

				Expect(responseWriter.Code).To(Equal(http.StatusOK), responseWriter.Body.String())
				Expect(entries).To(HaveKeyWithValue("971555ab39d1dfe8dff8b78c2b20e85e01c06595", []byte("1\n\n")))
				Expect(entries).To(HaveKeyWithValue("bbd33de01c17b165b4ce00308e8a19a942023ab8", []byte("2\n\n")))
				Expect(entries).To(HaveKeyWithValue("27cc6f77ee63df90ab3285f9d5fc4ebcb2448c12", []byte("3\n\n")))
			})
		})

		Describe("/app_stash/matches", func() {
			It("returns StatusUnprocessableEntity when body is invalid", func() {
				router.ServeHTTP(responseWriter, httptest.NewRequest(
					"POST", "/app_stash/matches", strings.NewReader("some invalid format")))

				Expect(responseWriter.Code).To(Equal(http.StatusUnprocessableEntity), responseWriter.Body.String())
			})

			It("returns StatusOK and missing fingerprints when body is valid", func() {
				entries["abc"] = []byte("not relevant")

				router.ServeHTTP(responseWriter, httptest.NewRequest(
					"POST", "/app_stash/matches", strings.NewReader("[{\"sha1\":\"abc\"}, {\"sha1\":\"def\"}]")))

				Expect(*responseWriter).To(haveStatusCodeAndBody(
					Equal(http.StatusOK),
					Equal("[{\"sha1\":\"def\"}]")))
			})
		})
	})
})

func haveStatusCodeAndBody(statusCode types.GomegaMatcher, body types.GomegaMatcher) types.GomegaMatcher {
	return MatchFields(IgnoreExtras, Fields{
		"Code": statusCode,
		"Body": WithTransform(func(body *bytes.Buffer) string { return body.String() }, body),
	})
}

// TODO: either remove or add tests that use this function, e.g. tests where blobstore return an error
func writeStatusCodeAndBody(statusCode int, body string) func([]Param) ReturnValues {
	return func(params []Param) ReturnValues {
		for _, param := range params {
			if responseWriter, ok := param.(http.ResponseWriter); ok {
				responseWriter.WriteHeader(statusCode)
				responseWriter.Write([]byte(body))
				return nil
			}
		}
		panic("Unexpected: no ResponseWriter in parameter list.")
	}
}

func newHttpTestPutRequest(path string, formFiles map[string]map[string]io.Reader) *http.Request {
	return newHttpTestRequest("PUT", path, formFiles)
}

func newHttpTestPostRequest(path string, formFiles map[string]map[string]io.Reader) *http.Request {
	return newHttpTestRequest("POST", path, formFiles)
}

func newHttpTestRequest(method string, path string, formFiles map[string]map[string]io.Reader) *http.Request {
	bodyBuf := &bytes.Buffer{}
	header, e := httputil.AddFormFileTo(bodyBuf, formFiles)
	Expect(e).NotTo(HaveOccurred())
	request := httptest.NewRequest(method, path, bodyBuf)
	httputil.AddHeaderTo(request, header)
	return request
}

func AnyReadSeeker() io.ReadSeeker {
	RegisterMatcher(NewAnyMatcher(reflect.TypeOf((*io.ReadSeeker)(nil)).Elem()))
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"fmt"
	"reflect"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/fs/inode"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

type fileSystem struct {
	fuseutil.NotImplementedFileSystem

	/////////////////////////
	// Dependencies
	/////////////////////////

	clock  timeutil.Clock
	bucket gcs.Bucket

	/////////////////////////
	// Mutable state
	/////////////////////////

	// When acquiring this lock, the caller must hold no inode or dir handle
	// locks.
	mu syncutil.InvariantMutex

	// The user and group owning everything in the file system.
	//
	// GUARDED_BY(Mu)
	uid uint32
	gid uint32

	// The collection of live inodes, keyed by inode ID. No ID less than
	// fuse.RootInodeID is ever used.
	//
	// INVARIANT: All values are of type *inode.DirInode or *inode.FileInode
	// INVARIANT: For all keys k, k >= fuse.RootInodeID
	// INVARIANT: inodes[fuse.RootInodeID] is of type *inode.DirInode
	//
	// GUARDED_BY(mu)
	inodes map[fuse.InodeID]interface{}

	// The next inode ID to hand out. We assume that this will never overflow,
	// since even if we were handing out inode IDs at 4 GHz, it would still take
	// over a century to do so.
	//
	// INVARIANT: For all keys k in inodes, k < nextInodeID
	//
	// GUARDED_BY(mu)
	nextInodeID fuse.InodeID

	// An index of all directory inodes by Name().
	//
	// INVARIANT: For each key k, dirNameIndex[k].Name() == k
	//
	// INVARIANT: The values are all and only the values of the inodes map of
	// type *inode.DirHandle.
	//
	// GUARDED_BY(mu)
	dirNameIndex map[string]*inode.DirInode

	// The collection of live handles, keyed by handle ID.
	//
	// INVARIANT: All values are of type *dirHandle
	//
	// GUARDED_BY(mu)
	handles map[fuse.HandleID]interface{}

	// The next handle ID to hand out. We assume that this will never overflow.
	//
	// INVARIANT: For all keys k in handles, k < nextHandleID
	//
	// GUARDED_BY(mu)
	nextHandleID fuse.HandleID
}

// Create a fuse file system whose root directory is the root of the supplied
// bucket. The supplied clock will be used for cache invalidation, modification
// times, etc.
func NewFileSystem(
	clock timeutil.Clock,
	bucket gcs.Bucket) (ffs fuse.FileSystem, err error) {
	// Set up the basic struct.
	fs := &fileSystem{
		clock:        clock,
		bucket:       bucket,
		inodes:       make(map[fuse.InodeID]interface{}),
		nextInodeID:  fuse.RootInodeID + 1,
		dirNameIndex: make(map[string]*inode.DirInode),
		handles:      make(map[fuse.HandleID]interface{}),
	}

	// Set up the root inode.
	root := inode.NewDirInode(bucket, "")
	fs.inodes[fuse.RootInodeID] = root
	fs.dirNameIndex[""] = root

	// Set up invariant checking.
	fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants)

	ffs = fs
	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func isDirName(name string) bool {
	return name == "" || name[len(name)-1] == '/'
}

func (fs *fileSystem) checkInvariants() {
	// Check inode keys.
	for id, _ := range fs.inodes {
		if id < fuse.RootInodeID || id >= fs.nextInodeID {
			panic(fmt.Sprintf("Illegal inode ID: %v", id))
		}
	}

	// Check the root inode.
	_ = fs.inodes[fuse.RootInodeID].(*inode.DirInode)

	// Check each inode, and the indexes over them. Keep a count of each type
	// seen.
	dirsSeen := 0
	filesSeen := 0
	for _, in := range fs.inodes {
		switch typed := in.(type) {
		case *inode.DirInode:
			dirsSeen++
			if fs.dirNameIndex[typed.Name()] != typed {
				panic(fmt.Sprintf("dirNameIndex mismatch: %s", typed.Name()))
			}

		case *inode.FileInode:
			filesSeen++

		default:
			panic(fmt.Sprintf("Unexpected inode type: %v", reflect.TypeOf(in)))
		}
	}

	// Make sure that the indexes are exhaustive.
	if len(fs.dirNameIndex) != dirsSeen {
		panic(
			fmt.Sprintf(
				"dirNameIndex length mismatch: %v vs. %v",
				len(fs.dirNameIndex),
				dirsSeen))
	}

	// Check handles.
	for id, h := range fs.handles {
		if id >= fs.nextHandleID {
			panic(fmt.Sprintf("Illegal handle ID: %v", id))
		}

		_ = h.(*dirHandle)
	}
}

// Find the given inode and return it with its lock held for reading. Panic if
// it doesn't exist or is the wrong type.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// SHARED_LOCK_FUNCTION(in.mu)
func (fs *fileSystem) getDirForReadingOrDie(
	id fuse.InodeID) (in *inode.DirInode) {
	in = fs.inodes[id].(*inode.DirInode)
	in.Mu.RLock()
	return
}

// Get attributes for the given directory inode, fixing up ownership information.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// SHARED_LOCK_FUNCTION(d.mu)
func (fs *fileSystem) getAttributes(
	ctx context.Context,
	d *inode.DirInode) (attrs fuse.InodeAttributes, err error) {
	attrs, err = d.Attributes(ctx)
	if err != nil {
		return
	}

	attrs.Uid = fs.uid
	attrs.Gid = fs.gid

	return
}

// Finish the LookUpInode process for a directory with the given object record.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
func (fs *fileSystem) lookUpDirInode(
	ctx context.Context,
	o *storage.Object) (resp *fuse.LookUpInodeResponse, err error)

// Finish the LookUpInode process for a file with the given object record.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
func (fs *fileSystem) lookUpFileInode(
	ctx context.Context,
	o *storage.Object) (resp *fuse.LookUpInodeResponse, err error)

////////////////////////////////////////////////////////////////////////
// fuse.FileSystem methods
////////////////////////////////////////////////////////////////////////

func (fs *fileSystem) Init(
	ctx context.Context,
	req *fuse.InitRequest) (resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Store the mounting user's info for later.
	fs.uid = req.Header.Uid
	fs.gid = req.Header.Gid

	return
}

func (fs *fileSystem) LookUpInode(
	ctx context.Context,
	req *fuse.LookUpInodeRequest) (resp *fuse.LookUpInodeResponse, err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Find the parent directory in question.
	parent := fs.inodes[req.Parent].(*inode.DirInode)

	// Find a record for the child with the given name.
	o, err := parent.LookUpChild(ctx, req.Name)
	if err != nil {
		err = fmt.Errorf("LookUpChild: %v", err)
		return
	}

	// Is the child a directory or a file?
	if isDirName(o.Name) {
		return fs.lookUpDirInode(ctx, o)
	}

	return fs.lookUpFileInode(ctx, o)
}

func (fs *fileSystem) GetInodeAttributes(
	ctx context.Context,
	req *fuse.GetInodeAttributesRequest) (
	resp *fuse.GetInodeAttributesResponse, err error) {
	resp = &fuse.GetInodeAttributesResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Find the inode.
	in := fs.inodes[req.Inode]

	// Grab its attributes.
	switch typed := in.(type) {
	case *inode.DirInode:
		resp.Attributes, err = fs.getAttributes(ctx, typed)
		if err != nil {
			err = fmt.Errorf("DirInode.Attributes: %v", err)
			return
		}

	default:
		panic(
			fmt.Sprintf(
				"Unknown inode type for ID %v: %v",
				req.Inode,
				reflect.TypeOf(in)))
	}

	return
}

func (fs *fileSystem) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) {
	resp = &fuse.OpenDirResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Make sure the inode still exists and is a directory. If not, something has
	// screwed up because the VFS layer shouldn't have let us forget the inode
	// before opening it.
	in := fs.getDirForReadingOrDie(req.Inode)
	defer in.Mu.RUnlock()

	// Allocate a handle.
	handleID := fs.nextHandleID
	fs.nextHandleID++

	fs.handles[handleID] = newDirHandle(in)
	resp.Handle = handleID

	return
}

func (fs *fileSystem) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) {
	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Find the handle.
	dh := fs.handles[req.Handle].(*dirHandle)
	dh.Mu.Lock()
	defer dh.Mu.Unlock()

	// Serve the request.
	resp, err = dh.ReadDir(ctx, req)

	return
}

func (fs *fileSystem) ReleaseDirHandle(
	ctx context.Context,
	req *fuse.ReleaseDirHandleRequest) (
	resp *fuse.ReleaseDirHandleResponse, err error) {
	resp = &fuse.ReleaseDirHandleResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Sanity check that this handle exists and is of the correct type.
	_ = fs.handles[req.Handle].(*dirHandle)

	// Clear the entry from the map.
	delete(fs.handles, req.Handle)

	return
}
<commit_msg>Added a stub for lookUpFileInode.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"errors"
	"fmt"
	"reflect"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/fs/inode"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

type fileSystem struct {
	fuseutil.NotImplementedFileSystem

	/////////////////////////
	// Dependencies
	/////////////////////////

	clock  timeutil.Clock
	bucket gcs.Bucket

	/////////////////////////
	// Mutable state
	/////////////////////////

	// When acquiring this lock, the caller must hold no inode or dir handle
	// locks.
	mu syncutil.InvariantMutex

	// The user and group owning everything in the file system.
	//
	// GUARDED_BY(Mu)
	uid uint32
	gid uint32

	// The collection of live inodes, keyed by inode ID. No ID less than
	// fuse.RootInodeID is ever used.
	//
	// INVARIANT: All values are of type *inode.DirInode or *inode.FileInode
	// INVARIANT: For all keys k, k >= fuse.RootInodeID
	// INVARIANT: inodes[fuse.RootInodeID] is of type *inode.DirInode
	//
	// GUARDED_BY(mu)
	inodes map[fuse.InodeID]interface{}

	// The next inode ID to hand out. We assume that this will never overflow,
	// since even if we were handing out inode IDs at 4 GHz, it would still take
	// over a century to do so.
	//
	// INVARIANT: For all keys k in inodes, k < nextInodeID
	//
	// GUARDED_BY(mu)
	nextInodeID fuse.InodeID

	// An index of all directory inodes by Name().
	//
	// INVARIANT: For each key k, dirNameIndex[k].Name() == k
	//
	// INVARIANT: The values are all and only the values of the inodes map of
	// type *inode.DirHandle.
	//
	// GUARDED_BY(mu)
	dirNameIndex map[string]*inode.DirInode

	// The collection of live handles, keyed by handle ID.
	//
	// INVARIANT: All values are of type *dirHandle
	//
	// GUARDED_BY(mu)
	handles map[fuse.HandleID]interface{}

	// The next handle ID to hand out. We assume that this will never overflow.
	//
	// INVARIANT: For all keys k in handles, k < nextHandleID
	//
	// GUARDED_BY(mu)
	nextHandleID fuse.HandleID
}

// Create a fuse file system whose root directory is the root of the supplied
// bucket. The supplied clock will be used for cache invalidation, modification
// times, etc.
func NewFileSystem(
	clock timeutil.Clock,
	bucket gcs.Bucket) (ffs fuse.FileSystem, err error) {
	// Set up the basic struct.
	fs := &fileSystem{
		clock:        clock,
		bucket:       bucket,
		inodes:       make(map[fuse.InodeID]interface{}),
		nextInodeID:  fuse.RootInodeID + 1,
		dirNameIndex: make(map[string]*inode.DirInode),
		handles:      make(map[fuse.HandleID]interface{}),
	}

	// Set up the root inode.
	root := inode.NewDirInode(bucket, "")
	fs.inodes[fuse.RootInodeID] = root
	fs.dirNameIndex[""] = root

	// Set up invariant checking.
	fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants)

	ffs = fs
	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func isDirName(name string) bool {
	return name == "" || name[len(name)-1] == '/'
}

func (fs *fileSystem) checkInvariants() {
	// Check inode keys.
	for id, _ := range fs.inodes {
		if id < fuse.RootInodeID || id >= fs.nextInodeID {
			panic(fmt.Sprintf("Illegal inode ID: %v", id))
		}
	}

	// Check the root inode.
	_ = fs.inodes[fuse.RootInodeID].(*inode.DirInode)

	// Check each inode, and the indexes over them. Keep a count of each type
	// seen.
	dirsSeen := 0
	filesSeen := 0
	for _, in := range fs.inodes {
		switch typed := in.(type) {
		case *inode.DirInode:
			dirsSeen++
			if fs.dirNameIndex[typed.Name()] != typed {
				panic(fmt.Sprintf("dirNameIndex mismatch: %s", typed.Name()))
			}

		case *inode.FileInode:
			filesSeen++

		default:
			panic(fmt.Sprintf("Unexpected inode type: %v", reflect.TypeOf(in)))
		}
	}

	// Make sure that the indexes are exhaustive.
	if len(fs.dirNameIndex) != dirsSeen {
		panic(
			fmt.Sprintf(
				"dirNameIndex length mismatch: %v vs. %v",
				len(fs.dirNameIndex),
				dirsSeen))
	}

	// Check handles.
	for id, h := range fs.handles {
		if id >= fs.nextHandleID {
			panic(fmt.Sprintf("Illegal handle ID: %v", id))
		}

		_ = h.(*dirHandle)
	}
}

// Find the given inode and return it with its lock held for reading. Panic if
// it doesn't exist or is the wrong type.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// SHARED_LOCK_FUNCTION(in.mu)
func (fs *fileSystem) getDirForReadingOrDie(
	id fuse.InodeID) (in *inode.DirInode) {
	in = fs.inodes[id].(*inode.DirInode)
	in.Mu.RLock()
	return
}

// Get attributes for the given directory inode, fixing up ownership information.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// SHARED_LOCK_FUNCTION(d.mu)
func (fs *fileSystem) getAttributes(
	ctx context.Context,
	d *inode.DirInode) (attrs fuse.InodeAttributes, err error) {
	attrs, err = d.Attributes(ctx)
	if err != nil {
		return
	}

	attrs.Uid = fs.uid
	attrs.Gid = fs.gid

	return
}

// Finish the LookUpInode process for a directory with the given object record.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
func (fs *fileSystem) lookUpDirInode(
	ctx context.Context,
	o *storage.Object) (resp *fuse.LookUpInodeResponse, err error)

// Finish the LookUpInode process for a file with the given object record.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
func (fs *fileSystem) lookUpFileInode(
	ctx context.Context,
	o *storage.Object) (resp *fuse.LookUpInodeResponse, err error) {
	err = errors.New("TODO(jacobsa): Implement lookUpFileInode.")
	return
}

////////////////////////////////////////////////////////////////////////
// fuse.FileSystem methods
////////////////////////////////////////////////////////////////////////

func (fs *fileSystem) Init(
	ctx context.Context,
	req *fuse.InitRequest) (resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Store the mounting user's info for later.
	fs.uid = req.Header.Uid
	fs.gid = req.Header.Gid

	return
}

func (fs *fileSystem) LookUpInode(
	ctx context.Context,
	req *fuse.LookUpInodeRequest) (resp *fuse.LookUpInodeResponse, err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Find the parent directory in question.
	parent := fs.inodes[req.Parent].(*inode.DirInode)

	// Find a record for the child with the given name.
	o, err := parent.LookUpChild(ctx, req.Name)
	if err != nil {
		err = fmt.Errorf("LookUpChild: %v", err)
		return
	}

	// Is the child a directory or a file?
	if isDirName(o.Name) {
		return fs.lookUpDirInode(ctx, o)
	}

	return fs.lookUpFileInode(ctx, o)
}

func (fs *fileSystem) GetInodeAttributes(
	ctx context.Context,
	req *fuse.GetInodeAttributesRequest) (
	resp *fuse.GetInodeAttributesResponse, err error) {
	resp = &fuse.GetInodeAttributesResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Find the inode.
	in := fs.inodes[req.Inode]

	// Grab its attributes.
	switch typed := in.(type) {
	case *inode.DirInode:
		resp.Attributes, err = fs.getAttributes(ctx, typed)
		if err != nil {
			err = fmt.Errorf("DirInode.Attributes: %v", err)
			return
		}

	default:
		panic(
			fmt.Sprintf(
				"Unknown inode type for ID %v: %v",
				req.Inode,
				reflect.TypeOf(in)))
	}

	return
}

func (fs *fileSystem) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) {
	resp = &fuse.OpenDirResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Make sure the inode still exists and is a directory. If not, something has
	// screwed up because the VFS layer shouldn't have let us forget the inode
	// before opening it.
	in := fs.getDirForReadingOrDie(req.Inode)
	defer in.Mu.RUnlock()

	// Allocate a handle.
	handleID := fs.nextHandleID
	fs.nextHandleID++

	fs.handles[handleID] = newDirHandle(in)
	resp.Handle = handleID

	return
}

func (fs *fileSystem) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) {
	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Find the handle.
	dh := fs.handles[req.Handle].(*dirHandle)
	dh.Mu.Lock()
	defer dh.Mu.Unlock()

	// Serve the request.
	resp, err = dh.ReadDir(ctx, req)

	return
}

func (fs *fileSystem) ReleaseDirHandle(
	ctx context.Context,
	req *fuse.ReleaseDirHandleRequest) (
	resp *fuse.ReleaseDirHandleResponse, err error) {
	resp = &fuse.ReleaseDirHandleResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Sanity check that this handle exists and is of the correct type.
	_ = fs.handles[req.Handle].(*dirHandle)

	// Clear the entry from the map.
	delete(fs.handles, req.Handle)

	return
}
<|endoftext|>