<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Matt Tracy (matt.r.tracy@gmail.com)

package kv

import (
	"bytes"
	"fmt"
	"sync"

	"github.com/biogo/store/llrb"
	"github.com/cockroachdb/cockroach/keys"
	"github.com/cockroachdb/cockroach/proto"
	"github.com/cockroachdb/cockroach/util/cache"
	"github.com/cockroachdb/cockroach/util/log"
)

// rangeCacheKey is the key type used to store and sort values in the
// RangeCache.
type rangeCacheKey proto.Key

// Compare implements the llrb.Comparable interface for rangeCacheKey, so that
// it can be used as a key for util.OrderedCache.
func (a rangeCacheKey) Compare(b llrb.Comparable) int {
	return bytes.Compare(a, b.(rangeCacheKey))
}

// rangeDescriptorDB is a type which can query range descriptors from an
// underlying datastore. This interface is used by rangeDescriptorCache to
// initially retrieve information which will be cached.
type rangeDescriptorDB interface {
	// getRangeDescriptors returns a sorted slice of RangeDescriptors for a set
	// of consecutive ranges, the first of which must contain the requested key.
	// The additional RangeDescriptors are returned with the intent of pre-
	// caching subsequent ranges which are likely to be requested soon by the
	// current workload.
	getRangeDescriptors(proto.Key, lookupOptions) ([]proto.RangeDescriptor, error)
}

// rangeDescriptorCache is used to retrieve range descriptors for
// arbitrary keys. Descriptors are initially queried from storage
// using a rangeDescriptorDB, but is cached for subsequent lookups.
type rangeDescriptorCache struct {
	// rangeDescriptorDB is used to retrieve range descriptors from the
	// database, which will be cached by this structure.
	db rangeDescriptorDB
	// rangeCache caches replica metadata for key ranges. The cache is
	// filled while servicing read and write requests to the key value
	// store.
	rangeCache *cache.OrderedCache
	// rangeCacheMu protects rangeCache for concurrent access
	rangeCacheMu sync.RWMutex
}

// newRangeDescriptorCache returns a new RangeDescriptorCache which
// uses the given rangeDescriptorDB as the underlying source of range
// descriptors.
func newRangeDescriptorCache(db rangeDescriptorDB, size int) *rangeDescriptorCache {
	return &rangeDescriptorCache{
		db: db,
		rangeCache: cache.NewOrderedCache(cache.Config{
			Policy: cache.CacheLRU,
			ShouldEvict: func(n int, k, v interface{}) bool {
				return n > size
			},
		}),
	}
}

func (rmc *rangeDescriptorCache) String() string {
	rmc.rangeCacheMu.RLock()
	defer rmc.rangeCacheMu.RUnlock()
	return rmc.stringLocked()
}

func (rmc *rangeDescriptorCache) stringLocked() string {
	var buf bytes.Buffer
	rmc.rangeCache.Do(func(k, v interface{}) {
		fmt.Fprintf(&buf, "key=%s desc=%+v\n", proto.Key(k.(rangeCacheKey)), v)
	})
	return buf.String()
}

// LookupRangeDescriptor attempts to locate a descriptor for the range
// containing the given Key. This is done by querying the two-level
// lookup table of range descriptors which cockroach maintains.
//
// This method first looks up the specified key in the first level of
// range metadata, which returns the location of the key within the
// second level of range metadata. This second level location is then
// queried to retrieve a descriptor for the range where the key's
// value resides. Range descriptors retrieved during each search are
// cached for subsequent lookups.
//
// This method returns the RangeDescriptor for the range containing
// the key's data, or an error if any occurred.
func (rmc *rangeDescriptorCache) LookupRangeDescriptor(key proto.Key,
	options lookupOptions) (*proto.RangeDescriptor, error) {
	if _, r := rmc.getCachedRangeDescriptor(key); r != nil {
		return r, nil
	}

	if log.V(1) {
		log.Infof("lookup range descriptor: key=%s\n%s", key, rmc)
	}

	rs, err := rmc.db.getRangeDescriptors(key, options)
	if err != nil {
		return nil, err
	}
	rmc.rangeCacheMu.Lock()
	for i := range rs {
		// Note: we append the end key of each range to meta[12] records
		// so that calls to rmc.rangeCache.Ceil() for a key will return
		// the correct range. Using the start key would require using
		// Floor() which is a possibility for our llrb-based OrderedCache
		// but not possible for RocksDB.
		rmc.rangeCache.Add(rangeCacheKey(keys.RangeMetaKey(rs[i].EndKey)), &rs[i])
	}
	if len(rs) == 0 {
		log.Fatalf("no range descriptors returned for %s", key)
	}
	rmc.rangeCacheMu.Unlock()
	return &rs[0], nil
}

// EvictCachedRangeDescriptor will evict any cached range descriptors
// for the given key. It is intended that this method be called from a
// consumer of rangeDescriptorCache if the returned range descriptor is
// discovered to be stale.
// seenDesc should always be passed in and is used as the basis of a
// compare-and-evict (as pointers); if it is nil, eviction is unconditional
// but a warning will be logged.
func (rmc *rangeDescriptorCache) EvictCachedRangeDescriptor(descKey proto.Key, seenDesc *proto.RangeDescriptor) {
	if seenDesc == nil {
		log.Warningf("compare-and-evict for key %s with nil descriptor; clearing unconditionally", descKey)
	}

	rmc.rangeCacheMu.Lock()
	defer rmc.rangeCacheMu.Unlock()

	rngKey, cachedDesc := rmc.getCachedRangeDescriptorLocked(descKey)
	// Note that we're doing a "compare-and-erase": If seenDesc is not nil,
	// we want to clean the cache only if it equals the cached range
	// descriptor as a pointer. If not, then likely some other caller
	// already evicted previously, and we can save work by not doing it
	// again (which would prompt another expensive lookup).
	if seenDesc != nil && seenDesc != cachedDesc {
		return
	}

	for !bytes.Equal(descKey, proto.KeyMin) {
		if log.V(1) {
			log.Infof("evict cached descriptor: key=%s desc=%+v\n%s", descKey, cachedDesc, rmc.stringLocked())
		}
		rmc.rangeCache.Del(rngKey)

		// Retrieve the metadata range key for the next level of metadata, and
		// evict that key as well. This loop ends after the meta1 range, which
		// returns KeyMin as its metadata key.
		descKey = keys.RangeMetaKey(descKey)
		rngKey, cachedDesc = rmc.getCachedRangeDescriptorLocked(descKey)
	}
}

// getCachedRangeDescriptor is a helper function to retrieve the descriptor of
// the range which contains the given key, if present in the cache. It
// acquires a read lock on rmc.rangeCacheMu before delegating to
// getCachedRangeDescriptorLocked.
func (rmc *rangeDescriptorCache) getCachedRangeDescriptor(key proto.Key) (
	rangeCacheKey, *proto.RangeDescriptor) {
	rmc.rangeCacheMu.RLock()
	defer rmc.rangeCacheMu.RUnlock()
	return rmc.getCachedRangeDescriptorLocked(key)
}

// getCachedRangeDescriptorLocked is a helper function to retrieve the
// descriptor of the range which contains the given key, if present in the
// cache. It is assumed that the caller holds a read lock on rmc.rangeCacheMu.
func (rmc *rangeDescriptorCache) getCachedRangeDescriptorLocked(key proto.Key) (
	rangeCacheKey, *proto.RangeDescriptor) {
	// We want to look up the range descriptor for key. The cache is
	// indexed using the end-key of the range, but the end-key is
	// non-inclusive. So we access the cache using key.Next().
	metaKey := keys.RangeMetaKey(key.Next())

	k, v, ok := rmc.rangeCache.Ceil(rangeCacheKey(metaKey))
	if !ok {
		return nil, nil
	}
	metaEndKey := k.(rangeCacheKey)
	rd := v.(*proto.RangeDescriptor)

	// Check that key actually belongs to range
	if !rd.ContainsKey(keys.KeyAddress(key)) {
		return nil, nil
	}
	return metaEndKey, rd
}
<commit_msg>Document the race<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Matt Tracy (matt.r.tracy@gmail.com)

package kv

import (
	"bytes"
	"fmt"
	"sync"

	"github.com/biogo/store/llrb"
	"github.com/cockroachdb/cockroach/keys"
	"github.com/cockroachdb/cockroach/proto"
	"github.com/cockroachdb/cockroach/util/cache"
	"github.com/cockroachdb/cockroach/util/log"
)

// rangeCacheKey is the key type used to store and sort values in the
// RangeCache.
type rangeCacheKey proto.Key

// Compare implements the llrb.Comparable interface for rangeCacheKey, so that
// it can be used as a key for util.OrderedCache.
func (a rangeCacheKey) Compare(b llrb.Comparable) int {
	return bytes.Compare(a, b.(rangeCacheKey))
}

// rangeDescriptorDB is a type which can query range descriptors from an
// underlying datastore. This interface is used by rangeDescriptorCache to
// initially retrieve information which will be cached.
type rangeDescriptorDB interface {
	// getRangeDescriptors returns a sorted slice of RangeDescriptors for a set
	// of consecutive ranges, the first of which must contain the requested key.
	// The additional RangeDescriptors are returned with the intent of pre-
	// caching subsequent ranges which are likely to be requested soon by the
	// current workload.
	getRangeDescriptors(proto.Key, lookupOptions) ([]proto.RangeDescriptor, error)
}

// rangeDescriptorCache is used to retrieve range descriptors for
// arbitrary keys. Descriptors are initially queried from storage
// using a rangeDescriptorDB, but is cached for subsequent lookups.
type rangeDescriptorCache struct {
	// rangeDescriptorDB is used to retrieve range descriptors from the
	// database, which will be cached by this structure.
	db rangeDescriptorDB
	// rangeCache caches replica metadata for key ranges. The cache is
	// filled while servicing read and write requests to the key value
	// store.
	rangeCache *cache.OrderedCache
	// rangeCacheMu protects rangeCache for concurrent access
	rangeCacheMu sync.RWMutex
}

// newRangeDescriptorCache returns a new RangeDescriptorCache which
// uses the given rangeDescriptorDB as the underlying source of range
// descriptors.
func newRangeDescriptorCache(db rangeDescriptorDB, size int) *rangeDescriptorCache {
	return &rangeDescriptorCache{
		db: db,
		rangeCache: cache.NewOrderedCache(cache.Config{
			Policy: cache.CacheLRU,
			ShouldEvict: func(n int, k, v interface{}) bool {
				return n > size
			},
		}),
	}
}

func (rmc *rangeDescriptorCache) String() string {
	rmc.rangeCacheMu.RLock()
	defer rmc.rangeCacheMu.RUnlock()
	return rmc.stringLocked()
}

func (rmc *rangeDescriptorCache) stringLocked() string {
	var buf bytes.Buffer
	rmc.rangeCache.Do(func(k, v interface{}) {
		fmt.Fprintf(&buf, "key=%s desc=%+v\n", proto.Key(k.(rangeCacheKey)), v)
	})
	return buf.String()
}

// LookupRangeDescriptor attempts to locate a descriptor for the range
// containing the given Key. This is done by querying the two-level
// lookup table of range descriptors which cockroach maintains.
//
// This method first looks up the specified key in the first level of
// range metadata, which returns the location of the key within the
// second level of range metadata. This second level location is then
// queried to retrieve a descriptor for the range where the key's
// value resides. Range descriptors retrieved during each search are
// cached for subsequent lookups.
//
// This method returns the RangeDescriptor for the range containing
// the key's data, or an error if any occurred.
func (rmc *rangeDescriptorCache) LookupRangeDescriptor(key proto.Key,
	options lookupOptions) (*proto.RangeDescriptor, error) {
	if _, r := rmc.getCachedRangeDescriptor(key); r != nil {
		return r, nil
	}

	if log.V(1) {
		log.Infof("lookup range descriptor: key=%s\n%s", key, rmc)
	}

	rs, err := rmc.db.getRangeDescriptors(key, options)
	if err != nil {
		return nil, err
	}
	// TODO(tamird): there is a race here; multiple readers may experience cache
	// misses and concurrently attempt to refresh the cache, duplicating work.
	// Locking over the getRangeDescriptors call is even worse though, because
	// that blocks the cache completely for the duration of a slow query to the
	// cluster.
	rmc.rangeCacheMu.Lock()
	for i := range rs {
		// Note: we append the end key of each range to meta[12] records
		// so that calls to rmc.rangeCache.Ceil() for a key will return
		// the correct range. Using the start key would require using
		// Floor() which is a possibility for our llrb-based OrderedCache
		// but not possible for RocksDB.
		rmc.rangeCache.Add(rangeCacheKey(keys.RangeMetaKey(rs[i].EndKey)), &rs[i])
	}
	if len(rs) == 0 {
		log.Fatalf("no range descriptors returned for %s", key)
	}
	rmc.rangeCacheMu.Unlock()
	return &rs[0], nil
}

// EvictCachedRangeDescriptor will evict any cached range descriptors
// for the given key. It is intended that this method be called from a
// consumer of rangeDescriptorCache if the returned range descriptor is
// discovered to be stale.
// seenDesc should always be passed in and is used as the basis of a
// compare-and-evict (as pointers); if it is nil, eviction is unconditional
// but a warning will be logged.
func (rmc *rangeDescriptorCache) EvictCachedRangeDescriptor(descKey proto.Key, seenDesc *proto.RangeDescriptor) {
	if seenDesc == nil {
		log.Warningf("compare-and-evict for key %s with nil descriptor; clearing unconditionally", descKey)
	}

	rmc.rangeCacheMu.Lock()
	defer rmc.rangeCacheMu.Unlock()

	rngKey, cachedDesc := rmc.getCachedRangeDescriptorLocked(descKey)
	// Note that we're doing a "compare-and-erase": If seenDesc is not nil,
	// we want to clean the cache only if it equals the cached range
	// descriptor as a pointer. If not, then likely some other caller
	// already evicted previously, and we can save work by not doing it
	// again (which would prompt another expensive lookup).
	if seenDesc != nil && seenDesc != cachedDesc {
		return
	}

	for !bytes.Equal(descKey, proto.KeyMin) {
		if log.V(1) {
			log.Infof("evict cached descriptor: key=%s desc=%+v\n%s", descKey, cachedDesc, rmc.stringLocked())
		}
		rmc.rangeCache.Del(rngKey)

		// Retrieve the metadata range key for the next level of metadata, and
		// evict that key as well. This loop ends after the meta1 range, which
		// returns KeyMin as its metadata key.
		descKey = keys.RangeMetaKey(descKey)
		rngKey, cachedDesc = rmc.getCachedRangeDescriptorLocked(descKey)
	}
}

// getCachedRangeDescriptor is a helper function to retrieve the descriptor of
// the range which contains the given key, if present in the cache. It
// acquires a read lock on rmc.rangeCacheMu before delegating to
// getCachedRangeDescriptorLocked.
func (rmc *rangeDescriptorCache) getCachedRangeDescriptor(key proto.Key) (
	rangeCacheKey, *proto.RangeDescriptor) {
	rmc.rangeCacheMu.RLock()
	defer rmc.rangeCacheMu.RUnlock()
	return rmc.getCachedRangeDescriptorLocked(key)
}

// getCachedRangeDescriptorLocked is a helper function to retrieve the
// descriptor of the range which contains the given key, if present in the
// cache. It is assumed that the caller holds a read lock on rmc.rangeCacheMu.
func (rmc *rangeDescriptorCache) getCachedRangeDescriptorLocked(key proto.Key) (
	rangeCacheKey, *proto.RangeDescriptor) {
	// We want to look up the range descriptor for key. The cache is
	// indexed using the end-key of the range, but the end-key is
	// non-inclusive. So we access the cache using key.Next().
	metaKey := keys.RangeMetaKey(key.Next())

	k, v, ok := rmc.rangeCache.Ceil(rangeCacheKey(metaKey))
	if !ok {
		return nil, nil
	}
	metaEndKey := k.(rangeCacheKey)
	rd := v.(*proto.RangeDescriptor)

	// Check that key actually belongs to range
	if !rd.ContainsKey(keys.KeyAddress(key)) {
		return nil, nil
	}
	return metaEndKey, rd
}
<|endoftext|>


<commit_before>package shell_test

import (
	"testing"
	"syscall"
	"time"
	"github.com/qadium/plumber/shell"
)

func TestRunAndLog(t *testing.T) {
	err := shell.RunAndLog("true")
	if err != nil {
		t.Error(err)
	}
}

func TestRunAndLogFails(t *testing.T) {
	err := shell.RunAndLog("-notlikely-to-be-a*-cmd")
	if err == nil {
		t.Error("Expected an error but never got one!")
	}
}

func TestInterrupt(t *testing.T) {
	// set the interrupt handler to go off after 1 second
	go func() {
		time.Sleep(50*time.Millisecond)
		syscall.Kill(syscall.Getpid(), syscall.SIGINT)
	}()

	err := shell.RunAndLog("/bin/bash", "-c", "while true; do true; done")
	if err == nil || err.Error() != "signal: interrupt" {
		t.Error("Should've received a SIGINT")
	}
}

func BenchmarkRunAndLog(b *testing.B) {
	for i := 0; i < b.N; i++ {
		shell.RunAndLog("echo", "true")
	}
}
<commit_msg>updated shell test<commit_after>package shell_test

import (
	"testing"
	"syscall"
	"time"
	"github.com/qadium/plumber/shell"
	"log"
	"io/ioutil"
)

func TestRunAndLog(t *testing.T) {
	err := shell.RunAndLog("true")
	if err != nil {
		t.Error(err)
	}
}

func TestRunAndLogFails(t *testing.T) {
	err := shell.RunAndLog("-notlikely-to-be-a*-cmd")
	if err == nil {
		t.Error("Expected an error but never got one!")
	}
}

func TestInterrupt(t *testing.T) {
	// set the interrupt handler to go off after 50 milliseconds
	go func() {
		time.Sleep(50*time.Millisecond)
		syscall.Kill(syscall.Getpid(), syscall.SIGINT)
	}()

	err := shell.RunAndLog("/bin/bash", "-c", "while true; do true; done")
	if err == nil || err.Error() != "signal: interrupt" {
		t.Error("Should've received a SIGINT")
	}
}

func BenchmarkRunAndLog(b *testing.B) {
	log.SetOutput(ioutil.Discard)
	for i := 0; i < b.N; i++ {
		shell.RunAndLog("echo", "true")
	}
}
<|endoftext|>


<commit_before>package raft

import (
	"fmt"
	"net"
	"net/http"
	"sync"
	"testing"
	"time"
)

// Ensure that we can start several servers and have them communicate.
func TestHTTPTransporter(t *testing.T) {
	transporter := NewHTTPTransporter("/raft")
	transporter.DisableKeepAlives = true

	servers := []*Server{}
	f0 := func(server *Server, httpServer *http.Server) {
		// Stop the leader and wait for an election.
		server.Stop()
		time.Sleep(testElectionTimeout * 2)

		if servers[1].State() != Leader && servers[2].State() != Leader {
			t.Fatal("Expected re-election:", servers[1].State(), servers[2].State())
		}
		server.Start()
	}
	f1 := func(server *Server, httpServer *http.Server) {
	}
	f2 := func(server *Server, httpServer *http.Server) {
	}
	runTestHttpServers(t, &servers, transporter, f0, f1, f2)
}

// Starts multiple independent Raft servers wrapped with HTTP servers.
func runTestHttpServers(t *testing.T, servers *[]*Server, transporter *HTTPTransporter, callbacks ...func(*Server, *http.Server)) {
	var wg sync.WaitGroup
	httpServers := []*http.Server{}
	listeners := []net.Listener{}
	for i, _ := range callbacks {
		wg.Add(1)
		port := 9000 + i

		// Create raft server.
		server := newTestServer(fmt.Sprintf("localhost:%d", port), transporter)
		server.SetHeartbeatTimeout(testHeartbeatTimeout)
		server.SetElectionTimeout(testElectionTimeout)
		server.Start()

		defer server.Stop()
		*servers = append(*servers, server)

		// Create listener for HTTP server and start it.
		listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
		if err != nil {
			panic(err)
		}
		defer listener.Close()
		listeners = append(listeners, listener)

		// Create wrapping HTTP server.
		mux := http.NewServeMux()
		transporter.Install(server, mux)
		httpServer := &http.Server{Addr: fmt.Sprintf(":%d", port), Handler: mux}
		httpServers = append(httpServers, httpServer)
		go func() { httpServer.Serve(listener) }()
	}

	// Setup configuration.
	for _, server := range *servers {
		if _, err := (*servers)[0].Do(&DefaultJoinCommand{Name: server.Name()}); err != nil {
			t.Fatalf("Server %s unable to join: %v", server.Name(), err)
		}
	}

	// Wait for configuration to propagate.
	time.Sleep(testHeartbeatTimeout * 2)

	c := make(chan bool)
	start := time.Now()

	for i := 0; i < 1000; i++ {
		go send(c, (*servers)[0])
	}

	for i := 0; i < 1000; i++ {
		<-c
	}
	end := time.Now()
	fmt.Println(end.Sub(start), "commands ", 1000*20)

	// Wait for configuration to propagate.
	time.Sleep(testHeartbeatTimeout * 2)

	// Execute all the callbacks at the same time.
	for _i, _f := range callbacks {
		i, f := _i, _f
		go func() {
			defer wg.Done()
			f((*servers)[i], httpServers[i])
		}()
	}

	// Wait until everything is done.
	wg.Wait()
}

func send(c chan bool, s *Server) {
	for i := 0; i < 20; i++ {
		s.Do(&NOPCommand{})
	}
	c <- true
}
<commit_msg>bench<commit_after>package raft

import (
	"fmt"
	"net"
	"net/http"
	"sync"
	"testing"
	"time"
)

// Ensure that we can start several servers and have them communicate.
func TestHTTPTransporter(t *testing.T) {
	transporter := NewHTTPTransporter("/raft")
	transporter.DisableKeepAlives = true

	servers := []*Server{}
	f0 := func(server *Server, httpServer *http.Server) {
		// Stop the leader and wait for an election.
		server.Stop()
		time.Sleep(testElectionTimeout * 2)

		if servers[1].State() != Leader && servers[2].State() != Leader {
			t.Fatal("Expected re-election:", servers[1].State(), servers[2].State())
		}
		server.Start()
	}
	f1 := func(server *Server, httpServer *http.Server) {
	}
	f2 := func(server *Server, httpServer *http.Server) {
	}
	runTestHttpServers(t, &servers, transporter, f0, f1, f2)
}

// Starts multiple independent Raft servers wrapped with HTTP servers.
func runTestHttpServers(t *testing.T, servers *[]*Server, transporter *HTTPTransporter, callbacks ...func(*Server, *http.Server)) {
	var wg sync.WaitGroup
	httpServers := []*http.Server{}
	listeners := []net.Listener{}
	for i, _ := range callbacks {
		wg.Add(1)
		port := 9000 + i

		// Create raft server.
		server := newTestServer(fmt.Sprintf("localhost:%d", port), transporter)
		server.SetHeartbeatTimeout(testHeartbeatTimeout)
		server.SetElectionTimeout(testElectionTimeout)
		server.Start()

		defer server.Stop()
		*servers = append(*servers, server)

		// Create listener for HTTP server and start it.
		listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
		if err != nil {
			panic(err)
		}
		defer listener.Close()
		listeners = append(listeners, listener)

		// Create wrapping HTTP server.
		mux := http.NewServeMux()
		transporter.Install(server, mux)
		httpServer := &http.Server{Addr: fmt.Sprintf(":%d", port), Handler: mux}
		httpServers = append(httpServers, httpServer)
		go func() { httpServer.Serve(listener) }()
	}

	// Setup configuration.
	for _, server := range *servers {
		if _, err := (*servers)[0].Do(&DefaultJoinCommand{Name: server.Name()}); err != nil {
			t.Fatalf("Server %s unable to join: %v", server.Name(), err)
		}
	}

	// Wait for configuration to propagate.
	time.Sleep(testHeartbeatTimeout * 2)

	// Execute all the callbacks at the same time.
	for _i, _f := range callbacks {
		i, f := _i, _f
		go func() {
			defer wg.Done()
			f((*servers)[i], httpServers[i])
		}()
	}

	// Wait until everything is done.
	wg.Wait()
}

func BenchmarkSpeed(b *testing.B) {

	transporter := NewHTTPTransporter("/raft")
	transporter.DisableKeepAlives = true

	servers := []*Server{}

	for i:= 0; i < 3; i++ {
		port := 9000 + i

		// Create raft server.
		server := newTestServer(fmt.Sprintf("localhost:%d", port), transporter)
		server.SetHeartbeatTimeout(testHeartbeatTimeout)
		server.SetElectionTimeout(testElectionTimeout)
		server.Start()

		defer server.Stop()
		servers = append(servers, server)

		// Create listener for HTTP server and start it.
		listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
		if err != nil {
			panic(err)
		}
		defer listener.Close()

		// Create wrapping HTTP server.
		mux := http.NewServeMux()
		transporter.Install(server, mux)
		httpServer := &http.Server{Addr: fmt.Sprintf(":%d", port), Handler: mux}

		go func() { httpServer.Serve(listener) }()
	}

	// Setup configuration.
	for _, server := range servers {
		(servers)[0].Do(&DefaultJoinCommand{Name: server.Name()})
	}

	c := make(chan bool)

	// Wait for configuration to propagate.
	time.Sleep(2 * time.Second)

	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		for i := 0; i < 1000; i++ {
			go send(c, servers[0])
		}

		for i := 0; i < 1000; i++ {
			<-c
		}
	}
}

func send(c chan bool, s *Server) {
  for i := 0; i < 20; i++ {
    s.Do(&NOPCommand{})
  }
  c <- true
}

<|endoftext|>


<commit_before>package updateBloomData

import (
	"Inf191BloomFilter/src/databaseAccessObj"
	"strconv"

	"github.com/willf/bloom"
)

const bitArraySize = 10000
const numberOfHashFunction = 5

type BloomFilter struct {
	bloomFilter *bloom.BloomFilter
}

func New() *BloomFilter {
	bloomFilter := bloom.New(bitArraySize, numberOfHashFunction)
	return &BloomFilter{bloomFilter}
}

func (bf *BloomFilter) UpdateBloomFilter() {
	// used when more unsubscribed emails have been added to the database
}

func (bf *BloomFilter) RepopulateBloomFilter() {
	// used when unsubscribed emails are removed from the database - resubscribed emails example
	newBloomFilter := bloom.New(bitArraySize, numberOfHashFunction)
	dao := databaseAccessObj.New("bloom:test@/unsubscribed")
	databaseResultMap := dao.SelectAll()
	for key, value := range databaseResultMap {
		for i := range value {
			newBloomFilter.AddString(strconv.Itoa(int(key)) + "_" + value[i])
		}
	}
	bf.bloomFilter = newBloomFilter.Copy()
}
<commit_msg>getArrayOfUserIDEmail function and code cleanup<commit_after>package updateBloomData

import (
	"Inf191BloomFilter/src/databaseAccessObj"
	"strconv"

	"github.com/willf/bloom"
)

const bitArraySize = 10000
const numberOfHashFunction = 5

// BloomFilter struct holds the pointer to the bloomFilter object
type BloomFilter struct {
	bloomFilter *bloom.BloomFilter
}

// New is called to instantiate a new BloomFilter object
func New() *BloomFilter {
	bloomFilter := bloom.New(bitArraySize, numberOfHashFunction)
	return &BloomFilter{bloomFilter}
}

// UpdateBloomFilter is used when more unsubscribed emails have been added to the database
func (bf *BloomFilter) UpdateBloomFilter() {

}

// RepopulateBloomFilter will be called if unsubscribed emails are removed from the
// database (customers resubscribe to emails)
func (bf *BloomFilter) RepopulateBloomFilter() {
	newBloomFilter := bloom.New(bitArraySize, numberOfHashFunction)
	var arrayOfUserIDEmail []string
	arrayOfUserIDEmail = getArrayOfUserIDEmail()
	for i := range arrayOfUserIDEmail {
		newBloomFilter.AddString(arrayOfUserIDEmail[i])
	}
	bf.bloomFilter = newBloomFilter.Copy()
}

// getArrayOfUserIDEmail retrieves all records in the database and returns an array
// of strings in the form of userid_email
func getArrayOfUserIDEmail() []string {
	var arrayOfUserIDEmail []string
	dao := databaseAccessObj.New("bloom:test@/unsubscribed")
	databaseResultMap := dao.SelectAll()
	for key, value := range databaseResultMap {
		for i := range value {
			arrayOfUserIDEmail = append(arrayOfUserIDEmail, strconv.Itoa(int(key))+"_"+value[i])
		}
	}
	dao.CloseConnection()
	return arrayOfUserIDEmail
}
<|endoftext|>


<commit_before>// Copyright 2016 Granitic. All rights reserved.
// Use of this source code is governed by an Apache 2.0 license that can be found in the LICENSE file at the root of this project.

/*
Package httpendpoint provides types that allow web-service handlers to be registered with an HTTP server.

Types in this package represent the interface between the HttpServer facility (which is a thin layer over Go's http.Server) and
the Granitic ws and handler packages that define web-services.

In most cases, user applications will not need to interact with the types in this package. Instead they will define
components of type handler.WsHandler (which already implements the key HttpEndpointProvider interface below) and the
framework will automatically register them with the HttpServer facility.

*/
package httpendpoint

import (
	"golang.org/x/net/context"
	"net/http"
)

// HttpEndPoint associates HTTP methods (GET, POST etc) and path-matching regular expressions with a handler.
type HttpEndPoint struct {
	// A map of HTTP method names to a regular expression pattern (eg GET=^/health-check$)
	MethodPatterns map[string]string

	// A handler implementing Go's built-in http.Handler interface
	Handler http.Handler
}

// Implemented by a component that is able to support a web-service request with a particular path.
type HttpEndpointProvider interface {
	//SupportedHttpMethods returns the HTTP methods (GET, POST, PUT etc) that the endpoint supports.
	SupportedHttpMethods() []string

	// RegexPattern returns an un-compiled regular expression that will be applied to the path element (e.g excluding scheme, domain and query parameters)
	// to potentially match the request to this endpoint.
	RegexPattern() string

	// ServeHTTP handles an HTTP request, including writing normal and abnormal responses. Returns a context that may have
	// been modified.
	ServeHTTP(ctx context.Context, w *HTTPResponseWriter, req *http.Request) context.Context

	// VersionAware returns true if this endpoint supports request version matching.
	VersionAware() bool

	// SupportsVersion returns true if this endpoint supports the version functionality required by the requester. Behaviour is undefined if
	// VersionAware is false. Version matching is application-specific and not defined by Granitic.
	SupportsVersion(version RequiredVersion) bool

	// AutoWireable returns false if this endpoint should not automatically be registered with HTTP servers.
	AutoWireable() bool
}

type RequiredVersion map[string]interface{}

type RequestedVersionExtractor interface {
	Extract(*http.Request) RequiredVersion
}
<commit_msg>GoDoc<commit_after>// Copyright 2016 Granitic. All rights reserved.
// Use of this source code is governed by an Apache 2.0 license that can be found in the LICENSE file at the root of this project.

/*
Package httpendpoint provides types that allow web-service handlers to be registered with an HTTP server.

Types in this package represent the interface between the HttpServer facility (which is a thin layer over Go's http.Server) and
the Granitic ws and handler packages that define web-services.

In most cases, user applications will not need to interact with the types in this package. Instead they will define
components of type handler.WsHandler (which already implements the key HttpEndpointProvider interface below) and the
framework will automatically register them with the HttpServer facility.

*/
package httpendpoint

import (
	"golang.org/x/net/context"
	"net/http"
)

// HttpEndPoint associates HTTP methods (GET, POST etc) and path-matching regular expressions with a handler.
type HttpEndPoint struct {
	// A map of HTTP method names to a regular expression pattern (eg GET=^/health-check$)
	MethodPatterns map[string]string

	// A handler implementing Go's built-in http.Handler interface
	Handler http.Handler
}

// Implemented by a component that is able to support a web-service request with a particular path.
type HttpEndpointProvider interface {
	//SupportedHttpMethods returns the HTTP methods (GET, POST, PUT etc) that the endpoint supports.
	SupportedHttpMethods() []string

	// RegexPattern returns an un-compiled regular expression that will be applied to the path element (e.g excluding scheme, domain and query parameters)
	// to potentially match the request to this endpoint.
	RegexPattern() string

	// ServeHTTP handles an HTTP request, including writing normal and abnormal responses. Returns a context that may have
	// been modified.
	ServeHTTP(ctx context.Context, w *HTTPResponseWriter, req *http.Request) context.Context

	// VersionAware returns true if this endpoint supports request version matching.
	VersionAware() bool

	// SupportsVersion returns true if this endpoint supports the version of functionality required by the requester. Behaviour is undefined if
	// VersionAware is false. Version matching is application-specific and not defined by Granitic.
	SupportsVersion(version RequiredVersion) bool

	// AutoWireable returns false if this endpoint should not automatically be registered with HTTP servers.
	AutoWireable() bool
}

type RequiredVersion map[string]interface{}

type RequestedVersionExtractor interface {
	Extract(*http.Request) RequiredVersion
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	tiff "github.com/garyhouston/tiff66"
	"io/ioutil"
	"log"
	"os"
)

// Decode a TIFF file, then re-encode it and write to a new file.
func main() {
	if len(os.Args) != 3 {
		fmt.Printf("Usage: %s file outfile\n", os.Args[0])
		return
	}
	buf, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	valid, order, ifdPos := tiff.GetHeader(buf)
	if !valid {
		log.Fatal("Not a valid TIFF file")
	}
	root, err := tiff.GetIFDTree(buf, order, ifdPos, tiff.TIFFSpace)
	if err != nil {
		log.Fatal(err)
	}
	root.Fix(order)
	headerSize := tiff.HeaderSize()
	fileSize := headerSize + root.TreeSize()
	out := make([]byte, fileSize)
	tiff.PutHeader(out, order, headerSize)
	next, err := root.PutIFDTree(out, headerSize, order)
	if err != nil {
		log.Fatal(err)
	}
	out = out[:next]
	ioutil.WriteFile(os.Args[2], out, 0644)
}
<commit_msg>HeaderSize now const<commit_after>package main

import (
	"fmt"
	tiff "github.com/garyhouston/tiff66"
	"io/ioutil"
	"log"
	"os"
)

// Decode a TIFF file, then re-encode it and write to a new file.
func main() {
	if len(os.Args) != 3 {
		fmt.Printf("Usage: %s file outfile\n", os.Args[0])
		return
	}
	buf, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	valid, order, ifdPos := tiff.GetHeader(buf)
	if !valid {
		log.Fatal("Not a valid TIFF file")
	}
	root, err := tiff.GetIFDTree(buf, order, ifdPos, tiff.TIFFSpace)
	if err != nil {
		log.Fatal(err)
	}
	root.Fix(order)
	fileSize := tiff.HeaderSize + root.TreeSize()
	out := make([]byte, fileSize)
	tiff.PutHeader(out, order, tiff.HeaderSize)
	next, err := root.PutIFDTree(out, tiff.HeaderSize, order)
	if err != nil {
		log.Fatal(err)
	}
	out = out[:next]
	ioutil.WriteFile(os.Args[2], out, 0644)
}
<|endoftext|>


<commit_before>package run

import (
	"fmt"
	"io"
)

type readWriteKiller struct {
	io.Reader
	io.Writer
	kill func() error
}

type read struct {
	buf []byte
	err error
}

type opCode uint16

const (
	opCodeNone = opCode(iota)
	opCodeOrigin
	opCodeServices
	opCodeMessage
)

type opFlags uint16

const (
	opFlagPollout = opFlags(0x1)

	opFlagsMask = opFlagPollout
)

const (
	evCodePollout = uint16(iota)
	evCodeOrigin
	evCodeServices
	evCodeMessage
)

func ioLoop(origin io.ReadWriter, services ServiceRegistry, subject readWriteKiller) (err error) {
	originInput := originReadLoop(origin)
	defer func() {
		go func() {
			for range originInput {
			}
		}()
	}()

	messageInput := make(chan []byte)
	messenger := services.Messenger(messageInput)
	defer func() {
		for range messageInput {
		}
	}()
	defer func() {
		go messenger.Shutdown()
	}()

	subjectInput := subjectReadLoop(subject)
	defer func() {
		for range subjectInput {
		}
	}()

	subjectOutput, subjectOutputEnd := subjectWriteLoop(subject)
	defer func() {
		<-subjectOutputEnd
	}()
	defer close(subjectOutput)

	defer subject.kill()

	var (
		pendingEvs   [][]byte
		pendingPolls uint64
	)

	for {
		var (
			doEv            []byte
			doOriginInput   <-chan read
			doMessageInput  <-chan []byte
			doSubjectInput  <-chan read
			doSubjectOutput chan<- []byte
		)

		if len(pendingEvs) > 0 {
			doEv = pendingEvs[0]
		} else if pendingPolls > 0 {
			doEv = makePolloutEv(pendingPolls)
		}

		if doEv == nil {
			doOriginInput = originInput
			doMessageInput = messageInput
			doSubjectInput = subjectInput
		} else {
			doSubjectOutput = subjectOutput
		}

		select {
		case read, ok := <-doOriginInput:
			if !ok {
				originInput = nil
				break
			}
			if read.err != nil {
				err = read.err
				return
			}

			pendingEvs = append(pendingEvs, read.buf)

		case buf := <-doMessageInput:
			pendingEvs = append(pendingEvs, initMessageEv(buf))

		case read, ok := <-doSubjectInput:
			if !ok {
				return
			}
			if read.err != nil {
				err = read.err
				return
			}

			ev, poll, opErr := handleOp(read.buf, origin, services, messenger)
			if opErr != nil {
				err = opErr
				return
			}

			if ev != nil {
				pendingEvs = append(pendingEvs, ev)
			}
			if poll {
				pendingPolls++
			}

		case doSubjectOutput <- doEv:
			if len(pendingEvs) > 0 {
				pendingEvs = pendingEvs[1:]
			} else {
				pendingPolls = 0
			}

		case <-subjectOutputEnd:
			return
		}
	}
}

func originReadLoop(r io.Reader) <-chan read {
	reads := make(chan read)

	go func() {
		defer close(reads)

		for {
			buf := make([]byte, maxPacketSize)
			n, err := r.Read(buf[headerSize:])
			buf = buf[:headerSize+n]
			endian.PutUint32(buf[0:], uint32(len(buf)))
			endian.PutUint16(buf[4:], evCodeOrigin)
			reads <- read{buf: buf}

			if err != nil {
				if err == io.EOF {
					if n != 0 {
						buf := make([]byte, headerSize)
						endian.PutUint32(buf[0:], headerSize)
						endian.PutUint16(buf[4:], evCodeOrigin)
						reads <- read{buf: buf}
					}
				} else {
					reads <- read{err: fmt.Errorf("origin read: %v", err)}
				}
				return
			}
		}
	}()

	return reads
}

func subjectReadLoop(r io.Reader) <-chan read {
	reads := make(chan read)

	go func() {
		defer close(reads)

		header := make([]byte, headerSize)

		for {
			if _, err := io.ReadFull(r, header); err != nil {
				if err != io.EOF {
					reads <- read{err: fmt.Errorf("subject read: %v", err)}
				}
				return
			}

			size := endian.Uint32(header)
			if size < headerSize || size > maxPacketSize {
				reads <- read{err: fmt.Errorf("invalid op packet size: %d", size)}
				return
			}

			buf := make([]byte, size)
			copy(buf, header)

			if _, err := io.ReadFull(r, buf[headerSize:]); err != nil {
				reads <- read{err: fmt.Errorf("subject read: %v", err)}
				return
			}

			reads <- read{buf: buf}
		}
	}()

	return reads
}

func subjectWriteLoop(w io.Writer) (chan<- []byte, <-chan struct{}) {
	writes := make(chan []byte)
	end := make(chan struct{})

	go func() {
		defer close(end)

		for buf := range writes {
			if _, err := w.Write(buf); err != nil {
				return
			}
		}
	}()

	return writes, end
}

func handleOp(op []byte, origin io.ReadWriter, services ServiceRegistry, messenger Messenger) (ev []byte, poll bool, err error) {
	var (
		code  = opCode(endian.Uint16(op[4:]))
		flags = opFlags(endian.Uint16(op[6:]))
	)

	if (flags &^ opFlagsMask) != 0 {
		err = fmt.Errorf("invalid op packet flags: 0x%x", flags)
		return
	}

	poll = (flags & opFlagPollout) != 0

	switch code {
	case opCodeNone:

	case opCodeOrigin:
		_, err = origin.Write(op[headerSize:])
		if err != nil {
			err = fmt.Errorf("origin write: %v", err)
		}

	case opCodeServices:
		ev, err = handleServicesOp(op, services)

	case opCodeMessage:
		err = handleMessageOp(op, messenger)

	default:
		err = fmt.Errorf("invalid op packet code: %d", code)
	}

	return
}

func makePolloutEv(count uint64) (ev []byte) {
	const size = headerSize + 8

	ev = make([]byte, size)
	endian.PutUint32(ev[0:], size)
	endian.PutUint16(ev[4:], evCodePollout)
	endian.PutUint64(ev[8:], count)

	return
}
<commit_msg>run: buffered message channel<commit_after>package run

import (
	"fmt"
	"io"
)

type readWriteKiller struct {
	io.Reader
	io.Writer
	kill func() error
}

type read struct {
	buf []byte
	err error
}

type opCode uint16

const (
	opCodeNone = opCode(iota)
	opCodeOrigin
	opCodeServices
	opCodeMessage
)

type opFlags uint16

const (
	opFlagPollout = opFlags(0x1)

	opFlagsMask = opFlagPollout
)

const (
	evCodePollout = uint16(iota)
	evCodeOrigin
	evCodeServices
	evCodeMessage
)

func ioLoop(origin io.ReadWriter, services ServiceRegistry, subject readWriteKiller) (err error) {
	originInput := originReadLoop(origin)
	defer func() {
		go func() {
			for range originInput {
			}
		}()
	}()

	messageInput := make(chan []byte, 1) // service may send a reply message synchronously
	messenger := services.Messenger(messageInput)
	defer func() {
		for range messageInput {
		}
	}()
	defer func() {
		go messenger.Shutdown()
	}()

	subjectInput := subjectReadLoop(subject)
	defer func() {
		for range subjectInput {
		}
	}()

	subjectOutput, subjectOutputEnd := subjectWriteLoop(subject)
	defer func() {
		<-subjectOutputEnd
	}()
	defer close(subjectOutput)

	defer subject.kill()

	var (
		pendingEvs   [][]byte
		pendingPolls uint64
	)

	for {
		var (
			doEv            []byte
			doOriginInput   <-chan read
			doMessageInput  <-chan []byte
			doSubjectInput  <-chan read
			doSubjectOutput chan<- []byte
		)

		if len(pendingEvs) > 0 {
			doEv = pendingEvs[0]
		} else if pendingPolls > 0 {
			doEv = makePolloutEv(pendingPolls)
		}

		if doEv == nil {
			doOriginInput = originInput
			doMessageInput = messageInput
			doSubjectInput = subjectInput
		} else {
			doSubjectOutput = subjectOutput
		}

		select {
		case read, ok := <-doOriginInput:
			if !ok {
				originInput = nil
				break
			}
			if read.err != nil {
				err = read.err
				return
			}

			pendingEvs = append(pendingEvs, read.buf)

		case buf := <-doMessageInput:
			pendingEvs = append(pendingEvs, initMessageEv(buf))

		case read, ok := <-doSubjectInput:
			if !ok {
				return
			}
			if read.err != nil {
				err = read.err
				return
			}

			ev, poll, opErr := handleOp(read.buf, origin, services, messenger)
			if opErr != nil {
				err = opErr
				return
			}

			if ev != nil {
				pendingEvs = append(pendingEvs, ev)
			}
			if poll {
				pendingPolls++
			}

		case doSubjectOutput <- doEv:
			if len(pendingEvs) > 0 {
				pendingEvs = pendingEvs[1:]
			} else {
				pendingPolls = 0
			}

		case <-subjectOutputEnd:
			return
		}
	}
}

func originReadLoop(r io.Reader) <-chan read {
	reads := make(chan read)

	go func() {
		defer close(reads)

		for {
			buf := make([]byte, maxPacketSize)
			n, err := r.Read(buf[headerSize:])
			buf = buf[:headerSize+n]
			endian.PutUint32(buf[0:], uint32(len(buf)))
			endian.PutUint16(buf[4:], evCodeOrigin)
			reads <- read{buf: buf}

			if err != nil {
				if err == io.EOF {
					if n != 0 {
						buf := make([]byte, headerSize)
						endian.PutUint32(buf[0:], headerSize)
						endian.PutUint16(buf[4:], evCodeOrigin)
						reads <- read{buf: buf}
					}
				} else {
					reads <- read{err: fmt.Errorf("origin read: %v", err)}
				}
				return
			}
		}
	}()

	return reads
}

func subjectReadLoop(r io.Reader) <-chan read {
	reads := make(chan read)

	go func() {
		defer close(reads)

		header := make([]byte, headerSize)

		for {
			if _, err := io.ReadFull(r, header); err != nil {
				if err != io.EOF {
					reads <- read{err: fmt.Errorf("subject read: %v", err)}
				}
				return
			}

			size := endian.Uint32(header)
			if size < headerSize || size > maxPacketSize {
				reads <- read{err: fmt.Errorf("invalid op packet size: %d", size)}
				return
			}

			buf := make([]byte, size)
			copy(buf, header)

			if _, err := io.ReadFull(r, buf[headerSize:]); err != nil {
				reads <- read{err: fmt.Errorf("subject read: %v", err)}
				return
			}

			reads <- read{buf: buf}
		}
	}()

	return reads
}

func subjectWriteLoop(w io.Writer) (chan<- []byte, <-chan struct{}) {
	writes := make(chan []byte)
	end := make(chan struct{})

	go func() {
		defer close(end)

		for buf := range writes {
			if _, err := w.Write(buf); err != nil {
				return
			}
		}
	}()

	return writes, end
}

func handleOp(op []byte, origin io.ReadWriter, services ServiceRegistry, messenger Messenger) (ev []byte, poll bool, err error) {
	var (
		code  = opCode(endian.Uint16(op[4:]))
		flags = opFlags(endian.Uint16(op[6:]))
	)

	if (flags &^ opFlagsMask) != 0 {
		err = fmt.Errorf("invalid op packet flags: 0x%x", flags)
		return
	}

	poll = (flags & opFlagPollout) != 0

	switch code {
	case opCodeNone:

	case opCodeOrigin:
		_, err = origin.Write(op[headerSize:])
		if err != nil {
			err = fmt.Errorf("origin write: %v", err)
		}

	case opCodeServices:
		ev, err = handleServicesOp(op, services)

	case opCodeMessage:
		err = handleMessageOp(op, messenger)

	default:
		err = fmt.Errorf("invalid op packet code: %d", code)
	}

	return
}

func makePolloutEv(count uint64) (ev []byte) {
	const size = headerSize + 8

	ev = make([]byte, size)
	endian.PutUint32(ev[0:], size)
	endian.PutUint16(ev[4:], evCodePollout)
	endian.PutUint64(ev[8:], count)

	return
}
<|endoftext|>


<commit_before>package memory

import "time"

// TimeLimiter limits the rate of a set of serial operations.
// It does this by tracking how much time has been spent (updated via Add()),
// and comparing this to the window size and the limit, slowing down further operations as soon
// as one Add() is called informing it the per-window allowed budget has been exceeded.
// Limitations:
// * the last operation is allowed to exceed the budget (but the next call will be delayed to compensate)
// * concurrency is not supported
//
// For correctness, you should always follow up an Add() with a Wait()
type TimeLimiter struct {
	since     time.Time
	next      time.Time
	timeSpent time.Duration
	window    time.Duration
	limit     time.Duration
}

// NewTimeLimiter creates a new TimeLimiter.
func NewTimeLimiter(window, limit time.Duration, now time.Time) *TimeLimiter {
	l := TimeLimiter{
		since:  now,
		next:   now.Add(window),
		window: window,
		limit:  limit,
	}
	return &l
}

// Add increments the "time spent" counter by "d"
func (l *TimeLimiter) Add(d time.Duration) {
	l.add(time.Now(), d)
}

// add increments the "time spent" counter by "d" at a given time
func (l *TimeLimiter) add(now time.Time, d time.Duration) {
	if now.After(l.next) {
		l.timeSpent = d
		l.since = now.Add(-d)
		l.next = l.since.Add(l.window)
		return
	}
	l.timeSpent += d
}

// Wait returns when we are not rate limited
// * if we passed the window, we reset everything (this is only safe for callers
//     that behave correctly, i.e. that wait the instructed time after each add)
// * if limit is not reached, no sleep is needed
// * if limit has been exceeded, sleep until next period + extra multiple to compensate
//    this is perhaps best explained with an example:
//    if window is 1s and limit 100ms, but we spent 250ms, then we spent effectively 2.5 seconds worth of work.
//    let's say we are 800ms into the 1s window, that means we should sleep 2500-800 = 1.7s
//    in order to maximize work while honoring the imposed limit.
// * if limit has been met exactly, sleep until next period (this is a special case of the above)
func (l *TimeLimiter) Wait() {
	time.Sleep(l.wait(time.Now()))
}

// wait returns how long should be slept at a given time. See Wait() for more info
func (l *TimeLimiter) wait(now time.Time) time.Duration {

	// if we passed the window, reset and start over
	// if clock is adjusted backwards, best we can do is also just reset and start over
	if now.After(l.next) || now.Before(l.since) {
		l.timeSpent = 0
		l.since = now
		l.next = now.Add(l.window)
		return 0
	}
	if l.timeSpent < l.limit {
		return 0
	}

	// here we know that:
	// since <= now <= next
	// timespent >= limit
	multiplier := l.window / l.limit
	timeToPass := l.timeSpent * multiplier
	timePassed := now.Sub(l.since)

	// not sure if this should happen, but let's be safe anyway
	if timePassed > timeToPass {
		return 0
	}
	return timeToPass - timePassed
}
<commit_msg>fix cases where window/limit is not a round number<commit_after>package memory

import "time"

// TimeLimiter limits the rate of a set of serial operations.
// It does this by tracking how much time has been spent (updated via Add()),
// and comparing this to the window size and the limit, slowing down further operations as soon
// as one Add() is called informing it the per-window allowed budget has been exceeded.
// Limitations:
// * the last operation is allowed to exceed the budget (but the next call will be delayed to compensate)
// * concurrency is not supported
//
// For correctness, you should always follow up an Add() with a Wait()
type TimeLimiter struct {
	since     time.Time
	next      time.Time
	timeSpent time.Duration
	window    time.Duration
	limit     time.Duration
	factor    float64
}

// NewTimeLimiter creates a new TimeLimiter.
func NewTimeLimiter(window, limit time.Duration, now time.Time) *TimeLimiter {
	l := TimeLimiter{
		since:  now,
		next:   now.Add(window),
		window: window,
		limit:  limit,
		factor: float64(window) / float64(limit),
	}
	return &l
}

// Add increments the "time spent" counter by "d"
func (l *TimeLimiter) Add(d time.Duration) {
	l.add(time.Now(), d)
}

// add increments the "time spent" counter by "d" at a given time
func (l *TimeLimiter) add(now time.Time, d time.Duration) {
	if now.After(l.next) {
		l.timeSpent = d
		l.since = now.Add(-d)
		l.next = l.since.Add(l.window)
		return
	}
	l.timeSpent += d
}

// Wait returns when we are not rate limited
// * if we passed the window, we reset everything (this is only safe for callers
//     that behave correctly, i.e. that wait the instructed time after each add)
// * if limit is not reached, no sleep is needed
// * if limit has been exceeded, sleep until next period + extra multiple to compensate
//    this is perhaps best explained with an example:
//    if window is 1s and limit 100ms, but we spent 250ms, then we spent effectively 2.5 seconds worth of work.
//    let's say we are 800ms into the 1s window, that means we should sleep 2500-800 = 1.7s
//    in order to maximize work while honoring the imposed limit.
// * if limit has been met exactly, sleep until next period (this is a special case of the above)
func (l *TimeLimiter) Wait() {
	time.Sleep(l.wait(time.Now()))
}

// wait returns how long should be slept at a given time. See Wait() for more info
func (l *TimeLimiter) wait(now time.Time) time.Duration {

	// if we passed the window, reset and start over
	// if clock is adjusted backwards, best we can do is also just reset and start over
	if now.After(l.next) || now.Before(l.since) {
		l.timeSpent = 0
		l.since = now
		l.next = now.Add(l.window)
		return 0
	}
	if l.timeSpent < l.limit {
		return 0
	}

	// here we know that:
	// since <= now <= next
	// timespent >= limit
	timeToPass := time.Duration(float64(l.timeSpent) * l.factor)
	timePassed := now.Sub(l.since)

	// not sure if this should happen, but let's be safe anyway
	if timePassed > timeToPass {
		return 0
	}
	return timeToPass - timePassed
}
<|endoftext|>


<commit_before>/*Package consts implements constants for the entire project
 */
package consts

const DefaultUserName = "Thor"

// VisibleFlag is the constant given for a visible repository
const VisibleFlag = "VISIBLE"

// HiddenFlag is the constant given for an hidden repository
const HiddenFlag = "HIDDEN"

// ConfigurationFileName is the configuration file name of Goyave
const ConfigurationFileName = ".goyave"

// GitFileName is the name of the git directory, in a git repository
const GitFileName = ".git"

////////////////
//// ERRORS ////
////////////////

// RepositoryAlreadyExists is an error that raises when an existing path repository is in a list
const RepositoryAlreadyExists = "REPOSITORY_ALREADY_EXISTS"

// ItemIsNotIsSlice is an error that raises when an searched item is not in the given slice
const ItemIsNotInSlice = "ITEM_IS_NOT_IN_SLICE"
<commit_msg>Add comment and remove code<commit_after>/*Package consts implements constants for the entire project
 */
package consts

// DefaultUserName is a constant to define a new user, if the
// user local name can't be found
const DefaultUserName = "Thor"

// VisibleFlag is the constant given for a visible repository
const VisibleFlag = "VISIBLE"

// HiddenFlag is the constant given for an hidden repository
const HiddenFlag = "HIDDEN"

// ConfigurationFileName is the configuration file name of Goyave
const ConfigurationFileName = ".goyave"

// GitFileName is the name of the git directory, in a git repository
const GitFileName = ".git"
<|endoftext|>


<commit_before>package beanstalk

import "sync"

// ConsumerPool maintains a pool of Consumer objects.
type ConsumerPool struct {
	C         chan *Job
	consumers []*Consumer
	sync.Mutex
}

// NewConsumerPool creates a pool of Consumer objects.
func NewConsumerPool(sockets []string, tubes []string, options *Options) *ConsumerPool {
	pool := &ConsumerPool{C: make(chan *Job)}

	for _, socket := range sockets {
		pool.consumers = append(pool.consumers, NewConsumer(socket, tubes, pool.C, options))
	}

	return pool
}

// Stop shuts down all the consumers in the pool.
func (pool *ConsumerPool) Stop() {
	pool.Lock()
	defer pool.Unlock()

	for i, consumer := range pool.consumers {
		consumer.Stop()
		pool.consumers[i] = nil
	}
	pool.consumers = []*Consumer{}
}

// Play tells all the consumers to start reservering jobs.
func (pool *ConsumerPool) Play() {
	pool.Lock()
	defer pool.Unlock()

	for _, consumer := range pool.consumers {
		consumer.Play()
	}
}

// Pause tells all the consumer to stop reservering jobs.
func (pool *ConsumerPool) Pause() {
	pool.Lock()
	defer pool.Unlock()

	for _, consumer := range pool.consumers {
		consumer.Pause()
	}
}
<commit_msg>Change the exposed channel to receive-only<commit_after>package beanstalk

import "sync"

// ConsumerPool maintains a pool of Consumer objects.
type ConsumerPool struct {
	// The channel on which newly reserved jobs are offered.
	C <-chan *Job

	c         chan *Job
	consumers []*Consumer
	sync.Mutex
}

// NewConsumerPool creates a pool of Consumer objects.
func NewConsumerPool(sockets []string, tubes []string, options *Options) *ConsumerPool {
	c := make(chan *Job)
	pool := &ConsumerPool{C: c, c: c}

	for _, socket := range sockets {
		pool.consumers = append(pool.consumers, NewConsumer(socket, tubes, pool.c, options))
	}

	return pool
}

// Stop shuts down all the consumers in the pool.
func (pool *ConsumerPool) Stop() {
	pool.Lock()
	defer pool.Unlock()

	for i, consumer := range pool.consumers {
		consumer.Stop()
		pool.consumers[i] = nil
	}
	pool.consumers = []*Consumer{}
}

// Play tells all the consumers to start reservering jobs.
func (pool *ConsumerPool) Play() {
	pool.Lock()
	defer pool.Unlock()

	for _, consumer := range pool.consumers {
		consumer.Play()
	}
}

// Pause tells all the consumer to stop reservering jobs.
func (pool *ConsumerPool) Pause() {
	pool.Lock()
	defer pool.Unlock()

	for _, consumer := range pool.consumers {
		consumer.Pause()
	}
}
<|endoftext|>


<commit_before>package main

import(
    "flag"
    "os"
    "fmt"
    "io/ioutil"
    "strings"
    "image/png"
    "path"
    )
func main() {
  if len(os.Args)==1{
    fmt.Fprintf(os.Stderr,"wrong number of arguments, expected at least a file or folder path\n")
      os.Exit(1)
  }
onlydata := flag.Bool("t",false,"only display data with tabs")
            flag.Parse()
            scanDir := os.Args[len(os.Args)-1]
            stats, err := os.Stat(scanDir)
            if err != nil{
              fmt.Fprintf(os.Stderr,"Error, the path %s is not accessible\n",scanDir)
                os.Exit(2)
            }
          if(!*onlydata && stats.IsDir()){
            fmt.Printf("Listing image files in %s\n",scanDir)
          }

          if(!*onlydata && !stats.IsDir()){
            fmt.Printf("Examining file %s\n",scanDir)
          }
          files, _ := ioutil.ReadDir(scanDir)
            for _, f := range files {
              ParseFile(scanDir,f,onlydata)
            }
}

func ParseFile(scanDir string,f os.FileInfo,onlydata *bool){
  if strings.HasSuffix(strings.ToLower(f.Name()),"png"){
    if(!*onlydata){fmt.Printf("found PNG file %s\n",f.Name())}
    imReader,_ := os.Open(path.Join(scanDir,f.Name()))
      thisImg, err := png.Decode(imReader)
      if(err!=nil){
        fmt.Fprintf(os.Stderr,"Error while decoding image %s : %s\n",path.Join(scanDir,f.Name()),err)
          return
      }

bounds := thisImg.Bounds()
          sumR,sumG,sumB := 0., 0., 0.
          totPixels, totNonTransparent := 0, 0
          for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
            for x := bounds.Min.X; x < bounds.Max.X; x++ {
              r,g,b,a := thisImg.At(x, y).RGBA()
                totPixels++
                if a!=65535{
                  //fmt.Printf("found channel alpha %d %d %d %d at %d %d\n",a,r,g,b,x,y)
                  continue
                }
              totNonTransparent++
                sumR+=float64(r)
                sumG+=float64(g)
                sumB+=float64(b)
            }
          }
        if(!*onlydata){
          fmt.Printf("found %d non transparent pixels of %d total, (%d%%)\n",totNonTransparent,totPixels,(totNonTransparent*100)/totPixels)
            fmt.Printf("average RGB values: %f %f %f\n",sumR/float64(totNonTransparent),sumG/float64(totNonTransparent),sumB/float64(totNonTransparent))
        } else
        {
          fmt.Printf("%s\t%f\t%f\t%f\n",path.Join(scanDir,f.Name()),sumR/float64(totNonTransparent),sumG/float64(totNonTransparent),sumB/float64(totNonTransparent))
        }
  }

}
<commit_msg>manage file and dir paths with proper output messages<commit_after>package main

import(
    "flag"
    "os"
    "fmt"
    "io/ioutil"
    "strings"
    "image/png"
    "path"
    )
func main() {
  if len(os.Args)==1{
    fmt.Fprintf(os.Stderr,"wrong number of arguments, expected at least a file or folder path\n")
      os.Exit(1)
  }
onlydata := flag.Bool("t",false,"only display data with tabs")
            flag.Parse()
            scanDir := os.Args[len(os.Args)-1]
            stats, err := os.Stat(scanDir)
            if err != nil{
              fmt.Fprintf(os.Stderr,"Error, the path %s is not accessible\n",scanDir)
                os.Exit(2)
            }
          if(!*onlydata && stats.IsDir()){
            fmt.Printf("Listing image files in %s\n",scanDir)
          }

          if(!*onlydata && !stats.IsDir()){
            fmt.Printf("examining target %s...\n",scanDir)
          }
          if stats.IsDir() {
            files, _ := ioutil.ReadDir(scanDir)
              for _, f := range files {
                ParseFile(path.Join(scanDir,f.Name()),onlydata)
              }
          }else{
            ParseFile(scanDir,onlydata)
          }
}

func ParseFile(fpath string,onlydata *bool){
  if strings.HasSuffix(strings.ToLower(fpath),"png"){
    if(!*onlydata){fmt.Printf("found PNG file %s\n",fpath)}
    imReader,_ := os.Open(fpath)
      thisImg, err := png.Decode(imReader)
      if(err!=nil){
        fmt.Fprintf(os.Stderr,"Error while decoding image %s : %s\n",fpath,err)
          return
      }

bounds := thisImg.Bounds()
          sumR,sumG,sumB := 0., 0., 0.
          totPixels, totNonTransparent := 0, 0
          for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
            for x := bounds.Min.X; x < bounds.Max.X; x++ {
              r,g,b,a := thisImg.At(x, y).RGBA()
                totPixels++
                if a!=65535{
                  //fmt.Printf("found channel alpha %d %d %d %d at %d %d\n",a,r,g,b,x,y)
                  continue
                }
              totNonTransparent++
                sumR+=float64(r)
                sumG+=float64(g)
                sumB+=float64(b)
            }
          }
        if(!*onlydata){
          fmt.Printf("found %d non transparent pixels of %d total, (%d%%)\n",totNonTransparent,totPixels,(totNonTransparent*100)/totPixels)
            fmt.Printf("average RGB values: %f %f %f\n",sumR/float64(totNonTransparent),sumG/float64(totNonTransparent),sumB/float64(totNonTransparent))
        } else
        {
          fmt.Printf("%s\t%f\t%f\t%f\n",fpath,sumR/float64(totNonTransparent),sumG/float64(totNonTransparent),sumB/float64(totNonTransparent))
        }
  }

}
<|endoftext|>


<commit_before>package gautomator

import (
	"github.com/gonum/matrix/mat64" // Matrix
	"log"
	"math/rand" // Temp
	//"strconv"
	"sync"
	"time"
)

func random(min int, max int) int {
	var bytes int
	bytes = min + rand.Intn(max)
	return int(bytes)
	//rand.Seed(time.Now().UTC().UnixNano())
	//return rand.Intn(max - min) + min
}

// Ths runner goroutine is a goroutinei which:
// Consume the TaskGraphStructure from a channel
// run the task given as arguments if the deps are done
// Post the task to the doncChannel once done
//
func Runner(task *Task, doneChan chan<- *Task, wg *sync.WaitGroup) {
	log.Printf("[%v:%v] Queued", task.Id, task.Name)
	for {
		letsGo := <-task.TaskCanRunChan
		// For each dependency of the task
		// We can run if the sum of the element of the column Id of the current task is 0

		if letsGo == true {
			proto := "tcp"
			socket := task.Node
			//sleepTime := random(1, 5)
			// Stupid trick to make shell works... A Shell module will be implemented later"
			if task.Module == "shell" {
				task.Module = "echo"
				n := len(task.Args)
				task.Args[n] = "|"
				task.Args[n+1] = "/bin/ksh"
			}
			//task.Module = "sleep"
			//task.Args = []string{strconv.Itoa(sleepTime)}
			task.Status = -1
			log.Printf("[%v:%v] Running (%v %v)", task.Id, task.Name, task.Module, task.Args[0])
			//log.Printf("[%v] Connecting in %v on %v", task.Name, proto, socket)
			task.StartTime = time.Now()
			if task.Module != "dummy" && task.Module != "meta" {
				task.Status = Client(task, &proto, &socket)
			} else {
				task.Status = 0
			}
			task.EndTime = time.Now()
			// ... Do a lot of stufs...
			//time.Sleep(time.Duration(sleepTime) * time.Second)
			// Adjust the Status
			//task.Status = 2
			// Send it on the channel
			log.Printf("[%v:%v] Done", task.Id, task.Name)
			doneChan <- task
			wg.Done()
			return
		}
	}
}

// The advertize goroutine, reads the tasks from doneChannel and write the TaskGraphStructure back to the taskStructureChan
func Advertize(taskStructure *TaskGraphStructure, doneChan <-chan *Task) {
	// Let's launch the task that can initially run
	rowSize, _ := taskStructure.AdjacencyMatrix.Dims()
	for taskIndex, _ := range taskStructure.Tasks {
		sum := float64(0)
		for r := 0; r < rowSize; r++ {
			sum += taskStructure.AdjacencyMatrix.At(r, taskIndex)
		}
		if sum == 0 && taskStructure.Tasks[taskIndex].Status < 0 {
			taskStructure.Tasks[taskIndex].TaskCanRunChan <- true
		}
	}
	doneAdjacency := mat64.DenseCopyOf(taskStructure.AdjacencyMatrix)
	for {
		task := <-doneChan

		//TODO : There is absolutely no need to change the adjacency matrix AT ALL
		// It is a lot better to create a copy

		// Adapting the Adjacency matrix...
		// TaskId is finished, it cannot be the source of any task anymore
		// Set the row at 0
		rowSize, colSize := doneAdjacency.Dims()
		for c := 0; c < colSize; c++ {
			doneAdjacency.Set(task.Id, c, float64(0))
		}
		// For each dependency of the task
		// We can run if the sum of the element of the column Id of the current task is 0
		for taskIndex, _ := range taskStructure.Tasks {
			sum := float64(0)
			for r := 0; r < rowSize; r++ {
				sum += doneAdjacency.At(r, taskIndex)
			}

			if sum == 0 && taskStructure.Tasks[taskIndex].Status == -2 {
				taskStructure.Tasks[taskIndex].TaskCanRunChan <- true
			}
		}
	}
}
<commit_msg>Comment cleaning<commit_after>package gautomator

import (
	"github.com/gonum/matrix/mat64" // Matrix
	"log"
	"math/rand" // Temp
	//"strconv"
	"sync"
	"time"
)

func random(min int, max int) int {
	var bytes int
	bytes = min + rand.Intn(max)
	return int(bytes)
	//rand.Seed(time.Now().UTC().UnixNano())
	//return rand.Intn(max - min) + min
}

// Ths runner goroutine is a goroutinei which:
// Consume the TaskGraphStructure from a channel
// run the task given as arguments if the deps are done
// Post the task to the doncChannel once done
//
func Runner(task *Task, doneChan chan<- *Task, wg *sync.WaitGroup) {
	log.Printf("[%v:%v] Queued", task.Id, task.Name)
	for {
		letsGo := <-task.TaskCanRunChan
		// For each dependency of the task
		// We can run if the sum of the element of the column Id of the current task is 0

		if letsGo == true {
			proto := "tcp"
			socket := task.Node
			//sleepTime := random(1, 5)
			// Stupid trick to make shell works... A Shell module will be implemented later"
			if task.Module == "shell" {
				task.Module = "echo"
				n := len(task.Args)
				task.Args[n] = "|"
				task.Args[n+1] = "/bin/ksh"
			}
			//task.Module = "sleep"
			//task.Args = []string{strconv.Itoa(sleepTime)}
			task.Status = -1
			log.Printf("[%v:%v] Running (%v %v)", task.Id, task.Name, task.Module, task.Args[0])
			//log.Printf("[%v] Connecting in %v on %v", task.Name, proto, socket)
			task.StartTime = time.Now()
			if task.Module != "dummy" && task.Module != "meta" {
				task.Status = Client(task, &proto, &socket)
			} else {
				task.Status = 0
			}
			task.EndTime = time.Now()
			// ... Do a lot of stufs...
			//time.Sleep(time.Duration(sleepTime) * time.Second)
			// Adjust the Status
			//task.Status = 2
			// Send it on the channel
			log.Printf("[%v:%v] Done", task.Id, task.Name)
			doneChan <- task
			wg.Done()
			return
		}
	}
}

// The advertize goroutine, reads the tasks from doneChannel and write the TaskGraphStructure back to the taskStructureChan
func Advertize(taskStructure *TaskGraphStructure, doneChan <-chan *Task) {
	// Let's launch the task that can initially run
	rowSize, _ := taskStructure.AdjacencyMatrix.Dims()
	for taskIndex, _ := range taskStructure.Tasks {
		sum := float64(0)
		for r := 0; r < rowSize; r++ {
			sum += taskStructure.AdjacencyMatrix.At(r, taskIndex)
		}
		if sum == 0 && taskStructure.Tasks[taskIndex].Status < 0 {
			taskStructure.Tasks[taskIndex].TaskCanRunChan <- true
		}
	}
	doneAdjacency := mat64.DenseCopyOf(taskStructure.AdjacencyMatrix)
	for {
		task := <-doneChan

		// TaskId is finished, it cannot be the source of any task anymore
		// Set the row at 0
		rowSize, colSize := doneAdjacency.Dims()
		for c := 0; c < colSize; c++ {
			doneAdjacency.Set(task.Id, c, float64(0))
		}
		// For each dependency of the task
		// We can run if the sum of the element of the column Id of the current task is 0
		for taskIndex, _ := range taskStructure.Tasks {
			sum := float64(0)
			for r := 0; r < rowSize; r++ {
				sum += doneAdjacency.At(r, taskIndex)
			}

			if sum == 0 && taskStructure.Tasks[taskIndex].Status == -2 {
				taskStructure.Tasks[taskIndex].TaskCanRunChan <- true
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	docker "github.com/smashwilson/go-dockerclient"
)

// OutputCollector is an io.Writer that accumulates output from a specified stream in an attached
// Docker container and appends it to the appropriate field within a SubmittedJob.
type OutputCollector struct {
	context  *Context
	job      *SubmittedJob
	isStdout bool
}

// DescribeStream returns "stdout" or "stderr" to indicate which stream this collector is consuming.
func (c OutputCollector) DescribeStream() string {
	if c.isStdout {
		return "stdout"
	}
	return "stderr"
}

// Write appends bytes to the selected stream and updates the SubmittedJob.
func (c OutputCollector) Write(p []byte) (int, error) {
	log.WithFields(log.Fields{
		"length": len(p),
		"bytes":  string(p),
		"stream": c.DescribeStream(),
	}).Debug("Received output from a job")

	if c.isStdout {
		c.job.Stdout += string(p)
	} else {
		c.job.Stderr += string(p)
	}

	if err := c.context.UpdateJob(c.job); err != nil {
		return 0, err
	}

	return len(p), nil
}

// Runner is the main entry point for the job runner goroutine.
func Runner(c *Context) {
	for {
		Claim(c)

		time.Sleep(time.Duration(c.Poll) * time.Millisecond)
	}
}

// Claim acquires the oldest single pending job and launches a goroutine to execute its command in
// a new container.
func Claim(c *Context) {
	job, err := c.ClaimJob()
	if err != nil {
		log.WithFields(log.Fields{"error": err}).Error("Unable to claim a job.")
		return
	}
	if job == nil {
		// Nothing to claim.
		return
	}
	if err := job.Validate(); err != nil {
		fields := log.Fields{
			"jid":     job.JID,
			"account": job.Account,
			"error":   err,
		}

		log.WithFields(fields).Error("Invalid job in queue.")

		job.Status = StatusError
		if err := c.UpdateJob(job); err != nil {
			fields["error"] = err
			log.WithFields(fields).Error("Unable to update job status.")
		}

		return
	}

	go Execute(c, job)
}

// Execute launches a container to process the submitted job. It passes any provided stdin data
// to the container and consumes stdout and stderr, updating Mongo as it runs. Once completed, it
// acquires the job's result from its configured source and marks the job as finished.
func Execute(c *Context, job *SubmittedJob) {
	defaultFields := log.Fields{
		"jid":     job.JID,
		"account": job.Account,
	}

	// Logging utility messages.
	debug := func(message string) {
		log.WithFields(defaultFields).Debug(message)
	}
	reportErr := func(message string, err error) {
		fs := log.Fields{}
		for k, v := range defaultFields {
			fs[k] = v
		}
		fs["err"] = err
		log.WithFields(fs).Error(message)
	}
	checkErr := func(message string, err error) bool {
		if err == nil {
			debug(fmt.Sprintf("%s: ok", message))
			return false
		}

		reportErr(fmt.Sprintf("%s: ERROR", message), err)
		return true
	}

	// Update the job model in Mongo, reporting any errors along the way.
	// This also updates our job model with any changes from Mongo, such as the kill request flag.
	updateJob := func(message string) bool {
		if err := c.UpdateJob(job); err != nil {
			reportErr(fmt.Sprintf("Unable to update the job's %s.", message), err)
			return false
		}
		return true
	}

	log.WithFields(defaultFields).Info("Launching a job.")

	job.StartedAt = StoreTime(time.Now())
	job.QueueDelay = job.StartedAt.AsTime().Sub(job.CreatedAt.AsTime()).Nanoseconds()

	container, err := c.CreateContainer(docker.CreateContainerOptions{
		Name: job.ContainerName(),
		Config: &docker.Config{
			Image:     c.Image,
			Cmd:       []string{"/bin/bash", "-c", job.Command},
			OpenStdin: true,
			StdinOnce: true,
		},
	})
	if checkErr("Created the job's container", err) {
		job.Status = StatusError
		updateJob("status")
		return
	}

	// Record the job's container ID.
	job.ContainerID = container.ID
	if !updateJob("start timestamp and container id") {
		return
	}

	// Include container information in this job's logging messages.
	defaultFields["container id"] = container.ID
	defaultFields["container name"] = container.Name

	// Was a kill requested between the time the job was claimed, and the time the container was
	// created? If so: transition the job to StatusKilled and jump ahead to removing the container
	// we just created. If not: continue with job execution normally.

	// If a kill was requested before the job was claimed, it would have been removed from the queue.
	// If a kill is requested after the container was created, it will have the containerID that we
	// just sent and be able to kill the running container.

	if job.KillRequested {
		job.Status = StatusKilled
	} else {
		// Prepare the input and output streams.
		stdin := bytes.NewReader(job.Stdin)
		stdout := OutputCollector{
			context:  c,
			job:      job,
			isStdout: true,
		}
		stderr := OutputCollector{
			context:  c,
			job:      job,
			isStdout: false,
		}

		go func() {
			err = c.AttachToContainer(docker.AttachToContainerOptions{
				Container:    container.ID,
				Stream:       true,
				InputStream:  stdin,
				OutputStream: stdout,
				ErrorStream:  stderr,
				Stdin:        true,
				Stdout:       true,
				Stderr:       true,
			})
			checkErr("Attached to the container", err)
		}()

		// Start the created container.
		err = c.StartContainer(container.ID, &docker.HostConfig{})
		if checkErr("Started the container", err) {
			job.Status = StatusError
			updateJob("status")
			return
		}

		// Measure the container-launch overhead here.
		overhead := time.Now()
		job.OverheadDelay = overhead.Sub(job.StartedAt.AsTime()).Nanoseconds()
		updateJob("overhead delay")

		status, err := c.WaitContainer(container.ID)
		if checkErr("Waited for the container to complete", err) {
			job.Status = StatusError
			updateJob("status")
			return
		}

		job.FinishedAt = StoreTime(time.Now())
		job.Runtime = job.FinishedAt.AsTime().Sub(overhead).Nanoseconds()
		if status == 0 {
			// Successful termination.
			job.Status = StatusDone
		} else {
			// Something went wrong.

			// See if a kill was explicitly requested. If so, transition to StatusKilled. Otherwise,
			// transition to StatusError.
			killed, err := c.JobKillRequested(job.JID)
			if err != nil {
				reportErr("Check the job kill status: ERROR", err)
				return
			}

			if killed {
				job.Status = StatusKilled
			} else {
				job.Status = StatusError
			}
		}

		// Extract the result from the job.
		if job.ResultSource == "stdout" {
			job.Result = []byte(job.Stdout)
			debug("Acquired job result from stdout: ok")
		} else if strings.HasPrefix(job.ResultSource, "file:") {
			resultPath := job.ResultSource[len("file:"):len(job.ResultSource)]

			var resultBuffer bytes.Buffer
			err = c.CopyFromContainer(docker.CopyFromContainerOptions{
				Container:    container.ID,
				Resource:     resultPath,
				OutputStream: &resultBuffer,
			})
			if checkErr(fmt.Sprintf("Acquired the job's result from the file [%s]", resultPath), err) {
				job.Status = StatusError
			} else {
				// CopyFromContainer returns the file contents as a tarball.
				var content bytes.Buffer
				r := bytes.NewReader(resultBuffer.Bytes())
				tr := tar.NewReader(r)

				for {
					_, err := tr.Next()
					if err == io.EOF {
						break
					}
					if err != nil {
						reportErr("Read tar-encoded content: ERROR", err)
						job.Status = StatusError
						break
					}

					if _, err = io.Copy(&content, tr); err != nil {
						reportErr("Copy decoded content: ERROR", err)
						job.Status = StatusError
						break
					}
				}

				job.Result = content.Bytes()
			}
		}

		// Job execution has completed successfully.
	}

	err = c.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID})
	checkErr("Removed the container", err)

	err = c.UpdateAccountUsage(job.Account, job.Runtime)
	if err != nil {
		reportErr("Update account usage: ERROR", err)
		return
	}
	updateJob("status and final result")

	log.WithFields(log.Fields{
		"jid":      job.JID,
		"account":  job.Account,
		"status":   job.Status,
		"runtime":  job.Runtime,
		"overhead": job.OverheadDelay,
		"queue":    job.QueueDelay,
	}).Info("Job complete.")
}
<commit_msg>If-else matching fail<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	docker "github.com/smashwilson/go-dockerclient"
)

// OutputCollector is an io.Writer that accumulates output from a specified stream in an attached
// Docker container and appends it to the appropriate field within a SubmittedJob.
type OutputCollector struct {
	context  *Context
	job      *SubmittedJob
	isStdout bool
}

// DescribeStream returns "stdout" or "stderr" to indicate which stream this collector is consuming.
func (c OutputCollector) DescribeStream() string {
	if c.isStdout {
		return "stdout"
	}
	return "stderr"
}

// Write appends bytes to the selected stream and updates the SubmittedJob.
func (c OutputCollector) Write(p []byte) (int, error) {
	log.WithFields(log.Fields{
		"length": len(p),
		"bytes":  string(p),
		"stream": c.DescribeStream(),
	}).Debug("Received output from a job")

	if c.isStdout {
		c.job.Stdout += string(p)
	} else {
		c.job.Stderr += string(p)
	}

	if err := c.context.UpdateJob(c.job); err != nil {
		return 0, err
	}

	return len(p), nil
}

// Runner is the main entry point for the job runner goroutine.
func Runner(c *Context) {
	for {
		Claim(c)

		time.Sleep(time.Duration(c.Poll) * time.Millisecond)
	}
}

// Claim acquires the oldest single pending job and launches a goroutine to execute its command in
// a new container.
func Claim(c *Context) {
	job, err := c.ClaimJob()
	if err != nil {
		log.WithFields(log.Fields{"error": err}).Error("Unable to claim a job.")
		return
	}
	if job == nil {
		// Nothing to claim.
		return
	}
	if err := job.Validate(); err != nil {
		fields := log.Fields{
			"jid":     job.JID,
			"account": job.Account,
			"error":   err,
		}

		log.WithFields(fields).Error("Invalid job in queue.")

		job.Status = StatusError
		if err := c.UpdateJob(job); err != nil {
			fields["error"] = err
			log.WithFields(fields).Error("Unable to update job status.")
		}

		return
	}

	go Execute(c, job)
}

// Execute launches a container to process the submitted job. It passes any provided stdin data
// to the container and consumes stdout and stderr, updating Mongo as it runs. Once completed, it
// acquires the job's result from its configured source and marks the job as finished.
func Execute(c *Context, job *SubmittedJob) {
	defaultFields := log.Fields{
		"jid":     job.JID,
		"account": job.Account,
	}

	// Logging utility messages.
	debug := func(message string) {
		log.WithFields(defaultFields).Debug(message)
	}
	reportErr := func(message string, err error) {
		fs := log.Fields{}
		for k, v := range defaultFields {
			fs[k] = v
		}
		fs["err"] = err
		log.WithFields(fs).Error(message)
	}
	checkErr := func(message string, err error) bool {
		if err == nil {
			debug(fmt.Sprintf("%s: ok", message))
			return false
		}

		reportErr(fmt.Sprintf("%s: ERROR", message), err)
		return true
	}

	// Update the job model in Mongo, reporting any errors along the way.
	// This also updates our job model with any changes from Mongo, such as the kill request flag.
	updateJob := func(message string) bool {
		if err := c.UpdateJob(job); err != nil {
			reportErr(fmt.Sprintf("Unable to update the job's %s.", message), err)
			return false
		}
		return true
	}

	log.WithFields(defaultFields).Info("Launching a job.")

	job.StartedAt = StoreTime(time.Now())
	job.QueueDelay = job.StartedAt.AsTime().Sub(job.CreatedAt.AsTime()).Nanoseconds()

	container, err := c.CreateContainer(docker.CreateContainerOptions{
		Name: job.ContainerName(),
		Config: &docker.Config{
			Image:     c.Image,
			Cmd:       []string{"/bin/bash", "-c", job.Command},
			OpenStdin: true,
			StdinOnce: true,
		},
	})
	if checkErr("Created the job's container", err) {
		job.Status = StatusError
		updateJob("status")
		return
	}

	// Record the job's container ID.
	job.ContainerID = container.ID
	if !updateJob("start timestamp and container id") {
		return
	}

	// Include container information in this job's logging messages.
	defaultFields["container id"] = container.ID
	defaultFields["container name"] = container.Name

	// Was a kill requested between the time the job was claimed, and the time the container was
	// created? If so: transition the job to StatusKilled and jump ahead to removing the container
	// we just created. If not: continue with job execution normally.

	// If a kill was requested before the job was claimed, it would have been removed from the queue.
	// If a kill is requested after the container was created, it will have the containerID that we
	// just sent and be able to kill the running container.

	if job.KillRequested {
		job.Status = StatusKilled
	} else {
		// Prepare the input and output streams.
		stdin := bytes.NewReader(job.Stdin)
		stdout := OutputCollector{
			context:  c,
			job:      job,
			isStdout: true,
		}
		stderr := OutputCollector{
			context:  c,
			job:      job,
			isStdout: false,
		}

		go func() {
			err = c.AttachToContainer(docker.AttachToContainerOptions{
				Container:    container.ID,
				Stream:       true,
				InputStream:  stdin,
				OutputStream: stdout,
				ErrorStream:  stderr,
				Stdin:        true,
				Stdout:       true,
				Stderr:       true,
			})
			checkErr("Attached to the container", err)
		}()

		// Start the created container.
		err = c.StartContainer(container.ID, &docker.HostConfig{})
		if checkErr("Started the container", err) {
			job.Status = StatusError
			updateJob("status")
			return
		}

		// Measure the container-launch overhead here.
		overhead := time.Now()
		job.OverheadDelay = overhead.Sub(job.StartedAt.AsTime()).Nanoseconds()
		updateJob("overhead delay")

		status, err := c.WaitContainer(container.ID)
		if checkErr("Waited for the container to complete", err) {
			job.Status = StatusError
			updateJob("status")
			return
		}

		job.FinishedAt = StoreTime(time.Now())
		job.Runtime = job.FinishedAt.AsTime().Sub(overhead).Nanoseconds()
		if status == 0 {
			// Successful termination.
			job.Status = StatusDone

			// Extract the result from the job.
			if job.ResultSource == "stdout" {
				job.Result = []byte(job.Stdout)
				debug("Acquired job result from stdout: ok")
			} else if strings.HasPrefix(job.ResultSource, "file:") {
				resultPath := job.ResultSource[len("file:"):len(job.ResultSource)]

				var resultBuffer bytes.Buffer
				err = c.CopyFromContainer(docker.CopyFromContainerOptions{
					Container:    container.ID,
					Resource:     resultPath,
					OutputStream: &resultBuffer,
				})
				if checkErr(fmt.Sprintf("Acquired the job's result from the file [%s]", resultPath), err) {
					job.Status = StatusError
				} else {
					// CopyFromContainer returns the file contents as a tarball.
					var content bytes.Buffer
					r := bytes.NewReader(resultBuffer.Bytes())
					tr := tar.NewReader(r)

					for {
						_, err := tr.Next()
						if err == io.EOF {
							break
						}
						if err != nil {
							reportErr("Read tar-encoded content: ERROR", err)
							job.Status = StatusError
							break
						}

						if _, err = io.Copy(&content, tr); err != nil {
							reportErr("Copy decoded content: ERROR", err)
							job.Status = StatusError
							break
						}
					}

					job.Result = content.Bytes()
				}
			}
		} else {
			// Something went wrong.

			// See if a kill was explicitly requested. If so, transition to StatusKilled. Otherwise,
			// transition to StatusError.
			killed, err := c.JobKillRequested(job.JID)
			if err != nil {
				reportErr("Check the job kill status: ERROR", err)
				return
			}

			if killed {
				job.Status = StatusKilled
			} else {
				job.Status = StatusError
			}
		}

		// Job execution has completed successfully.
	}

	err = c.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID})
	checkErr("Removed the container", err)

	err = c.UpdateAccountUsage(job.Account, job.Runtime)
	if err != nil {
		reportErr("Update account usage: ERROR", err)
	}
	updateJob("status and final result")

	log.WithFields(log.Fields{
		"jid":      job.JID,
		"account":  job.Account,
		"status":   job.Status,
		"runtime":  job.Runtime,
		"overhead": job.OverheadDelay,
		"queue":    job.QueueDelay,
	}).Info("Job complete.")
}
<|endoftext|>