<commit_before>package requests_test

import (
	"testing"

	"encoding/json"

	"github.com/solher/arangolite/requests"
)

const indentedQuery = `
    FOR x
    IN documents
    RETURN {
        document: x,
        foo: bar
    }
`

type aql struct {
	Query     string                 `json:"query"`
	BindVars  map[string]interface{} `json:"bindVars,omitempty"`
	Cache     bool                   `json:"cache"`
	BatchSize int                    `json:"batchSize,omitempty"`
}

// TestAQL runs tests on the AQL request.
func TestAQL(t *testing.T) {
	var testCases = []struct {
		// Case description
		description string
		// Arguments
		query     string
		params    []interface{}
		bind      map[string]interface{}
		batchSize int
		cache     bool
		// Expected results
		output aql
	}{
		{
			description: "basic query",
			query:       "FOR x IN documents RETURN x",
			cache:       false,
			output: aql{
				Query: "FOR x IN documents RETURN x",
				Cache: false,
			},
		},
		{
			description: "cache",
			query:       "FOR x IN documents RETURN x",
			cache:       true,
			output: aql{
				Query: "FOR x IN documents RETURN x",
				Cache: true,
			},
		},
		{
			description: "batch size",
			query:       "FOR x IN documents RETURN x",
			cache:       false,
			batchSize:   1000,
			output: aql{
				Query:     "FOR x IN documents RETURN x",
				Cache:     false,
				BatchSize: 1000,
			},
		},
		{
			description: "bind parameters",
			query:       "FOR x IN documents FILTER x.attr1 == @attr1 AND x.attr2 == @attr2 RETURN x",
			cache:       false,
			bind: map[string]interface{}{
				"attr1": "foobar",
				"attr2": 100,
			},
			output: aql{
				Query: "FOR x IN documents FILTER x.attr1 == @attr1 AND x.attr2 == @attr2 RETURN x",
				Cache: false,
				BindVars: map[string]interface{}{
					"attr1": "foobar",
					"attr2": 100,
				},
			},
		},
		{
			description: "indented query",
			query:       indentedQuery,
			cache:       false,
			output: aql{
				Query: "FOR x IN documents RETURN { document: x, foo: bar }",
				Cache: false,
			},
		},
		{
			description: "use of parameters",
			query:       `FOR x IN documents FILTER x.attr1 == "%s" RETURN x`,
			params:      []interface{}{"foobar"},
			cache:       false,
			output: aql{
				Query: `FOR x IN documents FILTER x.attr1 == 'foobar' RETURN x`,
				Cache: false,
			},
		},
	}

	for _, tc := range testCases {
		t.Run(tc.description, func(t *testing.T) {
			aql := requests.NewAQL(tc.query, tc.params...).Cache(tc.cache)
			if tc.batchSize != 0 {
				aql.BatchSize(tc.batchSize)
			}
			for name, value := range tc.bind {
				aql.Bind(name, value)
			}
			output, err := json.Marshal(tc.output)
			if err != nil {
				t.Error(err)
			}
			if string(output) != string(aql.Generate()) {
				t.Errorf("unexpected output. Expected %s, got %s", output, aql.Generate())
			}
		})
	}
}
<commit_msg>Test added<commit_after>package requests_test

import (
	"testing"

	"encoding/json"

	"github.com/solher/arangolite/requests"
)

const indentedQuery = `
    FOR x
    IN documents
    RETURN {
        document: x,
        foo: bar
    }
`

type aql struct {
	Query     string                 `json:"query"`
	BindVars  map[string]interface{} `json:"bindVars,omitempty"`
	Cache     bool                   `json:"cache"`
	BatchSize int                    `json:"batchSize,omitempty"`
}

// TestAQL runs tests on the AQL request.
func TestAQL(t *testing.T) {
	var testCases = []struct {
		// Case description
		description string
		// Arguments
		query     string
		params    []interface{}
		bind      map[string]interface{}
		batchSize int
		cache     bool
		// Expected results
		output aql
	}{
		{
			description: "basic query",
			query:       "FOR x IN documents RETURN x",
			cache:       false,
			output: aql{
				Query: "FOR x IN documents RETURN x",
				Cache: false,
			},
		},
		{
			description: "cache",
			query:       "FOR x IN documents RETURN x",
			cache:       true,
			output: aql{
				Query: "FOR x IN documents RETURN x",
				Cache: true,
			},
		},
		{
			description: "batch size",
			query:       "FOR x IN documents RETURN x",
			cache:       false,
			batchSize:   1000,
			output: aql{
				Query:     "FOR x IN documents RETURN x",
				Cache:     false,
				BatchSize: 1000,
			},
		},
		{
			description: "bind parameters",
			query:       "FOR x IN documents FILTER x.attr1 == @attr1 AND x.attr2 == @attr2 RETURN x",
			cache:       false,
			bind: map[string]interface{}{
				"attr1": "foobar",
				"attr2": 100,
			},
			output: aql{
				Query: "FOR x IN documents FILTER x.attr1 == @attr1 AND x.attr2 == @attr2 RETURN x",
				Cache: false,
				BindVars: map[string]interface{}{
					"attr1": "foobar",
					"attr2": 100,
				},
			},
		},
		{
			description: "indented query",
			query:       indentedQuery,
			cache:       false,
			output: aql{
				Query: "FOR x IN documents RETURN { document: x, foo: bar }",
				Cache: false,
			},
		},
		{
			description: "use of parameters",
			query:       `FOR x IN documents FILTER x.attr1 == "%s" RETURN x`,
			params:      []interface{}{"foobar"},
			cache:       false,
			output: aql{
				Query: `FOR x IN documents FILTER x.attr1 == 'foobar' RETURN x`,
				Cache: false,
			},
		},
		{
			description: "quotes are replaced by single quotes, even from parameters",
			query:       `UPSERT { "id":"foo" } INSERT %s UPDATE {} IN bar`,
			params:      []interface{}{`{ "id":"foo" }`},
			cache:       false,
			output: aql{
				Query: `UPSERT { 'id':'foo' } INSERT { 'id':'foo' } UPDATE {} IN bar`,
				Cache: false,
			},
		},
	}

	for _, tc := range testCases {
		t.Run(tc.description, func(t *testing.T) {
			aql := requests.NewAQL(tc.query, tc.params...).Cache(tc.cache)
			if tc.batchSize != 0 {
				aql.BatchSize(tc.batchSize)
			}
			for name, value := range tc.bind {
				aql.Bind(name, value)
			}
			output, err := json.Marshal(tc.output)
			if err != nil {
				t.Error(err)
			}
			if string(output) != string(aql.Generate()) {
				t.Errorf("unexpected output. Expected %s, got %s", output, aql.Generate())
			}
		})
	}
}
<|endoftext|>


<commit_before>package jsonconsul

func (c *JsonExport) RunWatcher() {
	for {
		err := c.Run()
		if err != nil {
			log.Println(err)
			break
		}

		log.Println("Waiting", time.Second*c.WatchFrequency)
		<-time.After(time.Second * c.WatchFrequency)
	}
}
<commit_msg>A minimum viable watch which is not poll<commit_after>// Adapted from
// https://github.com/hashicorp/envconsul/blob/master/runner.go

package jsonconsul

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	dep "github.com/hashicorp/consul-template/dependency"
	"github.com/hashicorp/consul-template/watch"
	"github.com/hashicorp/consul/api"
)

func (c *JsonExport) RunWatcher() {
	runner, err := NewRunner(c, false)
	if err != nil {
		log.Fatal(err)
	}

	runner.Start()
}

type Runner struct {
	sync.RWMutex

	// Prefix is the KeyPrefixDependency associated with this Runner.
	Prefix *dep.StoreKeyPrefix

	// ErrCh and DoneCh are channels where errors and finish notifications occur.
	ErrCh  chan error
	DoneCh chan struct{}

	// ExitCh is a channel for parent processes to read exit status values from
	// the child processes.
	ExitCh chan int

	// config is the Config that created this Runner. It is used internally to
	// construct other objects and pass data.
	config *JsonExport

	// client is the consul/api client.
	client *api.Client

	// once indicates the runner should get data exactly one time and then stop.
	once bool

	// minTimer and maxTimer are used for quiescence.
	minTimer, maxTimer <-chan time.Time

	// outStream and errStream are the io.Writer streams where the runner will
	// write information.
	outStream, errStream io.Writer

	// watcher is the watcher this runner is using.
	watcher *watch.Watcher

	// data is the latest representation of the data from Consul.
	data map[string][]*dep.KeyPair

	// killSignal is the signal to send to kill the process.
	killSignal os.Signal
}

// NewRunner accepts a JsonExport, and boolean value for once mode.
func NewRunner(config *JsonExport, once bool) (*Runner, error) {
	var err error

	log.Printf("[INFO] (runner) creating new runner (once: %v)", once)

	runner := &Runner{
		config: config,
		once:   once,
	}

	s := strings.TrimPrefix(config.Prefix, "/")
	runner.Prefix, err = dep.ParseStoreKeyPrefix(s)
	if err != nil {
		return nil, err
	}

	if err := runner.init(); err != nil {
		return nil, err
	}

	return runner, nil
}

// Start creates a new runner and begins watching dependencies and quiescence
// timers. This is the main event loop and will block until finished.
func (r *Runner) Start() {
	var (
		exitCh <-chan int
	)

	log.Printf("[INFO] (runner) starting")

	// Add the dependencies to the watcher
	r.watcher.Add(r.Prefix)

	for {
		select {
		case data := <-r.watcher.DataCh:
			r.Receive(data.Dependency, data.Data)

			// Drain all views that have data
		OUTER:
			for {
				select {
				case data = <-r.watcher.DataCh:
					r.Receive(data.Dependency, data.Data)
				default:
					break OUTER
				}
			}
		case <-r.minTimer:
			log.Printf("[INFO] (runner) quiescence minTimer fired")
			r.minTimer, r.maxTimer = nil, nil
		case <-r.maxTimer:
			log.Printf("[INFO] (runner) quiescence maxTimer fired")
			r.minTimer, r.maxTimer = nil, nil
		case err := <-r.watcher.ErrCh:
			// Intentionally do not send the error back up to the runner. Eventually,
			// once Consul API implements errwrap and multierror, we can check the
			// "type" of error and conditionally alert back.
			//
			// if err.Contains(Something) {
			//   errCh <- err
			// }
			log.Printf("[ERR] (runner) watcher reported error: %s", err)
		case <-r.watcher.FinishCh:
			log.Printf("[INFO] (runner) watcher reported finish")
			return
		case code := <-exitCh:
			r.ExitCh <- code
		case <-r.DoneCh:
			log.Printf("[INFO] (runner) received finish")
			return
		}

		// If we got this far, that means we got new data or one of the timers
		// fired, so attempt to re-process the environment.
		r.Run()
	}
}

// Stop halts the execution of this runner and its subprocesses.
func (r *Runner) Stop() {
	log.Printf("[INFO] (runner) stopping")
	r.watcher.Stop()

	close(r.DoneCh)
}

// Receive accepts data from Consul and maps that data to the prefix.
func (r *Runner) Receive(d dep.Dependency, data interface{}) {
	r.Lock()
	defer r.Unlock()
	r.data[d.HashCode()] = data.([]*dep.KeyPair)
}

// Run executes and manages the child process with the correct environment. The
// current enviornment is also copied into the child process environment.
func (r *Runner) Run() {
	log.Printf("[INFO] (runner) running")

	// TODO: Just call the app to consul again. This should
	// probably be updated to actually receive the values that we
	// got but MVP.
	r.config.Run()
}

// init creates the Runner's underlying data structures and returns an error if
// any problems occur.
func (r *Runner) init() error {
	// Print the final config for debugging
	result, err := json.MarshalIndent(r.config, "", "  ")
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] (runner) final config (tokens suppressed):\n\n%s\n\n",
		result)

	r.client = client

	// Create the watcher
	watcher, err := newWatcher(r.config, client, r.once)
	if err != nil {
		return fmt.Errorf("runner: %s", err)
	}
	r.watcher = watcher

	r.data = make(map[string][]*dep.KeyPair)

	r.outStream = os.Stdout
	r.errStream = os.Stderr

	r.ErrCh = make(chan error)
	r.DoneCh = make(chan struct{})
	r.ExitCh = make(chan int, 1)

	return nil
}

// newWatcher creates a new watcher.
func newWatcher(config *JsonExport, client *api.Client, once bool) (*watch.Watcher, error) {
	log.Printf("[INFO] (runner) creating Watcher")

	clientSet := dep.NewClientSet()
	if err := clientSet.Add(client); err != nil {
		return nil, err
	}

	watcher, err := watch.NewWatcher(&watch.WatcherConfig{
		Clients: clientSet,
		Once:    once,
	})
	if err != nil {
		return nil, err
	}

	return watcher, err
}
<|endoftext|>


<commit_before>package io

import (
	"io"

	"v2ray.com/core/common"
	"v2ray.com/core/common/alloc"
)

// Reader extends io.Reader with alloc.Buffer.
type Reader interface {
	common.Releasable
	// Read reads content from underlying reader, and put it into an alloc.Buffer.
	Read() (*alloc.Buffer, error)
}

// AdaptiveReader is a Reader that adjusts its reading speed automatically.
type AdaptiveReader struct {
	reader   io.Reader
	allocate func() *alloc.Buffer
}

// NewAdaptiveReader creates a new AdaptiveReader.
// The AdaptiveReader instance doesn't take the ownership of reader.
func NewAdaptiveReader(reader io.Reader) *AdaptiveReader {
	return &AdaptiveReader{
		reader:   reader,
		allocate: alloc.NewBuffer,
	}
}

// Read implements Reader.Read().
func (this *AdaptiveReader) Read() (*alloc.Buffer, error) {
	buffer := this.allocate().Clear()
	_, err := buffer.FillFrom(this.reader)
	if err != nil {
		buffer.Release()
		return nil, err
	}

	//if buffer.Len() >= alloc.BufferSize {
	//	this.allocate = alloc.NewLargeBuffer
	//} else {
	//	this.allocate = alloc.NewBuffer
	//}

	return buffer, nil
}

func (this *AdaptiveReader) Release() {
	this.reader = nil
}
<commit_msg>remove unused code<commit_after>package io

import (
	"io"

	"v2ray.com/core/common"
	"v2ray.com/core/common/alloc"
)

// Reader extends io.Reader with alloc.Buffer.
type Reader interface {
	common.Releasable
	// Read reads content from underlying reader, and put it into an alloc.Buffer.
	Read() (*alloc.Buffer, error)
}

// AdaptiveReader is a Reader that adjusts its reading speed automatically.
type AdaptiveReader struct {
	reader   io.Reader
	allocate func() *alloc.Buffer
}

// NewAdaptiveReader creates a new AdaptiveReader.
// The AdaptiveReader instance doesn't take the ownership of reader.
func NewAdaptiveReader(reader io.Reader) *AdaptiveReader {
	return &AdaptiveReader{
		reader:   reader,
		allocate: alloc.NewBuffer,
	}
}

// Read implements Reader.Read().
func (this *AdaptiveReader) Read() (*alloc.Buffer, error) {
	buffer := this.allocate().Clear()
	_, err := buffer.FillFrom(this.reader)
	if err != nil {
		buffer.Release()
		return nil, err
	}

	return buffer, nil
}

func (this *AdaptiveReader) Release() {
	this.reader = nil
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"github.com/meshbird/meshbird/log"
	"github.com/meshbird/meshbird/secure"
	"sync"
)

type LocalNode struct {
	Node

	secret *secure.NetworkSecret
	config *Config
	state  *State

	mutex     sync.Mutex
	waitGroup sync.WaitGroup

	services map[string]Service

	logger log.Logger
}

func NewLocalNode(cfg *Config) (*LocalNode, error) {
	var err error
	n := new(LocalNode)
	n.logger = log.L("local")

	n.secret, err = secure.NetworkSecretUnmarshal(cfg.SecretKey)
	if err != nil {
		return nil, err
	}

	n.config = cfg
	n.config.NetworkID = n.secret.InfoHash()
	n.state = NewState(n.secret)

	n.services = make(map[string]Service)

	n.AddService(&NetTable{})
	n.AddService(&ListenerService{})
	n.AddService(&DiscoveryDHT{})
	n.AddService(&InterfaceService{})
	n.AddService(&STUNService{})
	n.AddService(&UPnPService{})
	n.AddService(&HttpService{})
	return n, nil
}

func (n *LocalNode) Config() Config {
	return *n.config
}

func (n *LocalNode) State() State {
	return *n.state
}

func (n *LocalNode) AddService(srv Service) {
	n.services[srv.Name()] = srv
}

func (n *LocalNode) Start() error {
	for _, service := range n.services {
		n.logger.Info("initializing %s service", service.Name())
		if err := service.Init(n); err != nil {
			return fmt.Errorf("initialization of %q finished with error: %v", service.Name(), err)
		}
		n.waitGroup.Add(1)

		go func(srv Service) {
			defer n.waitGroup.Done()
			n.logger.Info("running %q service", srv.Name())
			if err := srv.Run(); err != nil {
				n.logger.Error("error on run %q service, %v", srv.Name(), err)
			}
		}(service)
	}
	return nil
}

func (n *LocalNode) Service(name string) Service {
	service, ok := n.services[name]
	if !ok {
		n.logger.Fatal("service %q not found", name)
	}
	return service
}

func (n *LocalNode) WaitStop() {
	n.waitGroup.Wait()
}

func (n *LocalNode) Stop() error {
	n.logger.Info("stopping...")
	for _, service := range n.services {
		service.Stop()
	}
	return nil
}

func (n *LocalNode) NetworkSecret() *secure.NetworkSecret {
	return n.secret
}

func (n *LocalNode) NetTable() *NetTable {
	service, ok := n.services["net-table"]
	if !ok {
		n.logger.Panic("net-table not found")
	}
	return service.(*NetTable)
}
<commit_msg>disable stun and upnp services<commit_after>package common

import (
	"fmt"
	"github.com/meshbird/meshbird/log"
	"github.com/meshbird/meshbird/secure"
	"sync"
)

type LocalNode struct {
	Node

	secret *secure.NetworkSecret
	config *Config
	state  *State

	mutex     sync.Mutex
	waitGroup sync.WaitGroup

	services map[string]Service

	logger log.Logger
}

func NewLocalNode(cfg *Config) (*LocalNode, error) {
	var err error
	n := new(LocalNode)
	n.logger = log.L("local")

	n.secret, err = secure.NetworkSecretUnmarshal(cfg.SecretKey)
	if err != nil {
		return nil, err
	}

	n.config = cfg
	n.config.NetworkID = n.secret.InfoHash()
	n.state = NewState(n.secret)

	n.services = make(map[string]Service)

	n.AddService(&NetTable{})
	n.AddService(&ListenerService{})
	n.AddService(&DiscoveryDHT{})
	n.AddService(&InterfaceService{})
	//n.AddService(&STUNService{})
	//n.AddService(&UPnPService{})
	n.AddService(&HttpService{})
	return n, nil
}

func (n *LocalNode) Config() Config {
	return *n.config
}

func (n *LocalNode) State() State {
	return *n.state
}

func (n *LocalNode) AddService(srv Service) {
	n.services[srv.Name()] = srv
}

func (n *LocalNode) Start() error {
	for _, service := range n.services {
		n.logger.Info("initializing %s service", service.Name())
		if err := service.Init(n); err != nil {
			return fmt.Errorf("initialization of %q finished with error: %v", service.Name(), err)
		}
		n.waitGroup.Add(1)

		go func(srv Service) {
			defer n.waitGroup.Done()
			n.logger.Info("running %q service", srv.Name())
			if err := srv.Run(); err != nil {
				n.logger.Error("error on run %q service, %v", srv.Name(), err)
			}
		}(service)
	}
	return nil
}

func (n *LocalNode) Service(name string) Service {
	service, ok := n.services[name]
	if !ok {
		n.logger.Fatal("service %q not found", name)
	}
	return service
}

func (n *LocalNode) WaitStop() {
	n.waitGroup.Wait()
}

func (n *LocalNode) Stop() error {
	n.logger.Info("stopping...")
	for _, service := range n.services {
		service.Stop()
	}
	return nil
}

func (n *LocalNode) NetworkSecret() *secure.NetworkSecret {
	return n.secret
}

func (n *LocalNode) NetTable() *NetTable {
	service, ok := n.services["net-table"]
	if !ok {
		n.logger.Panic("net-table not found")
	}
	return service.(*NetTable)
}
<|endoftext|>


<commit_before>package authentication

import (
	"context"
	"fmt"
	"log"
	"strings"

	"github.com/Azure/go-autorest/autorest"
	"github.com/Azure/go-autorest/autorest/adal"
	"github.com/manicminer/hamilton/auth"
	"github.com/manicminer/hamilton/environments"
)

// Config is the configuration structure used to instantiate a
// new Azure management client.
type Config struct {
	ClientID           string
	SubscriptionID     string
	TenantID           string
	AuxiliaryTenantIDs []string
	Environment        string
	MetadataHost       string

	GetAuthenticatedObjectID         func(context.Context) (*string, error)
	AuthenticatedAsAServicePrincipal bool

	// A Custom Resource Manager Endpoint
	// at this time this should only be applicable for Azure Stack.
	CustomResourceManagerEndpoint string

	// Beta opt-in for Microsoft Graph
	UseMicrosoftGraph bool

	authMethod authMethod
}

type OAuthConfig struct {
	OAuth            *adal.OAuthConfig
	MultiTenantOauth *adal.MultiTenantOAuthConfig
}

// GetAuthorizationToken returns an authorization token for the authentication method defined in the Config
func (c Config) GetOAuthConfig(activeDirectoryEndpoint string) (*adal.OAuthConfig, error) {
	log.Printf("Getting OAuth config for endpoint %s with  tenant %s", activeDirectoryEndpoint, c.TenantID)

	// fix for ADFS environments, if the login endpoint ends in `/adfs` it's an adfs environment
	// the login endpoint ends up residing in `ActiveDirectoryEndpoint`
	oAuthTenant := c.TenantID
	if strings.HasSuffix(strings.ToLower(activeDirectoryEndpoint), "/adfs") {
		log.Printf("[DEBUG] ADFS environment detected - overriding Tenant ID to `adfs`!")
		oAuthTenant = "adfs"
	}

	oauth, err := adal.NewOAuthConfig(activeDirectoryEndpoint, oAuthTenant)
	if err != nil {
		return nil, err
	}

	// OAuthConfigForTenant returns a pointer, which can be nil.
	if oauth == nil {
		return nil, fmt.Errorf("unable to configure OAuthConfig for tenant %s", c.TenantID)
	}

	return oauth, nil
}

// GetMultiTenantOAuthConfig returns a multi-tenant authorization token for the authentication method defined in the Config
func (c Config) GetMultiTenantOAuthConfig(activeDirectoryEndpoint string) (*adal.MultiTenantOAuthConfig, error) {
	log.Printf("Getting multi OAuth config for endpoint %s with  tenant %s (aux tenants: %v)", activeDirectoryEndpoint, c.TenantID, c.AuxiliaryTenantIDs)
	oauth, err := adal.NewMultiTenantOAuthConfig(activeDirectoryEndpoint, c.TenantID, c.AuxiliaryTenantIDs, adal.OAuthOptions{})
	if err != nil {
		return nil, err
	}

	// OAuthConfigForTenant returns a pointer, which can be nil.
	if oauth == nil {
		return nil, fmt.Errorf("unable to configure OAuthConfig for tenant %s (auxiliary tenants %v)", c.TenantID, c.AuxiliaryTenantIDs)
	}

	return &oauth, nil
}

// BuildOAuthConfig builds the authorization configuration for the specified Active Directory Endpoint
func (c Config) BuildOAuthConfig(activeDirectoryEndpoint string) (*OAuthConfig, error) {
	multiAuth := OAuthConfig{}
	var err error

	multiAuth.OAuth, err = c.GetOAuthConfig(activeDirectoryEndpoint)
	if err != nil {
		return nil, err
	}

	if len(c.AuxiliaryTenantIDs) > 0 {
		multiAuth.MultiTenantOauth, err = c.GetMultiTenantOAuthConfig(activeDirectoryEndpoint)
		if err != nil {
			return nil, err
		}
	}

	return &multiAuth, nil
}

// ADALBearerAuthorizerCallback returns a BearerAuthorizer valid only for the Primary Tenant
// this signs a request using the AccessToken returned from the primary Resource Manager authorizer
func (c Config) ADALBearerAuthorizerCallback(ctx context.Context, sender autorest.Sender, oauthConfig *OAuthConfig) *autorest.BearerAuthorizerCallback {
	return autorest.NewBearerAuthorizerCallback(sender, func(tenantID, resource string) (*autorest.BearerAuthorizer, error) {
		// a BearerAuthorizer is only valid for the primary tenant
		newAuthConfig := &OAuthConfig{
			OAuth: oauthConfig.OAuth,
		}

		storageSpt, err := c.GetADALToken(ctx, sender, newAuthConfig, resource)
		if err != nil {
			return nil, err
		}

		cast, ok := storageSpt.(*autorest.BearerAuthorizer)
		if !ok {
			return nil, fmt.Errorf("converting %+v to a BearerAuthorizer", storageSpt)
		}

		return cast, nil
	})
}

// MSALBearerAuthorizerCallback returns a BearerAuthorizer valid only for the Primary Tenant
// this signs a request using the AccessToken returned from the primary Resource Manager authorizer
func (c Config) MSALBearerAuthorizerCallback(ctx context.Context, api environments.Api, sender autorest.Sender, oauthConfig *OAuthConfig, endpoint string) *autorest.BearerAuthorizerCallback {
	authorizer, err := c.GetMSALToken(ctx, api, sender, oauthConfig, endpoint)
	if err != nil {
		return autorest.NewBearerAuthorizerCallback(nil, func(_, _ string) (*autorest.BearerAuthorizer, error) {
			return nil, fmt.Errorf("failed to acquire MSAL token for %s", api.Endpoint)
		})
	}

	cast, ok := authorizer.(*auth.CachedAuthorizer)
	if !ok {
		return autorest.NewBearerAuthorizerCallback(nil, func(_, _ string) (*autorest.BearerAuthorizer, error) {
			return nil, fmt.Errorf("authorizer was not an auth.CachedAuthorizer for %s", api.Endpoint)
		})
	}

	return cast.BearerAuthorizerCallback()
}

// GetADALToken returns an autorest.Authorizer using an ADAL token via the authentication method defined in the Config
func (c Config) GetADALToken(ctx context.Context, sender autorest.Sender, oauth *OAuthConfig, endpoint string) (autorest.Authorizer, error) {
	return c.authMethod.getADALToken(ctx, sender, oauth, endpoint)
}

// GetMSALToken returns an autorest.Authorizer using an MSAL token via the authentication method defined in the Config
func (c Config) GetMSALToken(ctx context.Context, api environments.Api, sender autorest.Sender, oauth *OAuthConfig, endpoint string) (autorest.Authorizer, error) {
	return c.authMethod.getMSALToken(ctx, api, sender, oauth, endpoint)
}
<commit_msg>Bugfix: add compatiblity for ADAL tokens in auth.Config.MSALBearerAuthorizerCallback()<commit_after>package authentication

import (
	"context"
	"fmt"
	"log"
	"strings"

	"github.com/Azure/go-autorest/autorest"
	"github.com/Azure/go-autorest/autorest/adal"
	"github.com/manicminer/hamilton/auth"
	"github.com/manicminer/hamilton/environments"
)

// Config is the configuration structure used to instantiate a
// new Azure management client.
type Config struct {
	ClientID           string
	SubscriptionID     string
	TenantID           string
	AuxiliaryTenantIDs []string
	Environment        string
	MetadataHost       string

	GetAuthenticatedObjectID         func(context.Context) (*string, error)
	AuthenticatedAsAServicePrincipal bool

	// A Custom Resource Manager Endpoint
	// at this time this should only be applicable for Azure Stack.
	CustomResourceManagerEndpoint string

	// Beta opt-in for Microsoft Graph
	UseMicrosoftGraph bool

	authMethod authMethod
}

type OAuthConfig struct {
	OAuth            *adal.OAuthConfig
	MultiTenantOauth *adal.MultiTenantOAuthConfig
}

// GetAuthorizationToken returns an authorization token for the authentication method defined in the Config
func (c Config) GetOAuthConfig(activeDirectoryEndpoint string) (*adal.OAuthConfig, error) {
	log.Printf("Getting OAuth config for endpoint %s with  tenant %s", activeDirectoryEndpoint, c.TenantID)

	// fix for ADFS environments, if the login endpoint ends in `/adfs` it's an adfs environment
	// the login endpoint ends up residing in `ActiveDirectoryEndpoint`
	oAuthTenant := c.TenantID
	if strings.HasSuffix(strings.ToLower(activeDirectoryEndpoint), "/adfs") {
		log.Printf("[DEBUG] ADFS environment detected - overriding Tenant ID to `adfs`!")
		oAuthTenant = "adfs"
	}

	oauth, err := adal.NewOAuthConfig(activeDirectoryEndpoint, oAuthTenant)
	if err != nil {
		return nil, err
	}

	// OAuthConfigForTenant returns a pointer, which can be nil.
	if oauth == nil {
		return nil, fmt.Errorf("unable to configure OAuthConfig for tenant %s", c.TenantID)
	}

	return oauth, nil
}

// GetMultiTenantOAuthConfig returns a multi-tenant authorization token for the authentication method defined in the Config
func (c Config) GetMultiTenantOAuthConfig(activeDirectoryEndpoint string) (*adal.MultiTenantOAuthConfig, error) {
	log.Printf("Getting multi OAuth config for endpoint %s with  tenant %s (aux tenants: %v)", activeDirectoryEndpoint, c.TenantID, c.AuxiliaryTenantIDs)
	oauth, err := adal.NewMultiTenantOAuthConfig(activeDirectoryEndpoint, c.TenantID, c.AuxiliaryTenantIDs, adal.OAuthOptions{})
	if err != nil {
		return nil, err
	}

	// OAuthConfigForTenant returns a pointer, which can be nil.
	if oauth == nil {
		return nil, fmt.Errorf("unable to configure OAuthConfig for tenant %s (auxiliary tenants %v)", c.TenantID, c.AuxiliaryTenantIDs)
	}

	return &oauth, nil
}

// BuildOAuthConfig builds the authorization configuration for the specified Active Directory Endpoint
func (c Config) BuildOAuthConfig(activeDirectoryEndpoint string) (*OAuthConfig, error) {
	multiAuth := OAuthConfig{}
	var err error

	multiAuth.OAuth, err = c.GetOAuthConfig(activeDirectoryEndpoint)
	if err != nil {
		return nil, err
	}

	if len(c.AuxiliaryTenantIDs) > 0 {
		multiAuth.MultiTenantOauth, err = c.GetMultiTenantOAuthConfig(activeDirectoryEndpoint)
		if err != nil {
			return nil, err
		}
	}

	return &multiAuth, nil
}

// ADALBearerAuthorizerCallback returns a BearerAuthorizer valid only for the Primary Tenant
// this signs a request using the AccessToken returned from the primary Resource Manager authorizer
func (c Config) ADALBearerAuthorizerCallback(ctx context.Context, sender autorest.Sender, oauthConfig *OAuthConfig) *autorest.BearerAuthorizerCallback {
	return autorest.NewBearerAuthorizerCallback(sender, func(tenantID, resource string) (*autorest.BearerAuthorizer, error) {
		// a BearerAuthorizer is only valid for the primary tenant
		newAuthConfig := &OAuthConfig{
			OAuth: oauthConfig.OAuth,
		}

		storageSpt, err := c.GetADALToken(ctx, sender, newAuthConfig, resource)
		if err != nil {
			return nil, err
		}

		cast, ok := storageSpt.(*autorest.BearerAuthorizer)
		if !ok {
			return nil, fmt.Errorf("converting %+v to a BearerAuthorizer", storageSpt)
		}

		return cast, nil
	})
}

// MSALBearerAuthorizerCallback returns a BearerAuthorizer valid only for the Primary Tenant
// this signs a request using the AccessToken returned from the primary Resource Manager authorizer
func (c Config) MSALBearerAuthorizerCallback(ctx context.Context, api environments.Api, sender autorest.Sender, oauthConfig *OAuthConfig, endpoint string) *autorest.BearerAuthorizerCallback {
	authorizer, err := c.GetMSALToken(ctx, api, sender, oauthConfig, endpoint)
	if err != nil {
		return autorest.NewBearerAuthorizerCallback(nil, func(_, _ string) (*autorest.BearerAuthorizer, error) {
			return nil, fmt.Errorf("failed to acquire MSAL token for %s", api.Endpoint)
		})
	}

	// For compatibility with Azure CLI which still uses ADAL, first check if we got an autorest.BearerAuthorizer
	if cast, ok := authorizer.(*autorest.BearerAuthorizer); ok {
		return autorest.NewBearerAuthorizerCallback(sender, func(_, _ string) (*autorest.BearerAuthorizer, error) {
			return cast, nil
		})
	}

	cast, ok := authorizer.(*auth.CachedAuthorizer)
	if !ok {
		return autorest.NewBearerAuthorizerCallback(nil, func(_, _ string) (*autorest.BearerAuthorizer, error) {
			return nil, fmt.Errorf("authorizer was not an auth.CachedAuthorizer for %s", api.Endpoint)
		})
	}

	return cast.BearerAuthorizerCallback()
}

// GetADALToken returns an autorest.Authorizer using an ADAL token via the authentication method defined in the Config
func (c Config) GetADALToken(ctx context.Context, sender autorest.Sender, oauth *OAuthConfig, endpoint string) (autorest.Authorizer, error) {
	return c.authMethod.getADALToken(ctx, sender, oauth, endpoint)
}

// GetMSALToken returns an autorest.Authorizer using an MSAL token via the authentication method defined in the Config
func (c Config) GetMSALToken(ctx context.Context, api environments.Api, sender autorest.Sender, oauth *OAuthConfig, endpoint string) (autorest.Authorizer, error) {
	return c.authMethod.getMSALToken(ctx, api, sender, oauth, endpoint)
}
<|endoftext|>


<commit_before>package common

import (
	"log"
	"net"
	"sync"
	"time"
)

type NetTable struct {
	BaseService

	localNode *LocalNode
	waitGroup sync.WaitGroup
	dhtInChan chan string

	lock      sync.RWMutex
	blackList map[string]time.Time
	peers     map[string]*RemoteNode
}

func (nt NetTable) Name() string {
	return "net-table"
}

func (nt *NetTable) Init(ln *LocalNode) error {
	nt.localNode = ln
	nt.dhtInChan = make(chan string, 10)
	nt.blackList = make(map[string]time.Time)
	nt.peers = make(map[string]*RemoteNode)
	return nil
}

func (nt *NetTable) Run() error {
	for i := 0; i < 10; i++ {
		go nt.processDHTIn()
	}
	return nil
}

func (nt *NetTable) Stop() {
	nt.SetStatus(StatusStopping)
}

func (nt *NetTable) GetDHTInChannel() chan<- string {
	return nt.dhtInChan
}

func (nt *NetTable) RemoteNodeByIP(ip net.IP) *RemoteNode {
	nt.lock.Lock()
	defer nt.lock.Unlock()
	return nt.peers[ip.String()]
}

func (nt *NetTable) AddRemoteNode(rn *RemoteNode) {
	nt.lock.Lock()
	defer nt.lock.Unlock()
	log.Printf("add remote node: %s/%s", rn.privateIP.String(), rn.publicAddress)
	nt.peers[rn.privateIP.String()] = rn
}

func (nt *NetTable) processDHTIn() {
	for nt.Status() != StatusStopping {
		select {
		case host, ok := <-nt.dhtInChan:
			if !ok {
				return
			}
			nt.lock.Lock()
			_, ok = nt.blackList[host]
			if !ok {
				for _, peer := range nt.peers {
					if peer.publicAddress == host {
						ok = true
						break
					}
				}
			}
			nt.lock.Unlock()

			if !ok {
				nt.tryConnect(host)
			}
		}
	}
}

func (nt *NetTable) tryConnect(h string) {
	rn, err := TryConnect(h, nt.localNode.NetworkSecret())
	if err != nil {
		nt.addToBlackList(h)
		return
	}
	nt.lock.Lock()
	defer nt.lock.Unlock()
	nt.peers[rn.privateIP.String()] = rn
}

func (nt *NetTable) addToBlackList(h string) {
	nt.lock.Lock()
	defer nt.lock.Unlock()
	nt.blackList[h] = time.Now()
}
<commit_msg>add empty common/NetTable.SendPacket(dstIP net.IP, payload []byte)<commit_after>package common

import (
	"log"
	"net"
	"sync"
	"time"
)

type NetTable struct {
	BaseService

	localNode *LocalNode
	waitGroup sync.WaitGroup
	dhtInChan chan string

	lock      sync.RWMutex
	blackList map[string]time.Time
	peers     map[string]*RemoteNode
}

func (nt NetTable) Name() string {
	return "net-table"
}

func (nt *NetTable) Init(ln *LocalNode) error {
	nt.localNode = ln
	nt.dhtInChan = make(chan string, 10)
	nt.blackList = make(map[string]time.Time)
	nt.peers = make(map[string]*RemoteNode)
	return nil
}

func (nt *NetTable) Run() error {
	for i := 0; i < 10; i++ {
		go nt.processDHTIn()
	}
	return nil
}

func (nt *NetTable) Stop() {
	nt.SetStatus(StatusStopping)
}

func (nt *NetTable) GetDHTInChannel() chan<- string {
	return nt.dhtInChan
}

func (nt *NetTable) RemoteNodeByIP(ip net.IP) *RemoteNode {
	nt.lock.Lock()
	defer nt.lock.Unlock()
	return nt.peers[ip.String()]
}

func (nt *NetTable) AddRemoteNode(rn *RemoteNode) {
	nt.lock.Lock()
	defer nt.lock.Unlock()
	log.Printf("add remote node: %s/%s", rn.privateIP.String(), rn.publicAddress)
	nt.peers[rn.privateIP.String()] = rn
}

func (nt *NetTable) processDHTIn() {
	for nt.Status() != StatusStopping {
		select {
		case host, ok := <-nt.dhtInChan:
			if !ok {
				return
			}
			nt.lock.Lock()
			_, ok = nt.blackList[host]
			if !ok {
				for _, peer := range nt.peers {
					if peer.publicAddress == host {
						ok = true
						break
					}
				}
			}
			nt.lock.Unlock()

			if !ok {
				nt.tryConnect(host)
			}
		}
	}
}

func (nt *NetTable) tryConnect(h string) {
	rn, err := TryConnect(h, nt.localNode.NetworkSecret())
	if err != nil {
		nt.addToBlackList(h)
		return
	}
	nt.lock.Lock()
	defer nt.lock.Unlock()
	nt.peers[rn.privateIP.String()] = rn
}

func (nt *NetTable) addToBlackList(h string) {
	nt.lock.Lock()
	defer nt.lock.Unlock()
	nt.blackList[h] = time.Now()
}

func (nt *NetTable) SendPacket(dstIP net.IP, payload []byte) {

}
<|endoftext|>


<commit_before>package cheshire

import (
	"encoding/json"
	"fmt"
	"github.com/trendrr/goshire/dynmap"
	"log"
	"net/http"
	// "net/url"
	"sync"
	// "bytes"
)

type HttpWriter struct {
	Writer        http.ResponseWriter
	HttpRequest   *http.Request
	Request       *Request
	ServerConfig  *ServerConfig
	headerWritten sync.Once
}

func (this *HttpWriter) Type() string {
	return "http"
}

func (conn *HttpWriter) Write(response *Response) (int, error) {
	bytes := 0
	json, err := json.Marshal(response)
	if err != nil {
		//TODO: uhh, do something..
		log.Print(err)
	}
	conn.headerWritten.Do(func() {
		conn.Writer.Header().Set("Content-Type", "application/json")
		conn.Writer.WriteHeader(response.StatusCode())
	})
	b, err := conn.Writer.Write(json)
	if err != nil {
		return bytes, err
	}
	bytes += b
	b, err = conn.Writer.Write([]byte("\n"))
	if err != nil {
		return bytes, err
	}
	bytes += b

	flusher, ok := conn.Writer.(http.Flusher)
	if !ok {
		return bytes, fmt.Errorf("Wrong type in http writer!")
	}
	flusher.Flush()
	return bytes, err
}

type httpHandler struct {
	serverConfig *ServerConfig
}

// Implement this interface for a controller to skip the normal cheshire life cycle
// This should be only used in special cases (static file serving, websockets, ect)
// controllers that implement this interface will skip the HandleRequest function alltogether
type HttpHijacker interface {
	HttpHijack(writer http.ResponseWriter, req *http.Request, serverConfig *ServerConfig)
}

func (this *httpHandler) ServeHTTP(writer http.ResponseWriter, req *http.Request) {
	controller := this.serverConfig.Router.Match(req.Method, req.URL.Path)

	//check if controller is the special HttpHijacker.
	h, hijack := controller.(HttpHijacker)
	if hijack {
		h.HttpHijack(writer, req, this.serverConfig)
		return
	}

	//we are already in a go routine, so no need to start another one.
	request := ToStrestRequest(req)

	conn := &HttpWriter{
		Writer:       writer,
		HttpRequest:  req,
		Request:      request,
		ServerConfig: this.serverConfig,
	}
	HandleRequest(request, conn, controller, this.serverConfig)
}

func ToStrestRequest(req *http.Request) *Request {

	//print out the http request.
	// log.Println("*******************")
	// var doc bytes.Buffer
	// req.Write(&doc)
	// log.Println(doc.String())
	// log.Println("*******************")

	var request = NewRequest(req.URL.Path, req.Method)
	request.SetUri(req.URL.Path)
	request.SetMethod(req.Method)
	request.SetTxnId(req.Header.Get("Strest-Txn-Id"))
	request.SetTxnAccept(req.Header.Get("Strest-Txn-Accept"))
	if len(request.TxnAccept()) == 0 {
		request.SetTxnAccept("single")
	}

	err := req.ParseForm()
	if err != nil {
		log.Printf("Error parsing form values: %s", err)
	}
	params := dynmap.New()
	err = params.UnmarshalURLValues(req.Form)
	if err != nil {
		log.Printf("Error parsing form values: %s", err)
	}
	request.SetParams(params)
	return request
}

func HttpListen(port int, serverConfig *ServerConfig) error {
	handler := &httpHandler{serverConfig}

	log.Println("HTTP Listener on port: ", port)
	return http.ListenAndServe(fmt.Sprintf(":%d", port), handler)
}
<commit_msg>handle json encoded posts and puts<commit_after>package cheshire

import (
	"encoding/json"
	"fmt"
	"github.com/trendrr/goshire/dynmap"
	"log"
	"net/http"
	// "net/url"
	"sync"
	"io"
	"strings"
	"io/ioutil"
	// "bytes"
)

type HttpWriter struct {
	Writer        http.ResponseWriter
	HttpRequest   *http.Request
	Request       *Request
	ServerConfig  *ServerConfig
	headerWritten sync.Once
}

func (this *HttpWriter) Type() string {
	return "http"
}

func (conn *HttpWriter) Write(response *Response) (int, error) {
	bytes := 0
	json, err := json.Marshal(response)
	if err != nil {
		//TODO: uhh, do something..
		log.Print(err)
	}
	conn.headerWritten.Do(func() {
		conn.Writer.Header().Set("Content-Type", "application/json")
		conn.Writer.WriteHeader(response.StatusCode())
	})
	b, err := conn.Writer.Write(json)
	if err != nil {
		return bytes, err
	}
	bytes += b
	b, err = conn.Writer.Write([]byte("\n"))
	if err != nil {
		return bytes, err
	}
	bytes += b

	flusher, ok := conn.Writer.(http.Flusher)
	if !ok {
		return bytes, fmt.Errorf("Wrong type in http writer!")
	}
	flusher.Flush()
	return bytes, err
}

type httpHandler struct {
	serverConfig *ServerConfig
}

// Implement this interface for a controller to skip the normal cheshire life cycle
// This should be only used in special cases (static file serving, websockets, ect)
// controllers that implement this interface will skip the HandleRequest function alltogether
type HttpHijacker interface {
	HttpHijack(writer http.ResponseWriter, req *http.Request, serverConfig *ServerConfig)
}

func (this *httpHandler) ServeHTTP(writer http.ResponseWriter, req *http.Request) {
	controller := this.serverConfig.Router.Match(req.Method, req.URL.Path)

	//check if controller is the special HttpHijacker.
	h, hijack := controller.(HttpHijacker)
	if hijack {
		h.HttpHijack(writer, req, this.serverConfig)
		return
	}

	//we are already in a go routine, so no need to start another one.
	request := ToStrestRequest(req)

	conn := &HttpWriter{
		Writer:       writer,
		HttpRequest:  req,
		Request:      request,
		ServerConfig: this.serverConfig,
	}
	HandleRequest(request, conn, controller, this.serverConfig)
}

func ToStrestRequest(req *http.Request) *Request {

	//print out the http request.
	// log.Println("*******************")
	// var doc bytes.Buffer
	// req.Write(&doc)
	// log.Println(doc.String())
	// log.Println("*******************")

	var request = NewRequest(req.URL.Path, req.Method)
	request.SetUri(req.URL.Path)
	request.SetMethod(req.Method)
	request.SetTxnId(req.Header.Get("Strest-Txn-Id"))
	request.SetTxnAccept(req.Header.Get("Strest-Txn-Accept"))
	if len(request.TxnAccept()) == 0 {
		request.SetTxnAccept("single")
	}

	//deal with the params

	params := dynmap.New()

	//we always do parse form since it will handle the
	// url params as well
	err := req.ParseForm()
	if err != nil {
		log.Printf("Error parsing form values: %s", err)
	}
	err = params.UnmarshalURLValues(req.Form)
	if err != nil {
		log.Printf("Error parsing form values: %s", err)
	}

	//now deal with possible different content types.
	ct := req.Header.Get("Content-Type")
	switch {
	case strings.Contains(ct, "json") :
		//parse as json
		bytes, err := ReadHttpBody(req)
		if err != nil {
			log.Printf("ERROR reading body %s", err)
		}
		err = params.UnmarshalJSON(bytes)
	}

	request.SetParams(params)
	return request
}

// reads the whole body of an http request
func ReadHttpBody(req *http.Request) ([]byte, error) {
	var reader io.Reader = req.Body
   	maxFormSize := int64(1<<63 - 1)
    maxFormSize = int64(10 << 20) // 10 MB is a lot of text.
   	reader = io.LimitReader(req.Body, maxFormSize+1)
	b, e := ioutil.ReadAll(reader)
	return b, e
}

func HttpListen(port int, serverConfig *ServerConfig) error {
	handler := &httpHandler{serverConfig}

	log.Println("HTTP Listener on port: ", port)
	return http.ListenAndServe(fmt.Sprintf(":%d", port), handler)
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/gcfg"
	"flag"
	"github.com/anchor/picolog"
	"log"
	"os"
)

const (
	Version = "0.3.0"
)

var Logger *picolog.Logger

func main() {
	var cfg Config
	configFile := flag.String("cfg", "/etc/chevalier.gcfg", "Path to configuration file. This file should be in gcfg[0] format. [0] https://code.google.com/p/gcfg/")
	indexerMode := flag.Bool("index", false, "Start indexer mode.")
	indexOnce := flag.Bool("index-once", false, "Run indexer once and then exit.")
	readerMode := flag.Bool("read", false, "Start reader mode. Reader mode will invoke the indexer once on startup; use -no-index to disable.")
	noIndex := flag.Bool("no-index", false, "Do not index once at startup when started in reader mode.")
	logFile := flag.String("log-file", "", "If set, log to this file rather than stdout.")
	debug := flag.Bool("debug", false, "Enable debug logging (overrides configured log level).")
	flag.Parse()
	err := gcfg.ReadFileInto(&cfg, *configFile)
	if err != nil {
		log.Fatalf("Could not read config file at %v: %v", *configFile, err)
	}
	logLevel, err := picolog.ParseLogLevel(cfg.Chevalier.LogLevel)
	if err != nil {
		log.Fatalf("Could not parse log level: %v", err)
	}
	if *debug {
		logLevel = picolog.LogDebug
	}
	logStream := os.Stdout
	if *logFile != "" {
		logStream, err = os.OpenFile(*logFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660)
		if err != nil {
			log.Fatalf("Could not open log file %v: %v", *logFile, err)
		}
	}
	Logger = picolog.NewLogger(logLevel, "chevalier", logStream)
	if *indexerMode {
		RunIndexer(cfg)
	} else if *readerMode {
		if !*noIndex {
			go RunIndexerOnce(cfg)
		}
		go RunStatus(cfg)
		RunReader(cfg)
	} else if *indexOnce {
		RunIndexerOnce(cfg)
	} else {
		Logger.Fatalf("Must specify one of -index, -read or -index-once.")
	}
}
<commit_msg>Bump to 1.0.0<commit_after>package main

import (
	"code.google.com/p/gcfg"
	"flag"
	"github.com/anchor/picolog"
	"log"
	"os"
)

const (
	Version = "1.0.0"
)

var Logger *picolog.Logger

func main() {
	var cfg Config
	configFile := flag.String("cfg", "/etc/chevalier.gcfg", "Path to configuration file. This file should be in gcfg[0] format. [0] https://code.google.com/p/gcfg/")
	indexerMode := flag.Bool("index", false, "Start indexer mode.")
	indexOnce := flag.Bool("index-once", false, "Run indexer once and then exit.")
	readerMode := flag.Bool("read", false, "Start reader mode. Reader mode will invoke the indexer once on startup; use -no-index to disable.")
	noIndex := flag.Bool("no-index", false, "Do not index once at startup when started in reader mode.")
	logFile := flag.String("log-file", "", "If set, log to this file rather than stdout.")
	debug := flag.Bool("debug", false, "Enable debug logging (overrides configured log level).")
	flag.Parse()
	err := gcfg.ReadFileInto(&cfg, *configFile)
	if err != nil {
		log.Fatalf("Could not read config file at %v: %v", *configFile, err)
	}
	logLevel, err := picolog.ParseLogLevel(cfg.Chevalier.LogLevel)
	if err != nil {
		log.Fatalf("Could not parse log level: %v", err)
	}
	if *debug {
		logLevel = picolog.LogDebug
	}
	logStream := os.Stdout
	if *logFile != "" {
		logStream, err = os.OpenFile(*logFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660)
		if err != nil {
			log.Fatalf("Could not open log file %v: %v", *logFile, err)
		}
	}
	Logger = picolog.NewLogger(logLevel, "chevalier", logStream)
	if *indexerMode {
		RunIndexer(cfg)
	} else if *readerMode {
		if !*noIndex {
			go RunIndexerOnce(cfg)
		}
		go RunStatus(cfg)
		RunReader(cfg)
	} else if *indexOnce {
		RunIndexerOnce(cfg)
	} else {
		Logger.Fatalf("Must specify one of -index, -read or -index-once.")
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Mikio Hara. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin freebsd linux netbsd

package tcp_test

import (
	"net"
	"runtime"
	"sync"
	"testing"

	"github.com/mikioh/tcp"
)

func server(t *testing.T, ln net.Listener) {
	c, err := ln.Accept()
	if err != nil {
		return
	}
	go server(t, ln)
	defer c.Close()
	var b [1]byte
	if _, err := c.Read(b[:]); err != nil {
		t.Error(err)
		return
	}
	if _, err := c.Write(b[:]); err != nil {
		t.Error(err)
		return
	}
}

func TestConcurrentReadWriteAndInfo(t *testing.T) {
	switch runtime.GOOS {
	case "darwin":
		vers := kernelVersion()
		if vers == nil || vers[0] < 15 {
			t.Skip("not supported on this version: %v, %s/%s", vers, runtime.GOOS, runtime.GOARCH)
		}
	case "freebsd", "linux", "netbsd":
	default:
		t.Skipf("%s/%s", runtime.GOOS, runtime.GOARCH)
	}

	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	defer ln.Close()
	go server(t, ln)

	const N = 10
	var wg sync.WaitGroup
	wg.Add(N)
	for i := 0; i < N; i++ {
		go func() {
			defer wg.Done()
			var d net.Dialer
			c, err := d.Dial(ln.Addr().Network(), ln.Addr().String())
			if err != nil {
				t.Error(err)
				return
			}
			defer c.Close()
			tc, err := tcp.NewConn(c)
			if err != nil {
				t.Error(err)
				return
			}
			var wwg sync.WaitGroup
			wwg.Add(1)
			go func() {
				defer wwg.Done()
				var b [1]byte
				if _, err := c.Write(b[:]); err != nil {
					t.Error(err)
					return
				}
				if _, err := c.Read(b[:]); err != nil {
					t.Error(err)
					return
				}
			}()
			wwg.Add(1)
			go func() {
				defer wwg.Done()
				if _, err := tc.Info(); err != nil {
					t.Error(err)
					return
				}
			}()
			wwg.Wait()
		}()
	}
	wg.Wait()
}
<commit_msg>tcp: fix nit in test<commit_after>// Copyright 2016 Mikio Hara. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin freebsd linux netbsd

package tcp_test

import (
	"net"
	"runtime"
	"sync"
	"testing"

	"github.com/mikioh/tcp"
)

func server(t *testing.T, ln net.Listener) {
	c, err := ln.Accept()
	if err != nil {
		return
	}
	go server(t, ln)
	defer c.Close()
	var b [1]byte
	if _, err := c.Read(b[:]); err != nil {
		t.Error(err)
		return
	}
	if _, err := c.Write(b[:]); err != nil {
		t.Error(err)
		return
	}
}

func TestConcurrentReadWriteAndInfo(t *testing.T) {
	switch runtime.GOOS {
	case "darwin":
		vers := kernelVersion()
		if vers == nil || vers[0] < 15 {
			t.Skipf("%v, %s/%s", vers, runtime.GOOS, runtime.GOARCH)
		}
	case "freebsd", "linux", "netbsd":
	default:
		t.Skipf("%s/%s", runtime.GOOS, runtime.GOARCH)
	}

	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	defer ln.Close()
	go server(t, ln)

	const N = 10
	var wg sync.WaitGroup
	wg.Add(N)
	for i := 0; i < N; i++ {
		go func() {
			defer wg.Done()
			var d net.Dialer
			c, err := d.Dial(ln.Addr().Network(), ln.Addr().String())
			if err != nil {
				t.Error(err)
				return
			}
			defer c.Close()
			tc, err := tcp.NewConn(c)
			if err != nil {
				t.Error(err)
				return
			}
			var wwg sync.WaitGroup
			wwg.Add(1)
			go func() {
				defer wwg.Done()
				var b [1]byte
				if _, err := c.Write(b[:]); err != nil {
					t.Error(err)
					return
				}
				if _, err := c.Read(b[:]); err != nil {
					t.Error(err)
					return
				}
			}()
			wwg.Add(1)
			go func() {
				defer wwg.Done()
				if _, err := tc.Info(); err != nil {
					t.Error(err)
					return
				}
			}()
			wwg.Wait()
		}()
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>// Copyright 2018 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"os/exec"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
)

// Ensure WindowsStrategy implements PlatformStrategy.
var _ PlatformStrategy = &WindowsStrategy{}

// WindowsStrategy is a Windows-specific PlatformStrategy.
// Implements PlatformStrategy.
type WindowsStrategy struct {
}

// startupTmpl is the Swarming bot startup task command template.
const startupTmpl = "C:\\tools\\python\\bin\\python.exe {{.BotCode}} start_bot"

// startupTask is the name of the Swarming bot startup task.
const startupTask = "swarming-start-bot"

// autostart configures the given Swarming bot code to be executed on startup
// for the given user, then starts the Swarming bot process.
// Implements PlatformStrategy.
func (*WindowsStrategy) autostart(c context.Context, path, user string) error {
	subs := map[string]string{
		"BotCode": path,
		"User":    user,
	}
	s, err := substitute(c, startupTmpl, subs)
	if err != nil {
		return errors.Annotate(err, "failed to prepare template: %s", startupTmpl).Err()
	}

	logging.Infof(c, "installing %q", startupTask)
	// Prevent the Swarming bot process from configuring its own autostart.
	if err := exec.Command("setx", "/m", "SWARMING_EXTERNAL_BOT_SETUP", "1").Run(); err != nil {
		return errors.Annotate(err, "failed to setx environment variable").Err()
	}
	if err := exec.Command("schtasks", "/create", "/f", "/rl", "HIGHEST", "/ru", user, "/sc", "onstart", "/tn", startupTask, "/tr", s).Run(); err != nil {
		return errors.Annotate(err, "failed to create task %q", startupTask).Err()
	}

	logging.Infof(c, "starting %q", startupTask)
	if err := exec.Command("schtasks", "/run", "/tn", startupTask).Run(); err != nil {
		return errors.Annotate(err, "failed to start task %q", startupTask).Err()
	}
	return nil
}

// chown modifies the given path to be owned by the given user.
// Implements PlatformStrategy.
func (*WindowsStrategy) chown(c context.Context, path, username string) error {
	if err := exec.Command("icacls", path, "/setowner", username).Run(); err != nil {
		return errors.Annotate(err, "failed to set owner: %s", path).Err()
	}
	return nil
}
<commit_msg>[GCE] Start the Swarming bot process on logon on Windows<commit_after>// Copyright 2018 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"os/exec"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
)

// Ensure WindowsStrategy implements PlatformStrategy.
var _ PlatformStrategy = &WindowsStrategy{}

// WindowsStrategy is a Windows-specific PlatformStrategy.
// Implements PlatformStrategy.
type WindowsStrategy struct {
}

// startupTmpl is the Swarming bot startup task command template.
const startupTmpl = "C:\\tools\\python\\bin\\python.exe {{.BotCode}} start_bot"

// startupTask is the name of the Swarming bot startup task.
const startupTask = "swarming-start-bot"

// autostart configures the given Swarming bot code to be executed on startup
// for the given user, then starts the Swarming bot process.
// Implements PlatformStrategy.
func (*WindowsStrategy) autostart(c context.Context, path, user string) error {
	subs := map[string]string{
		"BotCode": path,
		"User":    user,
	}
	s, err := substitute(c, startupTmpl, subs)
	if err != nil {
		return errors.Annotate(err, "failed to prepare template: %s", startupTmpl).Err()
	}

	logging.Infof(c, "installing %q", startupTask)
	// Prevent the Swarming bot process from configuring its own autostart.
	if err := exec.Command("setx", "/m", "SWARMING_EXTERNAL_BOT_SETUP", "1").Run(); err != nil {
		return errors.Annotate(err, "failed to setx environment variable").Err()
	}
	if err := exec.Command("schtasks", "/create", "/f", "/it", "/rl", "HIGHEST", "/ru", user, "/sc", "onlogon", "/tn", startupTask, "/tr", s).Run(); err != nil {
		return errors.Annotate(err, "failed to create task %q", startupTask).Err()
	}

	logging.Infof(c, "starting %q", startupTask)
	if err := exec.Command("schtasks", "/run", "/tn", startupTask).Run(); err != nil {
		return errors.Annotate(err, "failed to start task %q", startupTask).Err()
	}
	return nil
}

// chown modifies the given path to be owned by the given user.
// Implements PlatformStrategy.
func (*WindowsStrategy) chown(c context.Context, path, username string) error {
	if err := exec.Command("icacls", path, "/setowner", username).Run(); err != nil {
		return errors.Annotate(err, "failed to set owner: %s", path).Err()
	}
	return nil
}
<|endoftext|>


<commit_before>package koding

import (
	"fmt"
	"koding/db/mongodb"
	"strconv"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	aws "koding/kites/kloud/api/amazon"
	"koding/kites/kloud/klient"
	"koding/kites/kloud/machinestate"
	"koding/kites/kloud/protocol"
	"koding/kites/kloud/provider/amazon"

	"github.com/koding/kite"
	"github.com/koding/logging"
	"github.com/mitchellh/goamz/ec2"
)

// Checker checks various aspects of a machine. It is used for limiting certain
// aspects of a machine, such as the total allowed machine count, storage size
// and etc.
type Checker interface {
	// Total checks whether the user has reached the current plan's limit of
	// having a total number numbers of machines. It returns an error if the
	// limit is reached or an unexplained error happaned.
	Total() error

	// AlwaysOn checks whether the given machine has reached the current plans
	// always on limit
	AlwaysOn() error

	// Timeout checks whether the user has reached the current plan's
	// inactivity timeout.
	Timeout() error

	// Storage checks whether the user has reached the current plan's limit
	// total storage with the supplied wantStorage information. It returns an
	// error if the limit is reached or an unexplained error happaned.
	Storage(wantStorage int) error

	// AllowedInstances checks whether the given machine has the permisison to
	// create the given instance type
	AllowedInstances(wantInstance InstanceType) error
}

type PlanChecker struct {
	Api      *amazon.AmazonClient
	DB       *mongodb.MongoDB
	Machine  *protocol.Machine
	Provider *Provider
	Kite     *kite.Kite
	Username string
	Log      logging.Logger
	Plan     Plan
}

func (p *PlanChecker) AllowedInstances(wantInstance InstanceType) error {
	allowedInstances := p.Plan.Limits().AllowedInstances

	if _, ok := allowedInstances[wantInstance]; ok {
		return nil
	}

	return fmt.Errorf("not allowed to create instance type: %s", wantInstance)
}

func (p *PlanChecker) AlwaysOn() error {
	alwaysOnLimit := p.Plan.Limits().AlwaysOn

	// get all alwaysOn machines that belongs to this user
	alwaysOnMachines := 0
	if err := p.DB.Run("jMachines", func(c *mgo.Collection) error {
		var err error
		alwaysOnMachines, err = c.Find(bson.M{
			"credential":    p.Machine.Username,
			"meta.alwaysOn": true,
		}).Count()

		return err
	}); err != nil && err != mgo.ErrNotFound {
		// if it's something else just return an error, needs to be fixed
		return err
	}

	p.Log.Debug("[%s] checking alwaysOn limit. current alwaysOn count: %d (plan limit: %d, plan: %s)",
		p.Machine.Id, alwaysOnMachines, alwaysOnLimit, p.Plan)

	// the user has still not reached the limit
	if alwaysOnMachines <= alwaysOnLimit {
		p.Log.Debug("[%s] allowing user '%s'. current alwaysOn count: %d (plan limit: %d, plan: %s)",
			p.Machine.Id, p.Username, alwaysOnMachines, alwaysOnLimit, p.Plan)
		return nil // allow user, it didn't reach the limit
	}

	p.Log.Debug("[%s] denying user '%s'. current alwaysOn count: %d (plan limit: %d, plan: %s)",
		p.Machine.Id, p.Username, alwaysOnMachines, alwaysOnLimit, p.Plan)
	return fmt.Errorf("total alwaysOn limit has been reached. Current count: %d Plan limit: %d",
		alwaysOnMachines, alwaysOnLimit)
}

func (p *PlanChecker) Timeout() error {
	// Check klient state before rushing to AWS.
	klientRef, err := klient.Connect(p.Kite, p.Machine.QueryString)
	if err == kite.ErrNoKitesAvailable {
		p.Provider.startTimer(p.Machine)
		return err
	}

	// return if it's something else
	if err != nil {
		return err
	}

	// now the klient is connected and we can ping it, stop the timer and
	// remove it from the list of inactive machines if it's still there.
	p.Provider.stopTimer(p.Machine)

	// get the usage directly from the klient, which is the most predictable source
	usg, err := klientRef.Usage()
	if err != nil {
		klientRef.Close()
		return err
	}
	klientRef.Close()

	// replace with the real and authenticated username
	p.Machine.Builder["username"] = klientRef.Username
	p.Username = klientRef.Username

	// get the timeout from the plan in which the user belongs to
	planTimeout := p.Plan.Limits().Timeout

	p.Log.Debug("[%s] machine [%s] is inactive for %s (plan limit: %s, plan: %s).",
		p.Machine.Id, p.Machine.IpAddress, usg.InactiveDuration, planTimeout, p.Plan)

	// It still have plenty of time to work, do not stop it
	if usg.InactiveDuration <= planTimeout {
		return nil
	}

	p.Log.Info("[%s] machine [%s] has reached current plan limit of %s (plan: %s). Shutting down...",
		p.Machine.Id, p.Machine.IpAddress, usg.InactiveDuration, p.Plan)

	// lock so it doesn't interfere with others.
	p.Provider.Lock(p.Machine.Id)

	// mark our state as stopping so others know what we are doing
	stoppingReason := fmt.Sprintf("Stopping process started due inactivity of %.f minutes",
		planTimeout.Minutes())
	p.Provider.UpdateState(p.Machine.Id, stoppingReason, machinestate.Stopping)

	defer func() {
		// call it in defer, so even if "Stop" fails it should reset the state
		stopReason := fmt.Sprintf("Stopped due inactivity of %.f minutes", planTimeout.Minutes())
		p.Provider.UpdateState(p.Machine.Id, stopReason, machinestate.Stopped)

		p.Provider.Unlock(p.Machine.Id)
	}()

	// Hasta la vista, baby!
	return p.Provider.Stop(p.Machine)
}

func (p *PlanChecker) Total() error {
	allowedMachines := p.Plan.Limits().Total

	instances, err := p.userInstances()

	// no match, allow to create instance
	if err == aws.ErrNoInstances {
		p.Log.Debug("[%s] allowing user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
			p.Machine.Id, p.Username, len(instances), allowedMachines, p.Plan)
		return nil
	}

	// if it's something else don't allow it until it's solved
	if err != nil {
		return err
	}

	go p.checkGhostMachines(instances)

	if len(instances) >= allowedMachines {
		p.Log.Debug("[%s] denying user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
			p.Machine.Id, p.Username, len(instances), allowedMachines, p.Plan)
		return fmt.Errorf("total machine limit has been reached. Current count: %d Plan limit: ",
			len(instances), allowedMachines)
	}

	p.Log.Debug("[%s] allowing user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
		p.Machine.Id, p.Username, len(instances), allowedMachines, p.Plan)

	return nil
}

func (p *PlanChecker) checkGhostMachines(instances []ec2.Instance) {
	for _, instance := range instances {
		for _, tag := range instance.Tags {
			if tag.Key != "koding-machineId" {
				continue
			}

			machineId := tag.Value

			// this is just for logging, so we don't care about handling
			// the error
			p.DB.Run("jMachines", func(c *mgo.Collection) error {
				n, err := c.FindId(bson.ObjectIdHex(machineId)).Count()
				if err != nil {
					return err
				}

				if n != 0 {
					return nil
				}

				p.Log.Warning("Detected a Ghost Machine in AWS! Instance id: %s", instance.InstanceId)
				return nil
			})
		}
	}
}

func (p *PlanChecker) Storage(wantStorage int) error {
	totalStorage := p.Plan.Limits().Storage

	// no need for errors because instances will be empty in case of an error
	instances, _ := p.userInstances()

	// i hate for loops too, but unfortunaly the responses are always in form
	// of slices
	currentStorage := 0
	for _, instance := range instances {
		for _, blockDevice := range instance.BlockDevices {
			volumes, err := p.Api.Client.Volumes([]string{blockDevice.VolumeId}, ec2.NewFilter())
			if err != nil {
				return err
			}

			for _, volume := range volumes.Volumes {
				volumeStorage, err := strconv.Atoi(volume.Size)
				if err != nil {
					return err
				}

				currentStorage += volumeStorage
			}
		}
	}

	p.Log.Debug("[%s] Checking storage. Current: %dGB. Want: %dGB (plan limit: %dGB, plan: %s)",
		p.Machine.Id, currentStorage, wantStorage, totalStorage, p.Plan)

	if currentStorage+wantStorage > totalStorage {
		return fmt.Errorf("total storage limit has been reached. Can use %dGB of %dGB (plan: %s)",
			totalStorage-currentStorage, totalStorage, p.Plan)
	}

	p.Log.Debug("[%s] Allowing user '%s'. Current: %dGB. Want: %dGB (plan limit: %dGB, plan: %s)",
		p.Machine.Id, p.Username, currentStorage, wantStorage, totalStorage, p.Plan)

	// allow to create storage
	return nil
}

func (p *PlanChecker) userInstances() ([]ec2.Instance, error) {
	filter := ec2.NewFilter()
	filter.Add("tag-value", p.Username)

	// Anything except "terminated" and "shutting-down"
	filter.Add("instance-state-name", "pending", "running", "stopping", "stopped")

	instances, err := p.Api.InstancesByFilter(filter)
	if err != nil {
		return nil, err
	}

	filtered := []ec2.Instance{}

	// we don't use filters because they are timing out for us due to high
	// instances count we have. However it seems the filter `tag-value` has an
	// index internally inside AWS so somewhot that one is not timing out.
	for _, instance := range instances {
		for _, tag := range instance.Tags {
			if tag.Key == "koding-user" && tag.Value == p.Username {
				for _, tag := range instance.Tags {
					if tag.Key == "koding-env" && tag.Value == p.Kite.Config.Environment {

						// now we have the instance that matches both the correct username
						// and environment
						filtered = append(filtered, instance)
					}
				}
			}
		}
	}

	// garbage collect it
	instances = nil
	return filtered, nil
}
<commit_msg>checker: only call it once and also nil it for GC<commit_after>package koding

import (
	"fmt"
	"koding/db/mongodb"
	"strconv"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	aws "koding/kites/kloud/api/amazon"
	"koding/kites/kloud/klient"
	"koding/kites/kloud/machinestate"
	"koding/kites/kloud/protocol"
	"koding/kites/kloud/provider/amazon"

	"github.com/koding/kite"
	"github.com/koding/logging"
	"github.com/mitchellh/goamz/ec2"
)

// Checker checks various aspects of a machine. It is used for limiting certain
// aspects of a machine, such as the total allowed machine count, storage size
// and etc.
type Checker interface {
	// Total checks whether the user has reached the current plan's limit of
	// having a total number numbers of machines. It returns an error if the
	// limit is reached or an unexplained error happaned.
	Total() error

	// AlwaysOn checks whether the given machine has reached the current plans
	// always on limit
	AlwaysOn() error

	// Timeout checks whether the user has reached the current plan's
	// inactivity timeout.
	Timeout() error

	// Storage checks whether the user has reached the current plan's limit
	// total storage with the supplied wantStorage information. It returns an
	// error if the limit is reached or an unexplained error happaned.
	Storage(wantStorage int) error

	// AllowedInstances checks whether the given machine has the permisison to
	// create the given instance type
	AllowedInstances(wantInstance InstanceType) error
}

type PlanChecker struct {
	Api      *amazon.AmazonClient
	DB       *mongodb.MongoDB
	Machine  *protocol.Machine
	Provider *Provider
	Kite     *kite.Kite
	Username string
	Log      logging.Logger
	Plan     Plan
}

func (p *PlanChecker) AllowedInstances(wantInstance InstanceType) error {
	allowedInstances := p.Plan.Limits().AllowedInstances

	if _, ok := allowedInstances[wantInstance]; ok {
		return nil
	}

	return fmt.Errorf("not allowed to create instance type: %s", wantInstance)
}

func (p *PlanChecker) AlwaysOn() error {
	alwaysOnLimit := p.Plan.Limits().AlwaysOn

	// get all alwaysOn machines that belongs to this user
	alwaysOnMachines := 0
	if err := p.DB.Run("jMachines", func(c *mgo.Collection) error {
		var err error
		alwaysOnMachines, err = c.Find(bson.M{
			"credential":    p.Machine.Username,
			"meta.alwaysOn": true,
		}).Count()

		return err
	}); err != nil && err != mgo.ErrNotFound {
		// if it's something else just return an error, needs to be fixed
		return err
	}

	p.Log.Debug("[%s] checking alwaysOn limit. current alwaysOn count: %d (plan limit: %d, plan: %s)",
		p.Machine.Id, alwaysOnMachines, alwaysOnLimit, p.Plan)

	// the user has still not reached the limit
	if alwaysOnMachines <= alwaysOnLimit {
		p.Log.Debug("[%s] allowing user '%s'. current alwaysOn count: %d (plan limit: %d, plan: %s)",
			p.Machine.Id, p.Username, alwaysOnMachines, alwaysOnLimit, p.Plan)
		return nil // allow user, it didn't reach the limit
	}

	p.Log.Debug("[%s] denying user '%s'. current alwaysOn count: %d (plan limit: %d, plan: %s)",
		p.Machine.Id, p.Username, alwaysOnMachines, alwaysOnLimit, p.Plan)
	return fmt.Errorf("total alwaysOn limit has been reached. Current count: %d Plan limit: %d",
		alwaysOnMachines, alwaysOnLimit)
}

func (p *PlanChecker) Timeout() error {
	// Check klient state before rushing to AWS.
	klientRef, err := klient.Connect(p.Kite, p.Machine.QueryString)
	if err == kite.ErrNoKitesAvailable {
		p.Provider.startTimer(p.Machine)
		return err
	}

	// return if it's something else
	if err != nil {
		return err
	}

	// now the klient is connected and we can ping it, stop the timer and
	// remove it from the list of inactive machines if it's still there.
	p.Provider.stopTimer(p.Machine)

	// replace with the real and authenticated username
	p.Machine.Builder["username"] = klientRef.Username
	p.Username = klientRef.Username

	// get the usage directly from the klient, which is the most predictable source
	usg, err := klientRef.Usage()

	klientRef.Close()
	klientRef = nil

	if err != nil {
		return err
	}

	// get the timeout from the plan in which the user belongs to
	planTimeout := p.Plan.Limits().Timeout

	p.Log.Debug("[%s] machine [%s] is inactive for %s (plan limit: %s, plan: %s).",
		p.Machine.Id, p.Machine.IpAddress, usg.InactiveDuration, planTimeout, p.Plan)

	// It still have plenty of time to work, do not stop it
	if usg.InactiveDuration <= planTimeout {
		return nil
	}

	p.Log.Info("[%s] machine [%s] has reached current plan limit of %s (plan: %s). Shutting down...",
		p.Machine.Id, p.Machine.IpAddress, usg.InactiveDuration, p.Plan)

	// lock so it doesn't interfere with others.
	p.Provider.Lock(p.Machine.Id)

	// mark our state as stopping so others know what we are doing
	stoppingReason := fmt.Sprintf("Stopping process started due inactivity of %.f minutes",
		planTimeout.Minutes())
	p.Provider.UpdateState(p.Machine.Id, stoppingReason, machinestate.Stopping)

	defer func() {
		// call it in defer, so even if "Stop" fails it should reset the state
		stopReason := fmt.Sprintf("Stopped due inactivity of %.f minutes", planTimeout.Minutes())
		p.Provider.UpdateState(p.Machine.Id, stopReason, machinestate.Stopped)

		p.Provider.Unlock(p.Machine.Id)
	}()

	// Hasta la vista, baby!
	return p.Provider.Stop(p.Machine)
}

func (p *PlanChecker) Total() error {
	allowedMachines := p.Plan.Limits().Total

	instances, err := p.userInstances()

	// no match, allow to create instance
	if err == aws.ErrNoInstances {
		p.Log.Debug("[%s] allowing user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
			p.Machine.Id, p.Username, len(instances), allowedMachines, p.Plan)
		return nil
	}

	// if it's something else don't allow it until it's solved
	if err != nil {
		return err
	}

	go p.checkGhostMachines(instances)

	if len(instances) >= allowedMachines {
		p.Log.Debug("[%s] denying user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
			p.Machine.Id, p.Username, len(instances), allowedMachines, p.Plan)
		return fmt.Errorf("total machine limit has been reached. Current count: %d Plan limit: ",
			len(instances), allowedMachines)
	}

	p.Log.Debug("[%s] allowing user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
		p.Machine.Id, p.Username, len(instances), allowedMachines, p.Plan)

	return nil
}

func (p *PlanChecker) checkGhostMachines(instances []ec2.Instance) {
	for _, instance := range instances {
		for _, tag := range instance.Tags {
			if tag.Key != "koding-machineId" {
				continue
			}

			machineId := tag.Value

			// this is just for logging, so we don't care about handling
			// the error
			p.DB.Run("jMachines", func(c *mgo.Collection) error {
				n, err := c.FindId(bson.ObjectIdHex(machineId)).Count()
				if err != nil {
					return err
				}

				if n != 0 {
					return nil
				}

				p.Log.Warning("Detected a Ghost Machine in AWS! Instance id: %s", instance.InstanceId)
				return nil
			})
		}
	}
}

func (p *PlanChecker) Storage(wantStorage int) error {
	totalStorage := p.Plan.Limits().Storage

	// no need for errors because instances will be empty in case of an error
	instances, _ := p.userInstances()

	// i hate for loops too, but unfortunaly the responses are always in form
	// of slices
	currentStorage := 0
	for _, instance := range instances {
		for _, blockDevice := range instance.BlockDevices {
			volumes, err := p.Api.Client.Volumes([]string{blockDevice.VolumeId}, ec2.NewFilter())
			if err != nil {
				return err
			}

			for _, volume := range volumes.Volumes {
				volumeStorage, err := strconv.Atoi(volume.Size)
				if err != nil {
					return err
				}

				currentStorage += volumeStorage
			}
		}
	}

	p.Log.Debug("[%s] Checking storage. Current: %dGB. Want: %dGB (plan limit: %dGB, plan: %s)",
		p.Machine.Id, currentStorage, wantStorage, totalStorage, p.Plan)

	if currentStorage+wantStorage > totalStorage {
		return fmt.Errorf("total storage limit has been reached. Can use %dGB of %dGB (plan: %s)",
			totalStorage-currentStorage, totalStorage, p.Plan)
	}

	p.Log.Debug("[%s] Allowing user '%s'. Current: %dGB. Want: %dGB (plan limit: %dGB, plan: %s)",
		p.Machine.Id, p.Username, currentStorage, wantStorage, totalStorage, p.Plan)

	// allow to create storage
	return nil
}

func (p *PlanChecker) userInstances() ([]ec2.Instance, error) {
	filter := ec2.NewFilter()
	filter.Add("tag-value", p.Username)

	// Anything except "terminated" and "shutting-down"
	filter.Add("instance-state-name", "pending", "running", "stopping", "stopped")

	instances, err := p.Api.InstancesByFilter(filter)
	if err != nil {
		return nil, err
	}

	filtered := []ec2.Instance{}

	// we don't use filters because they are timing out for us due to high
	// instances count we have. However it seems the filter `tag-value` has an
	// index internally inside AWS so somewhot that one is not timing out.
	for _, instance := range instances {
		for _, tag := range instance.Tags {
			if tag.Key == "koding-user" && tag.Value == p.Username {
				for _, tag := range instance.Tags {
					if tag.Key == "koding-env" && tag.Value == p.Kite.Config.Environment {

						// now we have the instance that matches both the correct username
						// and environment
						filtered = append(filtered, instance)
					}
				}
			}
		}
	}

	// garbage collect it
	instances = nil
	return filtered, nil
}
<|endoftext|>


<commit_before>package models

type ChannelContainer struct {
	Channel             Channel         `json:"channel"`
	IsParticipant       bool            `json:"isParticipant"`
	ParticipantCount    int             `json:"participantCount"`
	ParticipantsPreview []int64         `json:"participantsPreview,omitempty"`
	LastMessage         *ChannelMessage `json:"lastMessage,omitempty"`
}

func NewChannelContainer() *ChannelContainer {
	return &ChannelContainer{}
}

func PopulateChannelContainers(channelList []Channel, accountId int64) ([]*ChannelContainer, error) {
	channelContainers := make([]*ChannelContainer, len(channelList))

	var err error
	for i, channel := range channelList {
		channelContainers[i], err = PopulateChannelContainer(channel, accountId)
		if err != nil {
			return nil, err
		}
	}

	return channelContainers, nil
}

func PopulateChannelContainer(channel Channel, accountId int64) (*ChannelContainer, error) {
	cp := NewChannelParticipant()
	cp.ChannelId = channel.Id

	// add participantCount
	participantCount, err := cp.FetchParticipantCount()
	if err != nil {
		return nil, err
	}

	// add participant preview
	cpList, err := cp.ListAccountIds(5)
	if err != nil {
		return nil, err
	}

	// add participation status
	isParticipant, err := cp.IsParticipant(accountId)
	if err != nil {
		return nil, err
	}

	cc := NewChannelContainer()
	cc.Channel = channel
	cc.IsParticipant = isParticipant
	cc.ParticipantCount = participantCount
	cc.ParticipantsPreview = cpList

	// add last message of the channel
	cm, err := channel.FetchLastMessage()
	if err != nil {
		return nil, err
	}

	cc.LastMessage = cm

	return cc, nil
}
<commit_msg>Social: last message should be ChannelMessageContainer<commit_after>package models

type ChannelContainer struct {
	Channel             Channel                  `json:"channel"`
	IsParticipant       bool                     `json:"isParticipant"`
	ParticipantCount    int                      `json:"participantCount"`
	ParticipantsPreview []int64                  `json:"participantsPreview,omitempty"`
	LastMessage         *ChannelMessageContainer `json:"lastMessage,omitempty"`
}

func NewChannelContainer() *ChannelContainer {
	return &ChannelContainer{}
}

func PopulateChannelContainers(channelList []Channel, accountId int64) ([]*ChannelContainer, error) {
	channelContainers := make([]*ChannelContainer, len(channelList))

	var err error
	for i, channel := range channelList {
		channelContainers[i], err = PopulateChannelContainer(channel, accountId)
		if err != nil {
			return nil, err
		}
	}

	return channelContainers, nil
}

func PopulateChannelContainer(channel Channel, accountId int64) (*ChannelContainer, error) {
	cp := NewChannelParticipant()
	cp.ChannelId = channel.Id

	// add participantCount
	participantCount, err := cp.FetchParticipantCount()
	if err != nil {
		return nil, err
	}

	// add participant preview
	cpList, err := cp.ListAccountIds(5)
	if err != nil {
		return nil, err
	}

	// add participation status
	isParticipant, err := cp.IsParticipant(accountId)
	if err != nil {
		return nil, err
	}

	cc := NewChannelContainer()
	cc.Channel = channel
	cc.IsParticipant = isParticipant
	cc.ParticipantCount = participantCount
	cc.ParticipantsPreview = cpList

	// add last message of the channel
	cm, err := channel.FetchLastMessage()
	if err != nil {
		return nil, err
	}

	if cm != nil {
		cmc, err := cm.BuildEmptyMessageContainer()
		if err != nil {
			return nil, err
		}
		cc.LastMessage = cmc
	}

	return cc, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"errors"
	"fmt"
	"io/ioutil"
	"math"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// A sentinel error returned by ObjectProxy.Sync.
var ErrNotCurrent error = errors.New("Source generation not current.")

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not currently
	// exist in the bucket.
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be zero if our source is a "doesn't exist"
	// generation.
	srcGeneration uint64

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// the generation identified by srcGeneration.
	localFile *os.File

	// false if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If srcGeneration == 0, then dirty
	// INVARIANT: If dirty, then localFile != nil
	dirty bool
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Create a view on the given GCS object generation, or zero if branching from
// a non-existent object (in which case the initial contents are empty).
func NewObjectProxy(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	srcGeneration uint64) (op *ObjectProxy, err error) {
	// Set up the basic struct.
	op = &ObjectProxy{
		bucket:        bucket,
		name:          name,
		srcGeneration: srcGeneration,
	}

	// For "doesn't exist" source generations, we must establish an empty local
	// file and mark the proxy dirty.
	if srcGeneration == 0 {
		if err = op.ensureLocalFile(ctx); err != nil {
			return
		}

		op.dirty = true
	}

	return
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket.
func (op *ObjectProxy) Name() string {
	return op.name
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants() {
	// INVARIANT: If srcGeneration == 0, then dirty
	if op.srcGeneration == 0 && !op.dirty {
		panic("Expected dirty.")
	}

	// INVARIANT: If dirty, then localFile != nil
	if op.dirty && op.localFile == nil {
		panic("Expected non-nil localFile.")
	}
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (op *ObjectProxy) Destroy() (err error) {
	// Make sure that when we exit no invariants are violated.
	defer func() {
		op.srcGeneration = 1
		op.localFile = nil
		op.dirty = false
	}()

	// If we have no local file, there's nothing to do.
	if op.localFile == nil {
		return
	}

	// Close the local file.
	if err = op.localFile.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
func (op *ObjectProxy) Stat(
	ctx context.Context) (size uint64, clobbered bool, err error) {
	panic("TODO")
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Serve the read from the file.
	n, err = op.localFile.ReadAt(buf, offset)

	return
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// TODO(jacobsa): Make sure the dirty flag modification below is tested by
	// removing it and looking for a failure.
	op.dirty = true
	n, err = op.localFile.WriteAt(buf, offset)

	return
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(ctx context.Context, n uint64) (err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Convert to signed, which is what os.File wants.
	if n > math.MaxInt64 {
		err = fmt.Errorf("Illegal offset: %v", n)
		return
	}

	// TODO(jacobsa): Make sure the dirty flag modification below is tested by
	// removing it and looking for a failure.
	op.dirty = true
	err = op.localFile.Truncate(int64(n))

	return
}

// If the proxy is dirty due to having been written to or due to having a nil
// source, save its current contents to GCS and return a generation number for
// a generation with exactly those contents. Do so with a precondition such
// that the creation will fail if the source generation is not current. In that
// case, return ErrNotCurrent.
func (op *ObjectProxy) Sync(ctx context.Context) (gen uint64, err error) {
	panic("TODO")
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Set up an unlinked local temporary file for the given generation of the
// given object. Special case: generation == 0 means an empty file.
func makeLocalFile(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	generation uint64) (f *os.File, err error) {
	// Create the file.
	f, err = ioutil.TempFile("", "object_proxy")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	// Unlink the file so that its inode will be garbage collected when the file
	// is closed.
	if err = os.Remove(f.Name()); err != nil {
		f.Close()
		err = fmt.Errorf("Remove: %v", err)
		return
	}

	// Fetch the object's contents if necessary.
	if generation != 0 {
		panic("TODO")
	}

	return
}

// Ensure that op.localFile is non-nil with an authoritative view of op's
// contents.
func (op *ObjectProxy) ensureLocalFile(ctx context.Context) (err error) {
	// Is there anything to do?
	if op.localFile != nil {
		return
	}

	// Set up the file.
	f, err := makeLocalFile(ctx, op.bucket, op.name, op.srcGeneration)
	if err != nil {
		err = fmt.Errorf("makeLocalFile: %v", err)
		return
	}

	op.localFile = f
	return
}
<commit_msg>Eliminated custom error sentinel.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"fmt"
	"io/ioutil"
	"math"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not currently
	// exist in the bucket.
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be zero if our source is a "doesn't exist"
	// generation.
	srcGeneration uint64

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// the generation identified by srcGeneration.
	localFile *os.File

	// false if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If srcGeneration == 0, then dirty
	// INVARIANT: If dirty, then localFile != nil
	dirty bool
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Create a view on the given GCS object generation, or zero if branching from
// a non-existent object (in which case the initial contents are empty).
func NewObjectProxy(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	srcGeneration uint64) (op *ObjectProxy, err error) {
	// Set up the basic struct.
	op = &ObjectProxy{
		bucket:        bucket,
		name:          name,
		srcGeneration: srcGeneration,
	}

	// For "doesn't exist" source generations, we must establish an empty local
	// file and mark the proxy dirty.
	if srcGeneration == 0 {
		if err = op.ensureLocalFile(ctx); err != nil {
			return
		}

		op.dirty = true
	}

	return
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket.
func (op *ObjectProxy) Name() string {
	return op.name
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants() {
	// INVARIANT: If srcGeneration == 0, then dirty
	if op.srcGeneration == 0 && !op.dirty {
		panic("Expected dirty.")
	}

	// INVARIANT: If dirty, then localFile != nil
	if op.dirty && op.localFile == nil {
		panic("Expected non-nil localFile.")
	}
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (op *ObjectProxy) Destroy() (err error) {
	// Make sure that when we exit no invariants are violated.
	defer func() {
		op.srcGeneration = 1
		op.localFile = nil
		op.dirty = false
	}()

	// If we have no local file, there's nothing to do.
	if op.localFile == nil {
		return
	}

	// Close the local file.
	if err = op.localFile.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
func (op *ObjectProxy) Stat(
	ctx context.Context) (size uint64, clobbered bool, err error) {
	panic("TODO")
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Serve the read from the file.
	n, err = op.localFile.ReadAt(buf, offset)

	return
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// TODO(jacobsa): Make sure the dirty flag modification below is tested by
	// removing it and looking for a failure.
	op.dirty = true
	n, err = op.localFile.WriteAt(buf, offset)

	return
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(ctx context.Context, n uint64) (err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Convert to signed, which is what os.File wants.
	if n > math.MaxInt64 {
		err = fmt.Errorf("Illegal offset: %v", n)
		return
	}

	// TODO(jacobsa): Make sure the dirty flag modification below is tested by
	// removing it and looking for a failure.
	op.dirty = true
	err = op.localFile.Truncate(int64(n))

	return
}

// If the proxy is dirty due to having been written to or due to having a nil
// source, save its current contents to GCS and return a generation number for
// a generation with exactly those contents. Do so with a precondition such
// that the creation will fail if the source generation is not current. In that
// case, return an error of type *gcs.PreconditionError.
func (op *ObjectProxy) Sync(ctx context.Context) (gen uint64, err error) {
	panic("TODO")
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Set up an unlinked local temporary file for the given generation of the
// given object. Special case: generation == 0 means an empty file.
func makeLocalFile(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	generation uint64) (f *os.File, err error) {
	// Create the file.
	f, err = ioutil.TempFile("", "object_proxy")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	// Unlink the file so that its inode will be garbage collected when the file
	// is closed.
	if err = os.Remove(f.Name()); err != nil {
		f.Close()
		err = fmt.Errorf("Remove: %v", err)
		return
	}

	// Fetch the object's contents if necessary.
	if generation != 0 {
		panic("TODO")
	}

	return
}

// Ensure that op.localFile is non-nil with an authoritative view of op's
// contents.
func (op *ObjectProxy) ensureLocalFile(ctx context.Context) (err error) {
	// Is there anything to do?
	if op.localFile != nil {
		return
	}

	// Set up the file.
	f, err := makeLocalFile(ctx, op.bucket, op.name, op.srcGeneration)
	if err != nil {
		err = fmt.Errorf("makeLocalFile: %v", err)
		return
	}

	op.localFile = f
	return
}
<|endoftext|>


<commit_before>// connect to blobstore on AWS and perform read, write, delete, list file opetations
<commit_msg>Update blobstoreTODO.go<commit_after>// connect to blobstore on AWS and perform read, write, delete, list file operations
<|endoftext|>


<commit_before>package daemonizer

import (
	"errors"
	"log"
	"os/exec"
)

// Daemonizer is a struct that uses launchd to daemonize the specified command
type Daemonizer struct {
	toDaemon string
	Command  *exec.Cmd
}

// New returns a new instance of Daemonizer
func New(cmd string) *Daemonizer {
	return &Daemonizer{
		toDaemon: cmd,
		Command:  exec.Command("launchctl", "start"),
	}
}

// Run runs the launchctl command to daemonize the specified command
func (d *Daemonizer) Run(cmd string) error {
	log.Println("daemonizer_darwin:24 run")
	if cmd != "" {
		d.toDaemon = cmd
	}
	if d.toDaemon == "" {
		return errors.New("no command was given")
	}
	output, outErr := d.Command.Output()
	if outErr != nil {
		log.Println("OUT ERROR MOTHERFUCKER!!!")
		return outErr
	}
	runErr := d.Command.Run()
	log.Println("daemonizer_darwin:30 ", (string)(output))
	return runErr
}
<commit_msg>Remove profanity from codebase<commit_after>package daemonizer

import (
	"errors"
	"log"
	"os/exec"
)

// Daemonizer is a struct that uses launchd to daemonize the specified command
type Daemonizer struct {
	toDaemon string
	Command  *exec.Cmd
}

// New returns a new instance of Daemonizer
func New(cmd string) *Daemonizer {
	return &Daemonizer{
		toDaemon: cmd,
		Command:  exec.Command("launchctl", "start"),
	}
}

// Run runs the launchctl command to daemonize the specified command
func (d *Daemonizer) Run(cmd string) error {
	log.Println("daemonizer_darwin:24 run")
	if cmd != "" {
		d.toDaemon = cmd
	}
	if d.toDaemon == "" {
		return errors.New("no command was given")
	}
	output, outErr := d.Command.Output()
	if outErr != nil {
		return outErr
	}
	runErr := d.Command.Run()
	log.Println("daemonizer_darwin:30 ", (string)(output))
	return runErr
}
<|endoftext|>