<commit_before>package itembase

import (
	"crypto/rand"
	"errors"
	"fmt"
	"log"
	"net/http"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
)

func (c *client) newConf() *oauth2.Config {
	var endpointURL string

	if c.production {
		endpointURL = "https://accounts.itembase.com/oauth/v2"
		c.me = "https://users.itembase.com/v1/me"
		c.root = "https://api.itembase.io/v1"
		c.activation = "https://solutionservice.itembase.com"
	} else {
		endpointURL = "http://sandbox.accounts.itembase.io/oauth/v2"
		c.me = "http://sandbox.users.itembase.io/v1/me"
		c.root = "http://sandbox.api.itembase.io/v1"
		c.activation = "http://sandbox.solutionservice.itembase.io"
	}

	return &oauth2.Config{
		ClientID:     c.options.ClientID,
		ClientSecret: c.options.ClientSecret,
		Scopes:       c.options.Scopes,
		RedirectURL:  c.options.RedirectURL,
		Endpoint: oauth2.Endpoint{
			AuthURL:  endpointURL + "/auth",
			TokenURL: endpointURL + "/token",
		},
	}
}

func (c *client) SaveToken(userID string, token *oauth2.Token) (err error) {
	if c.options.TokenHandler.TokenSaver != nil {
		err = c.options.TokenHandler.TokenSaver(userID, token)
	} else {
		err = errors.New("No Token Store!")
	}
	return
}

func (c *client) GetCachedToken(userID string) (token *oauth2.Token, err error) {
	if c.options.TokenHandler.TokenLoader != nil {
		token, err = c.options.TokenHandler.TokenLoader(userID)
	} else {
		err = errors.New("No Token Cache!")
	}
	return
}

func (c *client) GiveTokenPermissions(authURL string) (authcode string, err error) {
	// add logic for handing retrieving code for oauth exchange and matching state
	// For example throw an error, and send email to user instead with this link

	if c.options.TokenHandler.TokenPermissions != nil {
		if authcode, err = c.options.TokenHandler.TokenPermissions(authURL); err != nil {
			log.Fatal(err)
		}
	} else {
		fmt.Println("Please go to the following URL : ", authURL)
		if _, err := fmt.Scan(&authcode); err != nil {
			log.Fatal(err)
		}
	}

	// Use the authorization code that is pushed to the redirect URL.
	// NewTransportWithCode will do the handshake to retrieve
	// an access token and initiate a Transport that is
	// authorized and authenticated by the retrieved token.
	return
}

// UserOAuthClient returns an oauth2 client for a specific user
func (c *client) UserOAuthClient(ctx context.Context, config *oauth2.Config, userID string) (client *http.Client, err error) {
	var userToken *oauth2.Token

	if userToken, err = c.GetCachedToken(userID); err != nil {
		// if token for user is not cached then go through oauth2 flow
		if userToken, err = c.newUserToken(ctx, config, userID); err != nil {
			return
		}
	}

	if !userToken.Valid() { // if user token is expired
		userToken = &oauth2.Token{RefreshToken: userToken.RefreshToken}
	}

	return config.Client(ctx, userToken), err
}

func (c *client) newUserToken(ctx context.Context, config *oauth2.Config, userID string) (*oauth2.Token, error) {
	stateBytes := make([]byte, 32)
	_, err := rand.Read(stateBytes)
	if err != nil {
		log.Fatalf("Unable to read random bytes: %v", err)
		return nil, err
	}
	state := fmt.Sprintf("%x", stateBytes)
	authURL := config.AuthCodeURL(state, oauth2.AccessTypeOffline)

	authcode, err := c.GiveTokenPermissions(authURL)

	token, err := config.Exchange(oauth2.NoContext, authcode)
	if err != nil {
		log.Fatalf("Exchange error: %v", err)
		return nil, err
	}
	c.SaveToken(userID, token) // save token to datastore

	return token, nil
}

func (c *client) getUserToken(userID string) (Token *oauth2.Token) {
	config := c.newConf()
	client, err := c.UserOAuthClient(oauth2.NoContext, config, userID)

	_, err = client.Get(c.me)
	if err == nil {
		fmt.Errorf("Fetch should return an error if no refresh token is set")
	}

	Token, err = client.Transport.(*oauth2.Transport).Source.Token()

	if err != nil {
		log.Fatal("Exchange error: %v", err)
	}

	Token, err = client.Transport.(*oauth2.Transport).Source.Token()

	if err != nil {
		if Token, err = c.newUserToken(oauth2.NoContext, config, userID); err != nil {
			return
		}
		if err != nil {
			log.Fatal("Exchange error:", err)
		}
	} else {
		c.SaveToken(userID, Token)
	}

	return
}
<commit_msg>TokenOAuthClient returns an oauth2 client for a specific token<commit_after>package itembase

import (
	"crypto/rand"
	"errors"
	"fmt"
	"log"
	"net/http"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
)

func (c *client) newConf() *oauth2.Config {
	var endpointURL string

	if c.production {
		endpointURL = "https://accounts.itembase.com/oauth/v2"
		c.me = "https://users.itembase.com/v1/me"
		c.root = "https://api.itembase.io/v1"
		c.activation = "https://solutionservice.itembase.com"
	} else {
		endpointURL = "http://sandbox.accounts.itembase.io/oauth/v2"
		c.me = "http://sandbox.users.itembase.io/v1/me"
		c.root = "http://sandbox.api.itembase.io/v1"
		c.activation = "http://sandbox.solutionservice.itembase.io"
	}

	return &oauth2.Config{
		ClientID:     c.options.ClientID,
		ClientSecret: c.options.ClientSecret,
		Scopes:       c.options.Scopes,
		RedirectURL:  c.options.RedirectURL,
		Endpoint: oauth2.Endpoint{
			AuthURL:  endpointURL + "/auth",
			TokenURL: endpointURL + "/token",
		},
	}
}

func (c *client) SaveToken(userID string, token *oauth2.Token) (err error) {
	if c.options.TokenHandler.TokenSaver != nil {
		err = c.options.TokenHandler.TokenSaver(userID, token)
	} else {
		err = errors.New("No Token Store!")
	}
	return
}

func (c *client) GetCachedToken(userID string) (token *oauth2.Token, err error) {
	if c.options.TokenHandler.TokenLoader != nil {
		token, err = c.options.TokenHandler.TokenLoader(userID)
	} else {
		err = errors.New("No Token Cache!")
	}
	return
}

func (c *client) GiveTokenPermissions(authURL string) (authcode string, err error) {
	// add logic for handing retrieving code for oauth exchange and matching state
	// For example throw an error, and send email to user instead with this link

	if c.options.TokenHandler.TokenPermissions != nil {
		if authcode, err = c.options.TokenHandler.TokenPermissions(authURL); err != nil {
			log.Fatal(err)
		}
	} else {
		fmt.Println("Please go to the following URL : ", authURL)
		if _, err := fmt.Scan(&authcode); err != nil {
			log.Fatal(err)
		}
	}

	// Use the authorization code that is pushed to the redirect URL.
	// NewTransportWithCode will do the handshake to retrieve
	// an access token and initiate a Transport that is
	// authorized and authenticated by the retrieved token.
	return
}

// UserOAuthClient returns an oauth2 client for a specific user
func (c *client) UserOAuthClient(ctx context.Context, config *oauth2.Config, userID string) (client *http.Client, err error) {
	var userToken *oauth2.Token

	if userToken, err = c.GetCachedToken(userID); err != nil {
		// if token for user is not cached then go through oauth2 flow
		if userToken, err = c.newUserToken(ctx, config, userID); err != nil {
			return
		}
	}

	if !userToken.Valid() { // if user token is expired
		userToken = &oauth2.Token{RefreshToken: userToken.RefreshToken}
	}

	return config.Client(ctx, userToken), err
}

// TokenOAuthClient returns an oauth2 client for a specific token
func (c *client) TokenOAuthClient(ctx context.Context, config *oauth2.Config, userToken *oauth2.Token) (client *http.Client, err error) {

	if !userToken.Valid() { // if user token is expired
		userToken = &oauth2.Token{RefreshToken: userToken.RefreshToken}
	}

	return config.Client(ctx, userToken), err
}

func (c *client) newUserToken(ctx context.Context, config *oauth2.Config, userID string) (*oauth2.Token, error) {
	stateBytes := make([]byte, 32)
	_, err := rand.Read(stateBytes)
	if err != nil {
		log.Fatalf("Unable to read random bytes: %v", err)
		return nil, err
	}
	state := fmt.Sprintf("%x", stateBytes)
	authURL := config.AuthCodeURL(state, oauth2.AccessTypeOffline)

	authcode, err := c.GiveTokenPermissions(authURL)

	token, err := config.Exchange(oauth2.NoContext, authcode)
	if err != nil {
		log.Fatalf("Exchange error: %v", err)
		return nil, err
	}
	c.SaveToken(userID, token) // save token to datastore

	return token, nil
}

func (c *client) getUserToken(userID string) (Token *oauth2.Token) {
	config := c.newConf()
	client, err := c.UserOAuthClient(oauth2.NoContext, config, userID)

	_, err = client.Get(c.me)
	if err == nil {
		fmt.Errorf("Fetch should return an error if no refresh token is set")
	}

	Token, err = client.Transport.(*oauth2.Transport).Source.Token()

	if err != nil {
		log.Fatal("Exchange error: %v", err)
	}

	Token, err = client.Transport.(*oauth2.Transport).Source.Token()

	if err != nil {
		if Token, err = c.newUserToken(oauth2.NoContext, config, userID); err != nil {
			return
		}
		if err != nil {
			log.Fatal("Exchange error:", err)
		}
	} else {
		c.SaveToken(userID, Token)
	}

	return
}
<|endoftext|>


<commit_before>/*
Ocean
=====
A simple lexer for go that supports shell-style quoting, commenting, piping,
redirecting, and escaping.

Code originally forked from go-shlex (http://code.google.com/p/go-shlex/).
Contributions made by flynn/go-shlex (https://github.com/flynn/go-shlex).

License
=======
Copyright 2014 Sean Hickey <sean@headzoo.io>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ocean

import (
	"errors"
	"fmt"
	"io"
	"strings"
)

// Split splits a string in to a slice of strings, based upon shell-style rules for
// quoting, escaping, and spaces.
func Split(s string) ([]TokenValue, error) {
	lexer := NewLexer(strings.NewReader(s))
	subStrings := []TokenValue{}

	for {
		word, err := lexer.NextWord()
		if err != nil {
			if err == io.EOF {
				return subStrings, nil
			}
			return subStrings, err
		}
		subStrings = append(subStrings, word)
	}

	return subStrings, nil
}

// errorf returns a new error with a formatted message.
func errorf(format string, a ...interface{}) error {
	return errors.New(fmt.Sprintf(format, a...))
}
<commit_msg>Renamed Split to Tokenize<commit_after>/*
Ocean
=====
A simple lexer for go that supports shell-style quoting, commenting, piping,
redirecting, and escaping.

Code originally forked from go-shlex (http://code.google.com/p/go-shlex/).
Contributions made by flynn/go-shlex (https://github.com/flynn/go-shlex).

License
=======
Copyright 2014 Sean Hickey <sean@headzoo.io>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ocean

import (
	"errors"
	"fmt"
	"io"
	"strings"
)

// Split splits a string in to a slice of strings, based upon shell-style rules for
// quoting, escaping, and spaces.
func Tokenize(s string) ([]TokenValue, error) {
	lexer := NewLexer(strings.NewReader(s))
	subStrings := []TokenValue{}

	for {
		word, err := lexer.NextWord()
		if err != nil {
			if err == io.EOF {
				return subStrings, nil
			}
			return subStrings, err
		}
		subStrings = append(subStrings, word)
	}

	return subStrings, nil
}

// errorf returns a new error with a formatted message.
func errorf(format string, a ...interface{}) error {
	return errors.New(fmt.Sprintf(format, a...))
}
<|endoftext|>


<commit_before>package core

import (
	"log"
	"sync"

	"github.com/nikhan/go-fetch"
)

type KernelFunc func(chan bool, map[string]Message) (map[string]Message, bool)

type Spec struct {
	Name    string
	Inputs  []string
	Outputs []string
	Kernel  KernelFunc
}

// A Block is the basic processing unit in streamtools. It has inbound and outbound routes.
type Block struct {
	Name     string // for logging
	Inputs   map[string]*Input
	Outputs  map[string]*Output
	QuitChan chan bool
	Kernel   KernelFunc
	sync.Mutex
}

// NewBlock returns a block with no inputs and no outputs.
func NewBlock(s Spec) *Block {

	nb := &Block{
		Name:     s.Name,
		Inputs:   make(map[string]*Input),
		Outputs:  make(map[string]*Output),
		QuitChan: make(chan bool),
	}

	for _, v := range s.Inputs {
		nb.AddInput(v)
	}

	for _, v := range s.Outputs {
		nb.AddOutput(v)
	}

	nb.Kernel = s.Kernel

	return nb
}

func (b *Block) Serve() {
	var values map[string]Message
	var output map[string]Message
	var ok bool
	for {
		if values, ok = b.Receive(); !ok {
			return
		}

		if output, ok = b.Kernel(b.QuitChan, values); !ok {
			return
		}

		if ok = b.Broadcast(output); !ok {
			return
		}
	}
}

// Add a named input to the block
func (b *Block) AddInput(id string) bool {
	b.Lock()
	defer b.Unlock()
	_, ok := b.Inputs[id]
	if ok {
		return false
	}
	b.Inputs[id] = NewInput()
	return true
}

// Set an input's Path
func (b *Block) SetPath(id, path string) error {
	query, err := fetch.Parse(path)
	if err != nil {
		return err
	}
	b.Lock()
	b.Inputs[id].Path = query
	b.Unlock()
	return nil
}

// call this whenever you want to set a value, or make a new connection
func stopValuePusher(in *Input) {
	select {
	case in.quitChan <- true:
	default:
		// wasn't running (is there a race here?)
	}
}

// Set an input's Value
func (i *Input) SetValue(value Message) error {
	// we store the marshalled value in the Input so we can access it later
	//i.Lock()
	//i.Value = value
	//i.Unlock()

	// then, to set an input to a particular value, we just push
	// that value to that input, as though we had a little pusher block.

	// first kill any existing value pusher
	stopValuePusher(i)

	// then set the pusher going
	go func() {
		for {
			select {
			case i.Connection <- value:
			case <-i.quitChan:
				return
			}
		}
	}()
	return nil
}

// Remove a named input to the block
func (b *Block) RemoveInput(id string) bool {
	b.Lock()
	defer b.Unlock()
	_, ok := b.Inputs[id]
	if !ok {
		return false
	}
	delete(b.Inputs, id)
	return true
}

// GetInput returns the input Connection
func (b *Block) GetInput(id string) *Input {
	b.Lock()
	input, ok := b.Inputs[id]
	b.Unlock()
	if !ok {
		return nil
	}
	return input
}

// GetOutput returns the specified output
func (b *Block) GetOutput(id string) *Output {
	b.Lock()
	output, ok := b.Outputs[id]
	b.Unlock()
	if !ok {
		return nil
	}
	return output
}

// AddOutput registers a new output for the block
func (b *Block) AddOutput(id string) bool {
	b.Lock()
	defer b.Unlock()
	_, ok := b.Outputs[id]
	if ok {
		return false
	}
	b.Outputs[id] = NewOutput()
	return true
}

// RemoveOutput deletes the output from the block
func (b *Block) RemoveOutput(id string) bool {
	b.Lock()
	defer b.Unlock()
	_, ok := b.Outputs[id]
	if !ok {
		return false
	}
	delete(b.Outputs, id)
	return true
}

// Stop is called when removing a block from the streamtools pattern. This is the default, and can be overwritten.
func (b *Block) Stop() {
	b.QuitChan <- true
}

// Broadcast is called when sending a message to an Output. If Broadcast returns false your block must immediately return.
func (b Block) Broadcast(outputs map[string]Message) bool {
	for k, v := range outputs {
		o := b.GetOutput(k)
		for c, _ := range o.GetConnections() {
			select {
			case c <- v:
			case <-b.QuitChan:
				return false
			}
		}
	}
	return true
}

func (b Block) getName() string {
	return b.Name
}

/*
func (b Block) Merge(β Block) *Block {
	out := NewBlock(b.getName() + "_" + β.getName())
	for id, input := range b.Inputs {
		out.Inputs[id] = input
	}
	for id, output := range β.Outputs {
		out.Outputs[id] = output
	}
	out.Kernel = func(msgs ...Message) (map[string]Message, error) {
		outMsg, err := b.Kernel(msgs)
		if err != nil {
			return nil, err
		}
		inMsg := map[string]Message{
			"in": outMsg["out"],
		}
		return β.Kernel(inMsg)
	}
	return out
}
*/

func (b Block) Receive() (map[string]Message, bool) {
	var err error
	values := make(map[string]Message)
	for name, in := range b.Inputs {
		select {
		case m := <-in.Connection:
			in.Value, err = fetch.Run(in.Path, m)
			if err != nil {
				log.Fatal(err)
			}
		case <-b.QuitChan:
			return nil, false
		}
	}
	return values, true
}
<commit_msg>routing and blocks are now resolved<commit_after>package core

import (
	"log"
	"sync"

	"github.com/nikhan/go-fetch"
)

type KernelFunc func(chan bool, map[string]Message) (map[string]Message, bool)

type Spec struct {
	Name    string
	Inputs  []string
	Outputs []string
	Kernel  KernelFunc
}

// A Block is the basic processing unit in streamtools. It has inbound and outbound routes.
type Block struct {
	Name     string // for logging
	Inputs   map[string]*Input
	Outputs  map[string]*Output
	QuitChan chan bool
	Kernel   KernelFunc
	sync.Mutex
}

// NewBlock returns a block with no inputs and no outputs.
func NewBlock(s Spec) *Block {

	nb := &Block{
		Name:     s.Name,
		Inputs:   make(map[string]*Input),
		Outputs:  make(map[string]*Output),
		QuitChan: make(chan bool),
	}

	for _, v := range s.Inputs {
		nb.AddInput(v)
	}

	for _, v := range s.Outputs {
		nb.AddOutput(v)
	}

	nb.Kernel = s.Kernel

	return nb
}

func (b *Block) Serve() {
	var values map[string]Message
	var output map[string]Message
	var ok bool
	for {
		if values, ok = b.Receive(); !ok {
			return
		}

		if output, ok = b.Kernel(b.QuitChan, values); !ok {
			return
		}

		if ok = b.Broadcast(output); !ok {
			return
		}
	}
}

// Add a named input to the block
func (b *Block) AddInput(id string) bool {
	b.Lock()
	defer b.Unlock()
	_, ok := b.Inputs[id]
	if ok {
		return false
	}
	b.Inputs[id] = NewInput()
	return true
}

// Set an input's Path
func (b *Block) SetPath(id, path string) error {
	query, err := fetch.Parse(path)
	if err != nil {
		return err
	}
	b.Lock()
	b.Inputs[id].Path = query
	b.Unlock()
	return nil
}

// Remove a named input to the block
func (b *Block) RemoveInput(id string) bool {
	b.Lock()
	defer b.Unlock()
	_, ok := b.Inputs[id]
	if !ok {
		return false
	}
	delete(b.Inputs, id)
	return true
}

// GetInput returns the input Connection
func (b *Block) GetInput(id string) *Input {
	b.Lock()
	input, ok := b.Inputs[id]
	b.Unlock()
	if !ok {
		return nil
	}
	return input
}

// GetOutput returns the specified output
func (b *Block) GetOutput(id string) *Output {
	b.Lock()
	output, ok := b.Outputs[id]
	b.Unlock()
	if !ok {
		return nil
	}
	return output
}

// AddOutput registers a new output for the block
func (b *Block) AddOutput(id string) bool {
	b.Lock()
	defer b.Unlock()
	_, ok := b.Outputs[id]
	if ok {
		return false
	}
	b.Outputs[id] = NewOutput()
	return true
}

// RemoveOutput deletes the output from the block
func (b *Block) RemoveOutput(id string) bool {
	b.Lock()
	defer b.Unlock()
	_, ok := b.Outputs[id]
	if !ok {
		return false
	}
	delete(b.Outputs, id)
	return true
}

// Stop is called when removing a block from the streamtools pattern. This is the default, and can be overwritten.
func (b *Block) Stop() {
	b.QuitChan <- true
}

// Broadcast is called when sending a message to an Output. If Broadcast returns false your block must immediately return.
func (b Block) Broadcast(outputs map[string]Message) bool {
	for k, v := range outputs {
		o := b.GetOutput(k)
		for c, _ := range o.GetConnections() {
			select {
			case c <- v:
			case <-b.QuitChan:
				return false
			}
		}
	}
	return true
}

func (b Block) getName() string {
	return b.Name
}

/*
func (b Block) Merge(β Block) *Block {
	out := NewBlock(b.getName() + "_" + β.getName())
	for id, input := range b.Inputs {
		out.Inputs[id] = input
	}
	for id, output := range β.Outputs {
		out.Outputs[id] = output
	}
	out.Kernel = func(msgs ...Message) (map[string]Message, error) {
		outMsg, err := b.Kernel(msgs)
		if err != nil {
			return nil, err
		}
		inMsg := map[string]Message{
			"in": outMsg["out"],
		}
		return β.Kernel(inMsg)
	}
	return out
}
*/

func (b Block) Receive() (map[string]Message, bool) {
	var err error
	values := make(map[string]Message)
	for name, in := range b.Inputs {
		select {
		case m := <-in.Connection:
			in.Value, err = fetch.Run(in.Path, m)
			if err != nil {
				log.Fatal(err)
			}
		case <-b.QuitChan:
			return nil, false
		}
	}
	return values, true
}
<|endoftext|>


<commit_before>package bots

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"runtime/debug"
	"strings"
	"time"

	"context"
	"github.com/julienschmidt/httprouter"
	"github.com/pkg/errors"
	"github.com/strongo/app"
	"github.com/strongo/gamp"
	"github.com/strongo/log"
)

// ErrorIcon is used to report errors to user
var ErrorIcon = "🚨"

// WebhookDriver is doing initial request & final response processing.
// That includes logging, creating input messages in a general format, sending response.
type WebhookDriver interface {
	RegisterWebhookHandlers(httpRouter *httprouter.Router, pathPrefix string, webhookHandlers ...WebhookHandler)
	HandleWebhook(w http.ResponseWriter, r *http.Request, webhookHandler WebhookHandler)
}

// BotDriver keeps information about bots and map requests to appropriate handlers
type BotDriver struct {
	Analytics  AnalyticsSettings
	botHost    BotHost
	appContext BotAppContext
	//router          *WebhooksRouter
	panicTextFooter string
}

var _ WebhookDriver = (*BotDriver)(nil) // Ensure BotDriver is implementing interface WebhookDriver

// AnalyticsSettings keeps data for Google Analytics
type AnalyticsSettings struct {
	GaTrackingID string // TODO: Refactor to list of analytics providers
	Enabled      func(r *http.Request) bool
}

// NewBotDriver registers new bot driver (TODO: describe why we need it)
func NewBotDriver(gaSettings AnalyticsSettings, appContext BotAppContext, host BotHost, panicTextFooter string) WebhookDriver {
	if appContext.AppUserEntityKind() == "" {
		panic("appContext.AppUserEntityKind() is empty")
	}
	if host == nil {
		panic("BotHost == nil")
	}
	return BotDriver{
		Analytics:  gaSettings,
		appContext: appContext,
		botHost:    host,
		//router: router,
		panicTextFooter: panicTextFooter,
	}
}

// RegisterWebhookHandlers adds handlers to a bot driver
func (d BotDriver) RegisterWebhookHandlers(httpRouter *httprouter.Router, pathPrefix string, webhookHandlers ...WebhookHandler) {
	for _, webhookHandler := range webhookHandlers {
		webhookHandler.RegisterHttpHandlers(d, d.botHost, httpRouter, pathPrefix)
	}
}

// HandleWebhook takes and HTTP request and process it
func (d BotDriver) HandleWebhook(w http.ResponseWriter, r *http.Request, webhookHandler WebhookHandler) {
	started := time.Now()
	c := d.botHost.Context(r)
	//log.Debugf(c, "BotDriver.HandleWebhook()")
	if w == nil {
		panic("Parameter 'w http.ResponseWriter' is nil")
	}
	if r == nil {
		panic("Parameter 'r *http.Request' is nil")
	}
	if webhookHandler == nil {
		panic("Parameter 'webhookHandler WebhookHandler' is nil")
	}

	botContext, entriesWithInputs, err := webhookHandler.GetBotContextAndInputs(c, r)

	if d.invalidContextOrInputs(c, w, r, botContext, entriesWithInputs, err) {
		return
	}

	log.Debugf(c, "BotDriver.HandleWebhook() => botCode=%v, len(entriesWithInputs): %d", botContext.BotSettings.Code, len(entriesWithInputs))

	var (
		whc               WebhookContext // TODO: How do deal with Facebook multiple entries per request?
		measurementSender *gamp.BufferedClient
	)

	var sendStats bool
	{ // Initiate Google Analytics Measurement API client
		if d.Analytics.Enabled == nil {
			sendStats = botContext.BotSettings.Env == strongo.EnvProduction
			//log.Debugf(c, "d.AnalyticsSettings.Enabled == nil, botContext.BotSettings.Env: %v, sendStats: %v", strongo.EnvironmentNames[botContext.BotSettings.Env], sendStats)
		} else {
			sendStats = d.Analytics.Enabled(r)
			//log.Debugf(c, "d.AnalyticsSettings.Enabled != nil, sendStats: %v", sendStats)
		}
		if sendStats {
			botHost := botContext.BotHost
			measurementSender = gamp.NewBufferedClient("", botHost.GetHTTPClient(c), nil)
		}
	}

	defer func() {
		log.Debugf(c, "driver.deferred(recover) - checking for panic & flush GA")
		if sendStats {
			if d.Analytics.GaTrackingID == "" {
				log.Warningf(c, "driver.Analytics.GaTrackingID is not set")
			} else {
				timing := gamp.NewTiming(time.Now().Sub(started))
				timing.TrackingID = d.Analytics.GaTrackingID // TODO: What to do if different FB bots have different Tacking IDs? Can FB handler get messages for different bots? If not (what probably is the case) can we get ID from bot settings instead of driver?
				measurementSender.Queue(timing)
			}
		}

		reportError := func(recovered interface{}) {
			messageText := fmt.Sprintf("Server error (panic): %v\n\n%v", recovered, d.panicTextFooter)
			log.Criticalf(c, "Panic recovered: %s\n%s", messageText, debug.Stack())

			if sendStats { // Zero if GA is disabled
				d.reportErrorToGA(whc, measurementSender, messageText)
			}

			if whc != nil {
				if chatID, err := whc.BotChatID(); err == nil && chatID != "" {
					if responder := whc.Responder(); responder != nil {
						if _, err := responder.SendMessage(c, whc.NewMessage(ErrorIcon+" "+messageText), BotAPISendMessageOverResponse); err != nil {
							log.Errorf(c, errors.WithMessage(err, "failed to report error to user").Error())
						}
					}
				}
			}
		}

		if recovered := recover(); recovered != nil {
			reportError(recovered)
		} else if sendStats {
			log.Debugf(c, "Flushing GA...")
			if err = measurementSender.Flush(); err != nil {
				log.Warningf(c, "Failed to flush to GA: %v", err)
			} else {
				log.Debugf(c, "Sent to GA: %v items", measurementSender.QueueDepth())
			}
		} else {
			log.Debugf(c, "GA: sendStats=false")
		}
	}()

	if err != nil {
		log.Errorf(c, "Failed to create new WebhookContext: %v", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	botCoreStores := webhookHandler.CreateBotCoreStores(d.appContext, r)
	defer func() {
		if whc != nil { // TODO: How do deal with Facebook multiple entries per request?
			//log.Debugf(c, "Closing BotChatStore...")
			//chatEntity := whc.ChatEntity()
			//if chatEntity != nil && chatEntity.GetPreferredLanguage() == "" {
			//	chatEntity.SetPreferredLanguage(whc.Locale().Code5)
			//}
			if err := botCoreStores.BotChatStore.Close(c); err != nil {
				log.Errorf(c, "Failed to close BotChatStore: %v", err)
				var m MessageFromBot
				m.Text = ErrorIcon + " ERROR: Service is temporary unavailable. Probably a global outage, status at https://status.cloud.google.com/"
				if _, err := whc.Responder().SendMessage(c, m, BotAPISendMessageOverHTTPS); err != nil {
					log.Errorf(c, "Failed to report outage: %v", err)
				}
			}
		}
	}()

	for _, entryWithInputs := range entriesWithInputs {
		for i, input := range entryWithInputs.Inputs {
			if input == nil {
				panic(fmt.Sprintf("entryWithInputs.Inputs[%d] == nil", i))
			}
			d.logInput(c, i, input)
			whc = webhookHandler.CreateWebhookContext(d.appContext, r, *botContext, input, botCoreStores, measurementSender)
			responder := webhookHandler.GetResponder(w, whc) // TODO: Move inside webhookHandler.CreateWebhookContext()?
			botContext.BotSettings.Router.Dispatch(webhookHandler, responder, whc)
		}
	}
}

func (BotDriver) invalidContextOrInputs(c context.Context, w http.ResponseWriter, r *http.Request, botContext *BotContext, entriesWithInputs []EntryInputs, err error) bool {
	if err != nil {
		if _, ok := err.(ErrAuthFailed); ok {
			log.Warningf(c, "Auth failed: %v", err)
			http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
		} else if errors.Cause(err) == ErrNotImplemented {
			log.Debugf(c, err.Error())
			w.WriteHeader(http.StatusNoContent)
			//http.Error(w, "", http.StatusOK) // TODO: Decide how to handle it properly, return http.StatusNotImplemented?
		} else if _, ok := err.(*json.SyntaxError); ok {
			log.Debugf(c, errors.Wrap(err, "Request body is not valid JSON").Error())
			http.Error(w, err.Error(), http.StatusBadRequest)
		} else {
			log.Errorf(c, "Failed to call webhookHandler.GetBotContextAndInputs(router): %v", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return true
	} else if botContext == nil {
		if entriesWithInputs == nil {
			log.Warningf(c, "botContext == nil, entriesWithInputs == nil")
		} else if len(entriesWithInputs) == 0 {
			log.Warningf(c, "botContext == nil, len(entriesWithInputs) == 0")
		} else {
			log.Errorf(c, "botContext == nil, len(entriesWithInputs) == %v", len(entriesWithInputs))
		}
		return true
	} else if entriesWithInputs == nil {
		log.Errorf(c, "entriesWithInputs == nil")
		return true
	}

	switch botContext.BotSettings.Env {
	case strongo.EnvLocal:
		if r.Host != "localhost" && !strings.HasSuffix(r.Host, ".ngrok.io") {
			log.Warningf(c, "whc.GetBotSettings().Mode == Local, host: %v", r.Host)
			w.WriteHeader(http.StatusBadRequest)
			return true
		}
	case strongo.EnvProduction:
		if r.Host == "localhost" || strings.HasSuffix(r.Host, ".ngrok.io") {
			log.Warningf(c, "whc.GetBotSettings().Mode == Production, host: %v", r.Host)
			w.WriteHeader(http.StatusBadRequest)
			return true
		}
	}

	return false
}

func (BotDriver) reportErrorToGA(whc WebhookContext, measurementSender *gamp.BufferedClient, messageText string) {
	ga := whc.GA()
	gaMessage := gamp.NewException(messageText, true)

	if whc != nil { // TODO: How do deal with Facebook multiple entries per request?
		gaMessage.Common = ga.GaCommon()
	} else {
		gaMessage.Common.ClientID = "c7ea15eb-3333-4d47-a002-9d1a14996371" // TODO: move hardcoded value
		gaMessage.Common.DataSource = "bot-" + whc.BotPlatform().ID()
	}

	c := whc.Context()
	if err := ga.Queue(gaMessage); err != nil {
		log.Errorf(c, "Failed to queue exception message for GA: %v", err)
	} else {
		log.Debugf(c, "Exception message queued for GA.")
	}

	if err := measurementSender.Flush(); err != nil {
		log.Errorf(c, "Failed to flush GA buffer after exception: %v", err)
	} else {
		log.Debugf(c, "GA buffer flushed after exception")
	}
}

func (BotDriver) logInput(c context.Context, i int, input WebhookInput) {
	switch input.(type) {
	case WebhookTextMessage:
		sender := input.GetSender()
		log.Debugf(c, "BotUser#%v(%v %v) => text: %v", sender.GetID(), sender.GetFirstName(), sender.GetLastName(), input.(WebhookTextMessage).Text())
	case WebhookNewChatMembersMessage:
		newMembers := input.(WebhookNewChatMembersMessage).NewChatMembers()
		var b bytes.Buffer
		b.WriteString(fmt.Sprintf("NewChatMembers: %d", len(newMembers)))
		for i, member := range newMembers {
			b.WriteString(fmt.Sprintf("\t%d: (%v) - %v %v", i+1, member.GetUserName(), member.GetFirstName(), member.GetLastName()))
		}
		log.Debugf(c, b.String())
	case WebhookContactMessage:
		sender := input.GetSender()
		contactMessage := input.(WebhookContactMessage)
		log.Debugf(c, "BotUser#%v(%v %v) => Contact(name: %v|%v, phone number: %v)", sender.GetID(), sender.GetFirstName(), sender.GetLastName(), contactMessage.FirstName(), contactMessage.LastName(), contactMessage.PhoneNumber())
	case WebhookCallbackQuery:
		callbackQuery := input.(WebhookCallbackQuery)
		callbackData := callbackQuery.GetData()
		sender := input.GetSender()
		log.Debugf(c, "BotUser#%v(%v %v) => callback: %v", sender.GetID(), sender.GetFirstName(), sender.GetLastName(), callbackData)
	case WebhookInlineQuery:
		sender := input.GetSender()
		log.Debugf(c, "BotUser#%v(%v %v) => inline query: %v", sender.GetID(), sender.GetFirstName(), sender.GetLastName(), input.(WebhookInlineQuery).GetQuery())
	case WebhookChosenInlineResult:
		sender := input.GetSender()
		log.Debugf(c, "BotUser#%v(%v %v) => chosen InlineMessageID: %v", sender.GetID(), sender.GetFirstName(), sender.GetLastName(), input.(WebhookChosenInlineResult).GetInlineMessageID())
	case WebhookReferralMessage:
		sender := input.GetSender()
		log.Debugf(c, "BotUser#%v(%v %v) => text: %v", sender.GetID(), sender.GetFirstName(), sender.GetLastName(), input.(WebhookTextMessage).Text())
	default:
		log.Warningf(c, "Unhandled input[%v] type: %T", i, input)
	}
}
<commit_msg>Fixes & unit tests<commit_after>package bots

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"runtime/debug"
	"strings"
	"time"

	"context"
	"github.com/julienschmidt/httprouter"
	"github.com/pkg/errors"
	"github.com/strongo/app"
	"github.com/strongo/gamp"
	"github.com/strongo/log"
)

// ErrorIcon is used to report errors to user
var ErrorIcon = "🚨"

// WebhookDriver is doing initial request & final response processing.
// That includes logging, creating input messages in a general format, sending response.
type WebhookDriver interface {
	RegisterWebhookHandlers(httpRouter *httprouter.Router, pathPrefix string, webhookHandlers ...WebhookHandler)
	HandleWebhook(w http.ResponseWriter, r *http.Request, webhookHandler WebhookHandler)
}

// BotDriver keeps information about bots and map requests to appropriate handlers
type BotDriver struct {
	Analytics  AnalyticsSettings
	botHost    BotHost
	appContext BotAppContext
	//router          *WebhooksRouter
	panicTextFooter string
}

var _ WebhookDriver = (*BotDriver)(nil) // Ensure BotDriver is implementing interface WebhookDriver

// AnalyticsSettings keeps data for Google Analytics
type AnalyticsSettings struct {
	GaTrackingID string // TODO: Refactor to list of analytics providers
	Enabled      func(r *http.Request) bool
}

// NewBotDriver registers new bot driver (TODO: describe why we need it)
func NewBotDriver(gaSettings AnalyticsSettings, appContext BotAppContext, host BotHost, panicTextFooter string) WebhookDriver {
	if appContext.AppUserEntityKind() == "" {
		panic("appContext.AppUserEntityKind() is empty")
	}
	if host == nil {
		panic("BotHost == nil")
	}
	return BotDriver{
		Analytics:  gaSettings,
		appContext: appContext,
		botHost:    host,
		//router: router,
		panicTextFooter: panicTextFooter,
	}
}

// RegisterWebhookHandlers adds handlers to a bot driver
func (d BotDriver) RegisterWebhookHandlers(httpRouter *httprouter.Router, pathPrefix string, webhookHandlers ...WebhookHandler) {
	for _, webhookHandler := range webhookHandlers {
		webhookHandler.RegisterHttpHandlers(d, d.botHost, httpRouter, pathPrefix)
	}
}

// HandleWebhook takes and HTTP request and process it
func (d BotDriver) HandleWebhook(w http.ResponseWriter, r *http.Request, webhookHandler WebhookHandler) {
	started := time.Now()
	c := d.botHost.Context(r)
	//log.Debugf(c, "BotDriver.HandleWebhook()")
	if w == nil {
		panic("Parameter 'w http.ResponseWriter' is nil")
	}
	if r == nil {
		panic("Parameter 'r *http.Request' is nil")
	}
	if webhookHandler == nil {
		panic("Parameter 'webhookHandler WebhookHandler' is nil")
	}

	botContext, entriesWithInputs, err := webhookHandler.GetBotContextAndInputs(c, r)

	if d.invalidContextOrInputs(c, w, r, botContext, entriesWithInputs, err) {
		return
	}

	log.Debugf(c, "BotDriver.HandleWebhook() => botCode=%v, len(entriesWithInputs): %d", botContext.BotSettings.Code, len(entriesWithInputs))

	var (
		whc               WebhookContext // TODO: How do deal with Facebook multiple entries per request?
		measurementSender *gamp.BufferedClient
	)

	var sendStats bool
	{ // Initiate Google Analytics Measurement API client

		if d.Analytics.Enabled == nil {
			sendStats = botContext.BotSettings.Env == strongo.EnvProduction
		} else {
			if sendStats = d.Analytics.Enabled(r); !sendStats {

			}
			//log.Debugf(c, "d.AnalyticsSettings.Enabled != nil, sendStats: %v", sendStats)
		}
		if sendStats {
			botHost := botContext.BotHost
			measurementSender = gamp.NewBufferedClient("", botHost.GetHTTPClient(c), func(err error) {
				log.Errorf(c, "Failed to log to GA: %v", err)
			})
		} else {
			envName, ok := strongo.EnvironmentNames[botContext.BotSettings.Env]
			if !ok {
				envName = "UNKNOWN"
			}
			log.Debugf(c, "d.Analytics.Enabled=%v, botContext.BotSettings.Env=%v:%v, sendStats=%v",
				d.Analytics.Enabled, botContext.BotSettings.Env, envName, sendStats)
		}
	}

	defer func() {
		log.Debugf(c, "driver.deferred(recover) - checking for panic & flush GA")
		if sendStats {
			if d.Analytics.GaTrackingID == "" {
				log.Warningf(c, "driver.Analytics.GaTrackingID is not set")
			} else {
				timing := gamp.NewTiming(time.Now().Sub(started))
				timing.TrackingID = d.Analytics.GaTrackingID // TODO: What to do if different FB bots have different Tacking IDs? Can FB handler get messages for different bots? If not (what probably is the case) can we get ID from bot settings instead of driver?
				measurementSender.Queue(timing)
			}
		}

		reportError := func(recovered interface{}) {
			messageText := fmt.Sprintf("Server error (panic): %v\n\n%v", recovered, d.panicTextFooter)
			log.Criticalf(c, "Panic recovered: %s\n%s", messageText, debug.Stack())

			if sendStats { // Zero if GA is disabled
				d.reportErrorToGA(whc, measurementSender, messageText)
			}

			if whc != nil {
				if chatID, err := whc.BotChatID(); err == nil && chatID != "" {
					if responder := whc.Responder(); responder != nil {
						if _, err := responder.SendMessage(c, whc.NewMessage(ErrorIcon+" "+messageText), BotAPISendMessageOverResponse); err != nil {
							log.Errorf(c, errors.WithMessage(err, "failed to report error to user").Error())
						}
					}
				}
			}
		}

		if recovered := recover(); recovered != nil {
			reportError(recovered)
		} else if sendStats {
			log.Debugf(c, "Flushing GA...")
			if err = measurementSender.Flush(); err != nil {
				log.Warningf(c, "Failed to flush to GA: %v", err)
			} else {
				log.Debugf(c, "Sent to GA: %v items", measurementSender.QueueDepth())
			}
		} else {
			log.Debugf(c, "GA: sendStats=false")
		}
	}()

	if err != nil {
		log.Errorf(c, "Failed to create new WebhookContext: %v", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	botCoreStores := webhookHandler.CreateBotCoreStores(d.appContext, r)
	defer func() {
		if whc != nil { // TODO: How do deal with Facebook multiple entries per request?
			//log.Debugf(c, "Closing BotChatStore...")
			//chatEntity := whc.ChatEntity()
			//if chatEntity != nil && chatEntity.GetPreferredLanguage() == "" {
			//	chatEntity.SetPreferredLanguage(whc.Locale().Code5)
			//}
			if err := botCoreStores.BotChatStore.Close(c); err != nil {
				log.Errorf(c, "Failed to close BotChatStore: %v", err)
				var m MessageFromBot
				m.Text = ErrorIcon + " ERROR: Service is temporary unavailable. Probably a global outage, status at https://status.cloud.google.com/"
				if _, err := whc.Responder().SendMessage(c, m, BotAPISendMessageOverHTTPS); err != nil {
					log.Errorf(c, "Failed to report outage: %v", err)
				}
			}
		}
	}()

	for _, entryWithInputs := range entriesWithInputs {
		for i, input := range entryWithInputs.Inputs {
			if input == nil {
				panic(fmt.Sprintf("entryWithInputs.Inputs[%d] == nil", i))
			}
			d.logInput(c, i, input)
			whc = webhookHandler.CreateWebhookContext(d.appContext, r, *botContext, input, botCoreStores, measurementSender)
			responder := webhookHandler.GetResponder(w, whc) // TODO: Move inside webhookHandler.CreateWebhookContext()?
			botContext.BotSettings.Router.Dispatch(webhookHandler, responder, whc)
		}
	}
}

func (BotDriver) invalidContextOrInputs(c context.Context, w http.ResponseWriter, r *http.Request, botContext *BotContext, entriesWithInputs []EntryInputs, err error) bool {
	if err != nil {
		if _, ok := err.(ErrAuthFailed); ok {
			log.Warningf(c, "Auth failed: %v", err)
			http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
		} else if errors.Cause(err) == ErrNotImplemented {
			log.Debugf(c, err.Error())
			w.WriteHeader(http.StatusNoContent)
			//http.Error(w, "", http.StatusOK) // TODO: Decide how to handle it properly, return http.StatusNotImplemented?
		} else if _, ok := err.(*json.SyntaxError); ok {
			log.Debugf(c, errors.Wrap(err, "Request body is not valid JSON").Error())
			http.Error(w, err.Error(), http.StatusBadRequest)
		} else {
			log.Errorf(c, "Failed to call webhookHandler.GetBotContextAndInputs(router): %v", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return true
	} else if botContext == nil {
		if entriesWithInputs == nil {
			log.Warningf(c, "botContext == nil, entriesWithInputs == nil")
		} else if len(entriesWithInputs) == 0 {
			log.Warningf(c, "botContext == nil, len(entriesWithInputs) == 0")
		} else {
			log.Errorf(c, "botContext == nil, len(entriesWithInputs) == %v", len(entriesWithInputs))
		}
		return true
	} else if entriesWithInputs == nil {
		log.Errorf(c, "entriesWithInputs == nil")
		return true
	}

	switch botContext.BotSettings.Env {
	case strongo.EnvLocal:
		if r.Host != "localhost" && !strings.HasSuffix(r.Host, ".ngrok.io") {
			log.Warningf(c, "whc.GetBotSettings().Mode == Local, host: %v", r.Host)
			w.WriteHeader(http.StatusBadRequest)
			return true
		}
	case strongo.EnvProduction:
		if r.Host == "localhost" || strings.HasSuffix(r.Host, ".ngrok.io") {
			log.Warningf(c, "whc.GetBotSettings().Mode == Production, host: %v", r.Host)
			w.WriteHeader(http.StatusBadRequest)
			return true
		}
	}

	return false
}

func (BotDriver) reportErrorToGA(whc WebhookContext, measurementSender *gamp.BufferedClient, messageText string) {
	ga := whc.GA()
	gaMessage := gamp.NewException(messageText, true)

	if whc != nil { // TODO: How do deal with Facebook multiple entries per request?
		gaMessage.Common = ga.GaCommon()
	} else {
		gaMessage.Common.ClientID = "c7ea15eb-3333-4d47-a002-9d1a14996371" // TODO: move hardcoded value
		gaMessage.Common.DataSource = "bot-" + whc.BotPlatform().ID()
	}

	c := whc.Context()
	if err := ga.Queue(gaMessage); err != nil {
		log.Errorf(c, "Failed to queue exception message for GA: %v", err)
	} else {
		log.Debugf(c, "Exception message queued for GA.")
	}

	if err := measurementSender.Flush(); err != nil {
		log.Errorf(c, "Failed to flush GA buffer after exception: %v", err)
	} else {
		log.Debugf(c, "GA buffer flushed after exception")
	}
}

func (BotDriver) logInput(c context.Context, i int, input WebhookInput) {
	switch input.(type) {
	case WebhookTextMessage:
		sender := input.GetSender()
		log.Debugf(c, "BotUser#%v(%v %v) => text: %v", sender.GetID(), sender.GetFirstName(), sender.GetLastName(), input.(WebhookTextMessage).Text())
	case WebhookNewChatMembersMessage:
		newMembers := input.(WebhookNewChatMembersMessage).NewChatMembers()
		var b bytes.Buffer
		b.WriteString(fmt.Sprintf("NewChatMembers: %d", len(newMembers)))
		for i, member := range newMembers {
			b.WriteString(fmt.Sprintf("\t%d: (%v) - %v %v", i+1, member.GetUserName(), member.GetFirstName(), member.GetLastName()))
		}
		log.Debugf(c, b.String())
	case WebhookContactMessage:
		sender := input.GetSender()
		contactMessage := input.(WebhookContactMessage)
		log.Debugf(c, "BotUser#%v(%v %v) => Contact(name: %v|%v, phone number: %v)", sender.GetID(), sender.GetFirstName(), sender.GetLastName(), contactMessage.FirstName(), contactMessage.LastName(), contactMessage.PhoneNumber())
	case WebhookCallbackQuery:
		callbackQuery := input.(WebhookCallbackQuery)
		callbackData := callbackQuery.GetData()
		sender := input.GetSender()
		log.Debugf(c, "BotUser#%v(%v %v) => callback: %v", sender.GetID(), sender.GetFirstName(), sender.GetLastName(), callbackData)
	case WebhookInlineQuery:
		sender := input.GetSender()
		log.Debugf(c, "BotUser#%v(%v %v) => inline query: %v", sender.GetID(), sender.GetFirstName(), sender.GetLastName(), input.(WebhookInlineQuery).GetQuery())
	case WebhookChosenInlineResult:
		sender := input.GetSender()
		log.Debugf(c, "BotUser#%v(%v %v) => chosen InlineMessageID: %v", sender.GetID(), sender.GetFirstName(), sender.GetLastName(), input.(WebhookChosenInlineResult).GetInlineMessageID())
	case WebhookReferralMessage:
		sender := input.GetSender()
		log.Debugf(c, "BotUser#%v(%v %v) => text: %v", sender.GetID(), sender.GetFirstName(), sender.GetLastName(), input.(WebhookTextMessage).Text())
	default:
		log.Warningf(c, "Unhandled input[%v] type: %T", i, input)
	}
}
<|endoftext|>


<commit_before>package api

const (
	DefaultPortName         = "http"
	NavTruststoreFasitAlias = "nav_truststore"
	DeploymentStrategyRollingUpdate = "RollingUpdate"
	DeploymentStrategyRecreate = "Recreate"
)

func DefaultResourceRequests() []ResourceRequest {
	return []ResourceRequest{
		{
			Alias:        NavTruststoreFasitAlias,
			ResourceType: "certificate",
			PropertyMap:  map[string]string{"keystore": "NAV_TRUSTSTORE_PATH"},
		},
	}
}

func GetDefaultManifest(application string) NaisManifest {

	defaultManifest := NaisManifest{
		Replicas: Replicas{
			Min:                    2,
			Max:                    4,
			CpuThresholdPercentage: 50,
		},
		Port: 8080,
		DeploymentStrategy: DeploymentStrategyRollingUpdate,
		Prometheus: PrometheusConfig{
			Enabled: false,
			Port:    DefaultPortName,
			Path:    "/metrics",
		},
		Istio: IstioConfig{
			Enabled: false,
		},
		Healthcheck: Healthcheck{
			Liveness: Probe{
				Path:             "isAlive",
				InitialDelay:     20,
				PeriodSeconds:    10,
				FailureThreshold: 3,
				Timeout:          1,
			},
			Readiness: Probe{
				Path:             "isReady",
				InitialDelay:     20,
				PeriodSeconds:    10,
				FailureThreshold: 3,
				Timeout:          1,
			},
		},
		Ingress: Ingress{Disabled: false},
		Resources: ResourceRequirements{
			Limits: ResourceList{
				Cpu:    "500m",
				Memory: "512Mi",
			},
			Requests: ResourceList{
				Cpu:    "200m",
				Memory: "256Mi",
			},
		},
		LeaderElection: false,
		Redis:          Redis{Enabled: false},
		Secrets:        false,
	}
	defaultManifest.Image = "docker.adeo.no:5000/" + application

	return defaultManifest
}
<commit_msg>Enkelt formatering<commit_after>package api

const (
	DefaultPortName                 = "http"
	NavTruststoreFasitAlias         = "nav_truststore"
	DeploymentStrategyRollingUpdate = "RollingUpdate"
	DeploymentStrategyRecreate      = "Recreate"
)

func DefaultResourceRequests() []ResourceRequest {
	return []ResourceRequest{
		{
			Alias:        NavTruststoreFasitAlias,
			ResourceType: "certificate",
			PropertyMap:  map[string]string{"keystore": "NAV_TRUSTSTORE_PATH"},
		},
	}
}

func GetDefaultManifest(application string) NaisManifest {

	defaultManifest := NaisManifest{
		Replicas: Replicas{
			Min:                    2,
			Max:                    4,
			CpuThresholdPercentage: 50,
		},
		Port:               8080,
		DeploymentStrategy: DeploymentStrategyRollingUpdate,
		Prometheus: PrometheusConfig{
			Enabled: false,
			Port:    DefaultPortName,
			Path:    "/metrics",
		},
		Istio: IstioConfig{
			Enabled: false,
		},
		Healthcheck: Healthcheck{
			Liveness: Probe{
				Path:             "isAlive",
				InitialDelay:     20,
				PeriodSeconds:    10,
				FailureThreshold: 3,
				Timeout:          1,
			},
			Readiness: Probe{
				Path:             "isReady",
				InitialDelay:     20,
				PeriodSeconds:    10,
				FailureThreshold: 3,
				Timeout:          1,
			},
		},
		Ingress: Ingress{Disabled: false},
		Resources: ResourceRequirements{
			Limits: ResourceList{
				Cpu:    "500m",
				Memory: "512Mi",
			},
			Requests: ResourceList{
				Cpu:    "200m",
				Memory: "256Mi",
			},
		},
		LeaderElection: false,
		Redis:          Redis{Enabled: false},
		Secrets:        false,
	}
	defaultManifest.Image = "docker.adeo.no:5000/" + application

	return defaultManifest
}
<|endoftext|>


<commit_before>// Package oplog provides a generic oplog/replication system for REST APIs.
//
// Most of the time, the oplog service is used thru the oplogd agent which uses this
// package. But in the case your application is written in Go, you may want to integrate
// at the code level.
package oplog

import (
	"io"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/cenkalti/backoff"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type OpLog struct {
	s     *mgo.Session
	Stats *Stats
	// ObjectURL is a template URL to be used to generate reference URL to operation's objects.
	// The URL can use {{type}} and {{id}} template as follow: http://api.mydomain.com/{{type}}/{{id}}.
	// If not provided, no "ref" field will be included in oplog events.
	ObjectURL string
}

type OperationDataMap map[string]OperationData

// New returns an OpLog connected to the given provided mongo URL.
// If the capped collection does not exists, it will be created with the max
// size defined by maxBytes parameter.
func New(mongoURL string, maxBytes int) (*OpLog, error) {
	session, err := mgo.Dial(mongoURL)
	if err != nil {
		return nil, err
	}
	session.SetSyncTimeout(10 * time.Second)
	session.SetSocketTimeout(20 * time.Second)
	session.SetMode(mgo.Monotonic, true)
	session.SetSafe(&mgo.Safe{})
	stats := NewStats()
	oplog := &OpLog{
		s:     session,
		Stats: &stats,
	}
	oplog.init(maxBytes)
	return oplog, nil
}

// DB returns the Mongo database object used by the oplog
func (oplog *OpLog) DB() *mgo.Database {
	return oplog.s.Copy().DB("")
}

// init creates capped collection if it does not exists.
func (oplog *OpLog) init(maxBytes int) {
	oplogExists := false
	objectsExists := false
	names, _ := oplog.s.DB("").CollectionNames()
	for _, name := range names {
		switch name {
		case "oplog_events":
			oplogExists = true
		case "oplog_states":
			objectsExists = true
		}
	}
	if !oplogExists {
		log.Info("OPLOG creating capped collection")
		err := oplog.s.DB("").C("oplog_events").Create(&mgo.CollectionInfo{
			Capped:   true,
			MaxBytes: maxBytes,
		})
		if err != nil {
			log.Fatal(err)
		}
	}
	if !objectsExists {
		log.Info("OPLOG creating objects index")
		c := oplog.s.DB("").C("oplog_states")
		// Replication query
		if err := c.EnsureIndexKey("event", "ts"); err != nil {
			log.Fatal(err)
		}
		// Replication query with a filter on types
		if err := c.EnsureIndexKey("event", "data.t", "ts"); err != nil {
			log.Fatal(err)
		}
	}
}

// Ingest appends an operation into the OpLog thru a channel
func (oplog *OpLog) Ingest(ops <-chan *Operation) {
	db := oplog.DB()
	defer db.Session.Close()
	for {
		select {
		case op := <-ops:
			oplog.Stats.QueueSize.Set(int64(len(ops)))
			oplog.Append(op, db)
		}
	}
}

// Append appends an operation into the OpLog
//
// If the db parameter is not nil, the passed db connection is used. In case of
// error, the db pointer may be replaced by a new alive session.
func (oplog *OpLog) Append(op *Operation, db *mgo.Database) {
	if db == nil {
		db = oplog.DB()
		defer db.Session.Close()
	}
	log.Debugf("OPLOG ingest operation: %#v", op.Info())
	b := backoff.NewExponentialBackOff()
	b.MaxElapsedTime = 0 // Retry forever
	b.Reset()
	for {
		if err := db.C("oplog_events").Insert(op); err != nil {
			log.Warnf("OPLOG can't insert operation, retrying: %s", err)
			// Retry with backoff
			time.Sleep(b.NextBackOff())
			db.Session.Refresh()
			continue
		}
		break
	}
	// Apply the operation on the state collection
	event := op.Event
	if event == "update" {
		// Only store insert and delete events in the object stats collection as
		// only the final stat of the object is stored.
		event = "insert"
	}
	o := ObjectState{
		Id:        op.Data.GetId(),
		Event:     event,
		Timestamp: time.Now(),
		Data:      op.Data,
	}
	b.Reset()
	for {
		if _, err := db.C("oplog_states").Upsert(bson.M{"_id": o.Id}, o); err != nil {
			log.Warnf("OPLOG can't upsert object, retrying: %s", err)
			// Retry with backoff
			time.Sleep(b.NextBackOff())
			db.Session.Refresh()
			continue
		}
		break
	}
	oplog.Stats.EventsIngested.Add(1)
}

// Diff finds which objects must be created or deleted in order to fix the delta
//
// The createMap is a map pointing to all objects present in the source database.
// The function search of differences between the passed map and the oplog database and
// remove objects identical in both sides from the createMap and populate the deleteMap
// with objects that are present in the oplog database but not in the source database.
// If an object is present in both createMap and the oplog database but timestamp of the
// oplog object is earlier than createMap's, the object is added to the updateMap.
func (oplog *OpLog) Diff(createMap OperationDataMap, updateMap OperationDataMap, deleteMap OperationDataMap) error {
	db := oplog.DB()
	defer db.Session.Close()

	// Find the most recent timestamp
	dumpTime := time.Unix(0, 0)
	for _, obd := range createMap {
		if obd.Timestamp.After(dumpTime) {
			dumpTime = obd.Timestamp
		}
	}

	obs := ObjectState{}
	iter := db.C("oplog_states").Find(bson.M{}).Iter()
	for iter.Next(&obs) {
		if obs.Event == "deleted" {
			if obd, ok := createMap[obs.Id]; ok {
				// If the object is present in the dump but deleted in the oplog, it means
				// that it has been deleted between the dump creation and the sync
				// (if the oplog version is more recent)
				if obd.Timestamp.Before(obs.Data.Timestamp) {
					delete(createMap, obs.Id)
				}
			}
		} else {
			if obd, ok := createMap[obs.Id]; ok {
				// Object exists on both sides, remove it from the create map
				delete(createMap, obs.Id)
				// If the dump object is newer than oplog's, add it to the update map
				if obs.Data.Timestamp.Before(obd.Timestamp) {
					updateMap[obs.Id] = obd
				}
			} else {
				// The object only exists in the oplog db, add it to the delete map
				// if the timestamp of the found object is older than oldest object
				// in the dump in order to ensure we don't delete an object which
				// have been created between the dump creation and the sync.
				if obs.Data.Timestamp.Before(dumpTime) {
					deleteMap[obs.Id] = *obs.Data
					delete(createMap, obs.Id)
				}
			}
		}
	}
	if iter.Err() != nil {
		return iter.Err()
	}

	return nil
}

// HasId checks if an operation id is present in the capped collection.
func (oplog *OpLog) HasId(id LastId) (bool, error) {
	if olid, ok := id.(*OperationLastId); ok {
		db := oplog.DB()
		defer db.Session.Close()
		count, err := db.C("oplog_events").FindId(olid.ObjectId).Count()
		return count != 0, err
	}

	// Replication id are always found as they are timestamps
	return true, nil
}

// LastId returns the most recently inserted operation id if any or nil if oplog is empty
func (oplog *OpLog) LastId() (LastId, error) {
	db := oplog.DB()
	defer db.Session.Close()
	operation := &Operation{}
	err := db.C("oplog_events").Find(nil).Sort("-$natural").One(operation)
	if err == mgo.ErrNotFound {
		return nil, nil
	}
	if operation.Id != nil {
		return &OperationLastId{operation.Id}, nil
	}
	return nil, err
}

// iter creates either an iterator on the objects collection if the lastId is a timestamp
// or tailable cursor on the oplog capped collection otherwise
func (oplog *OpLog) iter(db *mgo.Database, lastId LastId, filter OpLogFilter) (iter *mgo.Iter, err error, streaming bool) {
	query := bson.M{}
	filter.apply(&query)

	if r, ok := lastId.(*ReplicationLastId); ok {
		if r.int64 > 0 {
			// Id is a timestamp, timestamp are always valid
			query["ts"] = bson.M{"$gte": time.Unix(0, r.int64*1000000)}
		}
		query["event"] = bson.M{"$ne": "delete"}
		iter = db.C("oplog_states").Find(query).Sort("ts").Iter()
		streaming = false
	} else {
		if lastId != nil {
			o := lastId.(*OperationLastId)
			query["_id"] = bson.M{"$gt": o.ObjectId}
		}
		iter = db.C("oplog_events").Find(query).Sort("$natural").Tail(5 * time.Second)
		streaming = true
	}
	return
}

// Tail tails all the new operations in the oplog and send the operation in
// the given channel. If the lastId parameter is given, all operation posted after
// this event will be returned.
//
// If the lastId is a ReplicationLastId (unix timestamp in milliseconds), the tailing will
// start by replicating all the objects last updated after the timestamp.
//
// Giving a lastId of 0 mean replicating all the stored objects before tailing the live updates.
//
// The filter argument can be used to filter on some type of objects or objects with given parrents.
//
// The create, update, delete events are streamed back to the sender thru the out channel
func (oplog *OpLog) Tail(lastId LastId, filter OpLogFilter, out chan<- io.WriterTo, stop <-chan bool) {
	var lastEv GenericEvent

	if r, ok := lastId.(*ReplicationLastId); ok && r.int64 == 0 {
		// When full replication is requested, start by sending a "reset" event to instruct
		// the consumer to reset its database before processing further operations.
		// The id is 1 so if connection is lost after this event and consumer processed the event,
		// the connection recover won't trigger a second "reset" event.
		out <- &OpLogEvent{
			Id:    "1",
			Event: "reset",
		}
	}

	done := false
	mu := &sync.RWMutex{}
	isDone := func() bool {
		mu.RLock()
		defer mu.RUnlock()
		return done
	}

	wg := sync.WaitGroup{}

	wg.Add(1)
	go func() {
		defer wg.Done()

		db := oplog.DB()
		defer db.Session.Close()

		b := backoff.NewExponentialBackOff()
		b.MaxElapsedTime = 0 // Retry forever
		b.Reset()

		for {
			iter, err, streaming := oplog.iter(db, lastId, filter)

			if err != nil {
				log.Warnf("OPLOG tail failed with error, try to reconnect: %s", err)
			} else if streaming {
				log.Debug("OPLOG start live updates")
				b.Reset()

				operation := Operation{}
				for {
					for iter.Next(&operation) {
						if isDone() {
							iter.Close()
							break
						}
						if oplog.ObjectURL != "" {
							operation.Data.genRef(oplog.ObjectURL)
						}
						out <- operation
						lastEv = operation
					}

					if isDone() || !iter.Timeout() {
						break
					}
				}

				if isDone() {
					break
				}

				if iter.Err() != nil {
					log.Warnf("OPLOG tail failed with error, try to reconnect: %s", iter.Err())
				}
			} else {
				log.Debug("OPLOG start replication")
				b.Reset()
				// Capture the current oplog position in order to resume at this position
				// once initial replication is done
				replicationFallbackId, err := oplog.LastId()

				if err == nil {
					object := ObjectState{}
					for iter.Next(&object) {
						if isDone() {
							iter.Close()
							break
						}
						if oplog.ObjectURL != "" {
							object.Data.genRef(oplog.ObjectURL)
						}
						out <- object
						lastEv = object
					}
				}

				if isDone() {
					break
				}

				if err != nil || iter.Err() != nil {
					log.Warnf("OPLOG replication failed with error, retrying: %s", iter.Err())
				} else {
					// Replication is done, notify and swtich to live event stream
					//
					// Send a "live" operation to inform the consumer it is no live event stream.
					// We use the last event id here in order to ensure the consumer will resume
					// the replication starting at this point in time in case of a failure after
					// the "live" event.
					liveId := "" // default value
					if lastEv != nil {
						liveId = lastEv.GetEventId().String()
					}
					out <- &OpLogEvent{
						Id:    liveId,
						Event: "live",
					}
					// Switch to live update at the last operation id inserted before the replication
					// was started
					lastId = replicationFallbackId
					lastEv = nil
				}
			}

			// Prepare for retry with backoff
			iter.Close()
			time.Sleep(b.NextBackOff())
			db.Session.Refresh()
			if lastEv != nil {
				lastId = lastEv.GetEventId()
			}
		}
	}()

	select {
	case <-stop:
		mu.Lock()
		done = true
		mu.Unlock()
		wg.Wait()
		log.Info("OPLOG tail closed")
	}
}
<commit_msg>Fix a collection init issue<commit_after>// Package oplog provides a generic oplog/replication system for REST APIs.
//
// Most of the time, the oplog service is used thru the oplogd agent which uses this
// package. But in the case your application is written in Go, you may want to integrate
// at the code level.
package oplog

import (
	"io"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/cenkalti/backoff"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type OpLog struct {
	s     *mgo.Session
	Stats *Stats
	// ObjectURL is a template URL to be used to generate reference URL to operation's objects.
	// The URL can use {{type}} and {{id}} template as follow: http://api.mydomain.com/{{type}}/{{id}}.
	// If not provided, no "ref" field will be included in oplog events.
	ObjectURL string
}

type OperationDataMap map[string]OperationData

// New returns an OpLog connected to the given provided mongo URL.
// If the capped collection does not exists, it will be created with the max
// size defined by maxBytes parameter.
func New(mongoURL string, maxBytes int) (*OpLog, error) {
	session, err := mgo.Dial(mongoURL)
	if err != nil {
		return nil, err
	}
	session.SetSyncTimeout(10 * time.Second)
	session.SetSocketTimeout(20 * time.Second)
	session.SetSafe(&mgo.Safe{})
	stats := NewStats()
	oplog := &OpLog{
		s:     session,
		Stats: &stats,
	}
	oplog.init(maxBytes)
	// Setting monotonic before collection fails with a "not master" error
	session.SetMode(mgo.Monotonic, true)
	return oplog, nil
}

// DB returns the Mongo database object used by the oplog
func (oplog *OpLog) DB() *mgo.Database {
	return oplog.s.Copy().DB("")
}

// init creates capped collection if it does not exists.
func (oplog *OpLog) init(maxBytes int) {
	oplogExists := false
	objectsExists := false
	names, _ := oplog.s.DB("").CollectionNames()
	for _, name := range names {
		switch name {
		case "oplog_events":
			oplogExists = true
		case "oplog_states":
			objectsExists = true
		}
	}
	if !oplogExists {
		log.Info("OPLOG creating capped collection")
		err := oplog.s.DB("").C("oplog_events").Create(&mgo.CollectionInfo{
			Capped:   true,
			MaxBytes: maxBytes,
		})
		if err != nil {
			log.Fatal(err)
		}
	}
	if !objectsExists {
		log.Info("OPLOG creating objects index")
		c := oplog.s.DB("").C("oplog_states")
		// Replication query
		if err := c.EnsureIndexKey("event", "ts"); err != nil {
			log.Fatal(err)
		}
		// Replication query with a filter on types
		if err := c.EnsureIndexKey("event", "data.t", "ts"); err != nil {
			log.Fatal(err)
		}
	}
}

// Ingest appends an operation into the OpLog thru a channel
func (oplog *OpLog) Ingest(ops <-chan *Operation) {
	db := oplog.DB()
	defer db.Session.Close()
	for {
		select {
		case op := <-ops:
			oplog.Stats.QueueSize.Set(int64(len(ops)))
			oplog.Append(op, db)
		}
	}
}

// Append appends an operation into the OpLog
//
// If the db parameter is not nil, the passed db connection is used. In case of
// error, the db pointer may be replaced by a new alive session.
func (oplog *OpLog) Append(op *Operation, db *mgo.Database) {
	if db == nil {
		db = oplog.DB()
		defer db.Session.Close()
	}
	log.Debugf("OPLOG ingest operation: %#v", op.Info())
	b := backoff.NewExponentialBackOff()
	b.MaxElapsedTime = 0 // Retry forever
	b.Reset()
	for {
		if err := db.C("oplog_events").Insert(op); err != nil {
			log.Warnf("OPLOG can't insert operation, retrying: %s", err)
			// Retry with backoff
			time.Sleep(b.NextBackOff())
			db.Session.Refresh()
			continue
		}
		break
	}
	// Apply the operation on the state collection
	event := op.Event
	if event == "update" {
		// Only store insert and delete events in the object stats collection as
		// only the final stat of the object is stored.
		event = "insert"
	}
	o := ObjectState{
		Id:        op.Data.GetId(),
		Event:     event,
		Timestamp: time.Now(),
		Data:      op.Data,
	}
	b.Reset()
	for {
		if _, err := db.C("oplog_states").Upsert(bson.M{"_id": o.Id}, o); err != nil {
			log.Warnf("OPLOG can't upsert object, retrying: %s", err)
			// Retry with backoff
			time.Sleep(b.NextBackOff())
			db.Session.Refresh()
			continue
		}
		break
	}
	oplog.Stats.EventsIngested.Add(1)
}

// Diff finds which objects must be created or deleted in order to fix the delta
//
// The createMap is a map pointing to all objects present in the source database.
// The function search of differences between the passed map and the oplog database and
// remove objects identical in both sides from the createMap and populate the deleteMap
// with objects that are present in the oplog database but not in the source database.
// If an object is present in both createMap and the oplog database but timestamp of the
// oplog object is earlier than createMap's, the object is added to the updateMap.
func (oplog *OpLog) Diff(createMap OperationDataMap, updateMap OperationDataMap, deleteMap OperationDataMap) error {
	db := oplog.DB()
	defer db.Session.Close()

	// Find the most recent timestamp
	dumpTime := time.Unix(0, 0)
	for _, obd := range createMap {
		if obd.Timestamp.After(dumpTime) {
			dumpTime = obd.Timestamp
		}
	}

	obs := ObjectState{}
	iter := db.C("oplog_states").Find(bson.M{}).Iter()
	for iter.Next(&obs) {
		if obs.Event == "deleted" {
			if obd, ok := createMap[obs.Id]; ok {
				// If the object is present in the dump but deleted in the oplog, it means
				// that it has been deleted between the dump creation and the sync
				// (if the oplog version is more recent)
				if obd.Timestamp.Before(obs.Data.Timestamp) {
					delete(createMap, obs.Id)
				}
			}
		} else {
			if obd, ok := createMap[obs.Id]; ok {
				// Object exists on both sides, remove it from the create map
				delete(createMap, obs.Id)
				// If the dump object is newer than oplog's, add it to the update map
				if obs.Data.Timestamp.Before(obd.Timestamp) {
					updateMap[obs.Id] = obd
				}
			} else {
				// The object only exists in the oplog db, add it to the delete map
				// if the timestamp of the found object is older than oldest object
				// in the dump in order to ensure we don't delete an object which
				// have been created between the dump creation and the sync.
				if obs.Data.Timestamp.Before(dumpTime) {
					deleteMap[obs.Id] = *obs.Data
					delete(createMap, obs.Id)
				}
			}
		}
	}
	if iter.Err() != nil {
		return iter.Err()
	}

	return nil
}

// HasId checks if an operation id is present in the capped collection.
func (oplog *OpLog) HasId(id LastId) (bool, error) {
	if olid, ok := id.(*OperationLastId); ok {
		db := oplog.DB()
		defer db.Session.Close()
		count, err := db.C("oplog_events").FindId(olid.ObjectId).Count()
		return count != 0, err
	}

	// Replication id are always found as they are timestamps
	return true, nil
}

// LastId returns the most recently inserted operation id if any or nil if oplog is empty
func (oplog *OpLog) LastId() (LastId, error) {
	db := oplog.DB()
	defer db.Session.Close()
	operation := &Operation{}
	err := db.C("oplog_events").Find(nil).Sort("-$natural").One(operation)
	if err == mgo.ErrNotFound {
		return nil, nil
	}
	if operation.Id != nil {
		return &OperationLastId{operation.Id}, nil
	}
	return nil, err
}

// iter creates either an iterator on the objects collection if the lastId is a timestamp
// or tailable cursor on the oplog capped collection otherwise
func (oplog *OpLog) iter(db *mgo.Database, lastId LastId, filter OpLogFilter) (iter *mgo.Iter, err error, streaming bool) {
	query := bson.M{}
	filter.apply(&query)

	if r, ok := lastId.(*ReplicationLastId); ok {
		if r.int64 > 0 {
			// Id is a timestamp, timestamp are always valid
			query["ts"] = bson.M{"$gte": time.Unix(0, r.int64*1000000)}
		}
		query["event"] = bson.M{"$ne": "delete"}
		iter = db.C("oplog_states").Find(query).Sort("ts").Iter()
		streaming = false
	} else {
		if lastId != nil {
			o := lastId.(*OperationLastId)
			query["_id"] = bson.M{"$gt": o.ObjectId}
		}
		iter = db.C("oplog_events").Find(query).Sort("$natural").Tail(5 * time.Second)
		streaming = true
	}
	return
}

// Tail tails all the new operations in the oplog and send the operation in
// the given channel. If the lastId parameter is given, all operation posted after
// this event will be returned.
//
// If the lastId is a ReplicationLastId (unix timestamp in milliseconds), the tailing will
// start by replicating all the objects last updated after the timestamp.
//
// Giving a lastId of 0 mean replicating all the stored objects before tailing the live updates.
//
// The filter argument can be used to filter on some type of objects or objects with given parrents.
//
// The create, update, delete events are streamed back to the sender thru the out channel
func (oplog *OpLog) Tail(lastId LastId, filter OpLogFilter, out chan<- io.WriterTo, stop <-chan bool) {
	var lastEv GenericEvent

	if r, ok := lastId.(*ReplicationLastId); ok && r.int64 == 0 {
		// When full replication is requested, start by sending a "reset" event to instruct
		// the consumer to reset its database before processing further operations.
		// The id is 1 so if connection is lost after this event and consumer processed the event,
		// the connection recover won't trigger a second "reset" event.
		out <- &OpLogEvent{
			Id:    "1",
			Event: "reset",
		}
	}

	done := false
	mu := &sync.RWMutex{}
	isDone := func() bool {
		mu.RLock()
		defer mu.RUnlock()
		return done
	}

	wg := sync.WaitGroup{}

	wg.Add(1)
	go func() {
		defer wg.Done()

		db := oplog.DB()
		defer db.Session.Close()

		b := backoff.NewExponentialBackOff()
		b.MaxElapsedTime = 0 // Retry forever
		b.Reset()

		for {
			iter, err, streaming := oplog.iter(db, lastId, filter)

			if err != nil {
				log.Warnf("OPLOG tail failed with error, try to reconnect: %s", err)
			} else if streaming {
				log.Debug("OPLOG start live updates")
				b.Reset()

				operation := Operation{}
				for {
					for iter.Next(&operation) {
						if isDone() {
							iter.Close()
							break
						}
						if oplog.ObjectURL != "" {
							operation.Data.genRef(oplog.ObjectURL)
						}
						out <- operation
						lastEv = operation
					}

					if isDone() || !iter.Timeout() {
						break
					}
				}

				if isDone() {
					break
				}

				if iter.Err() != nil {
					log.Warnf("OPLOG tail failed with error, try to reconnect: %s", iter.Err())
				}
			} else {
				log.Debug("OPLOG start replication")
				b.Reset()
				// Capture the current oplog position in order to resume at this position
				// once initial replication is done
				replicationFallbackId, err := oplog.LastId()

				if err == nil {
					object := ObjectState{}
					for iter.Next(&object) {
						if isDone() {
							iter.Close()
							break
						}
						if oplog.ObjectURL != "" {
							object.Data.genRef(oplog.ObjectURL)
						}
						out <- object
						lastEv = object
					}
				}

				if isDone() {
					break
				}

				if err != nil || iter.Err() != nil {
					log.Warnf("OPLOG replication failed with error, retrying: %s", iter.Err())
				} else {
					// Replication is done, notify and swtich to live event stream
					//
					// Send a "live" operation to inform the consumer it is no live event stream.
					// We use the last event id here in order to ensure the consumer will resume
					// the replication starting at this point in time in case of a failure after
					// the "live" event.
					liveId := "" // default value
					if lastEv != nil {
						liveId = lastEv.GetEventId().String()
					}
					out <- &OpLogEvent{
						Id:    liveId,
						Event: "live",
					}
					// Switch to live update at the last operation id inserted before the replication
					// was started
					lastId = replicationFallbackId
					lastEv = nil
				}
			}

			// Prepare for retry with backoff
			iter.Close()
			time.Sleep(b.NextBackOff())
			db.Session.Refresh()
			if lastEv != nil {
				lastId = lastEv.GetEventId()
			}
		}
	}()

	select {
	case <-stop:
		mu.Lock()
		done = true
		mu.Unlock()
		wg.Wait()
		log.Info("OPLOG tail closed")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package disk

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"testing"

	"github.com/stretchr/testify/assert"
)

// copied from k8s.io/client-go/transport/round_trippers_test.go
type testRoundTripper struct {
	Request  *http.Request
	Response *http.Response
	Err      error
}

func (rt *testRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	rt.Request = req
	return rt.Response, rt.Err
}

func TestCacheRoundTripper(t *testing.T) {
	rt := &testRoundTripper{}
	cacheDir, err := ioutil.TempDir("", "cache-rt")
	defer os.RemoveAll(cacheDir)
	if err != nil {
		t.Fatal(err)
	}
	cache := newCacheRoundTripper(cacheDir, rt)

	// First call, caches the response
	req := &http.Request{
		Method: http.MethodGet,
		URL:    &url.URL{Host: "localhost"},
	}
	rt.Response = &http.Response{
		Header:     http.Header{"ETag": []string{`"123456"`}},
		Body:       ioutil.NopCloser(bytes.NewReader([]byte("Content"))),
		StatusCode: http.StatusOK,
	}
	resp, err := cache.RoundTrip(req)
	if err != nil {
		t.Fatal(err)
	}
	content, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
	}
	if string(content) != "Content" {
		t.Errorf(`Expected Body to be "Content", got %q`, string(content))
	}

	// Second call, returns cached response
	req = &http.Request{
		Method: http.MethodGet,
		URL:    &url.URL{Host: "localhost"},
	}
	rt.Response = &http.Response{
		StatusCode: http.StatusNotModified,
		Body:       ioutil.NopCloser(bytes.NewReader([]byte("Other Content"))),
	}

	resp, err = cache.RoundTrip(req)
	if err != nil {
		t.Fatal(err)
	}

	// Read body and make sure we have the initial content
	content, err = ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		t.Fatal(err)
	}
	if string(content) != "Content" {
		t.Errorf("Invalid content read from cache %q", string(content))
	}
}

func TestCacheRoundTripperPathPerm(t *testing.T) {
	assert := assert.New(t)

	rt := &testRoundTripper{}
	cacheDir, err := ioutil.TempDir("", "cache-rt")
	os.RemoveAll(cacheDir)
	defer os.RemoveAll(cacheDir)

	if err != nil {
		t.Fatal(err)
	}
	cache := newCacheRoundTripper(cacheDir, rt)

	// First call, caches the response
	req := &http.Request{
		Method: http.MethodGet,
		URL:    &url.URL{Host: "localhost"},
	}
	rt.Response = &http.Response{
		Header:     http.Header{"ETag": []string{`"123456"`}},
		Body:       ioutil.NopCloser(bytes.NewReader([]byte("Content"))),
		StatusCode: http.StatusOK,
	}
	resp, err := cache.RoundTrip(req)
	if err != nil {
		t.Fatal(err)
	}
	content, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
	}
	if string(content) != "Content" {
		t.Errorf(`Expected Body to be "Content", got %q`, string(content))
	}

	err = filepath.Walk(cacheDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			assert.Equal(os.FileMode(0750), info.Mode().Perm())
		} else {
			assert.Equal(os.FileMode(0660), info.Mode().Perm())
		}
		return nil
	})
	assert.NoError(err)
}
<commit_msg>Add a benchmark for the discovery cache RoundTripper<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package disk

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"testing"

	"github.com/gregjones/httpcache/diskcache"
	"github.com/peterbourgon/diskv"
	"github.com/stretchr/testify/assert"
)

// copied from k8s.io/client-go/transport/round_trippers_test.go
type testRoundTripper struct {
	Request  *http.Request
	Response *http.Response
	Err      error
}

func (rt *testRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	rt.Request = req
	return rt.Response, rt.Err
}

// NOTE(negz): We're adding a benchmark for an external dependency in order to
// prove that one that will be added in a subsequent commit improves write
// performance.
func BenchmarkDiskCache(b *testing.B) {
	cacheDir, err := ioutil.TempDir("", "cache-rt")
	if err != nil {
		b.Fatal(err)
	}
	defer os.RemoveAll(cacheDir)

	d := diskv.New(diskv.Options{
		PathPerm: os.FileMode(0750),
		FilePerm: os.FileMode(0660),
		BasePath: cacheDir,
		TempDir:  filepath.Join(cacheDir, ".diskv-temp"),
	})

	k := "localhost:8080/apis/batch/v1.json"
	v, err := ioutil.ReadFile("../../testdata/apis/batch/v1.json")
	if err != nil {
		b.Fatal(err)
	}

	c := diskcache.NewWithDiskv(d)

	for n := 0; n < b.N; n++ {
		c.Set(k, v)
		c.Get(k)
		c.Delete(k)
	}
}

func TestCacheRoundTripper(t *testing.T) {
	rt := &testRoundTripper{}
	cacheDir, err := ioutil.TempDir("", "cache-rt")
	defer os.RemoveAll(cacheDir)
	if err != nil {
		t.Fatal(err)
	}
	cache := newCacheRoundTripper(cacheDir, rt)

	// First call, caches the response
	req := &http.Request{
		Method: http.MethodGet,
		URL:    &url.URL{Host: "localhost"},
	}
	rt.Response = &http.Response{
		Header:     http.Header{"ETag": []string{`"123456"`}},
		Body:       ioutil.NopCloser(bytes.NewReader([]byte("Content"))),
		StatusCode: http.StatusOK,
	}
	resp, err := cache.RoundTrip(req)
	if err != nil {
		t.Fatal(err)
	}
	content, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
	}
	if string(content) != "Content" {
		t.Errorf(`Expected Body to be "Content", got %q`, string(content))
	}

	// Second call, returns cached response
	req = &http.Request{
		Method: http.MethodGet,
		URL:    &url.URL{Host: "localhost"},
	}
	rt.Response = &http.Response{
		StatusCode: http.StatusNotModified,
		Body:       ioutil.NopCloser(bytes.NewReader([]byte("Other Content"))),
	}

	resp, err = cache.RoundTrip(req)
	if err != nil {
		t.Fatal(err)
	}

	// Read body and make sure we have the initial content
	content, err = ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		t.Fatal(err)
	}
	if string(content) != "Content" {
		t.Errorf("Invalid content read from cache %q", string(content))
	}
}

func TestCacheRoundTripperPathPerm(t *testing.T) {
	assert := assert.New(t)

	rt := &testRoundTripper{}
	cacheDir, err := ioutil.TempDir("", "cache-rt")
	os.RemoveAll(cacheDir)
	defer os.RemoveAll(cacheDir)

	if err != nil {
		t.Fatal(err)
	}
	cache := newCacheRoundTripper(cacheDir, rt)

	// First call, caches the response
	req := &http.Request{
		Method: http.MethodGet,
		URL:    &url.URL{Host: "localhost"},
	}
	rt.Response = &http.Response{
		Header:     http.Header{"ETag": []string{`"123456"`}},
		Body:       ioutil.NopCloser(bytes.NewReader([]byte("Content"))),
		StatusCode: http.StatusOK,
	}
	resp, err := cache.RoundTrip(req)
	if err != nil {
		t.Fatal(err)
	}
	content, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
	}
	if string(content) != "Content" {
		t.Errorf(`Expected Body to be "Content", got %q`, string(content))
	}

	err = filepath.Walk(cacheDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			assert.Equal(os.FileMode(0750), info.Mode().Perm())
		} else {
			assert.Equal(os.FileMode(0660), info.Mode().Perm())
		}
		return nil
	})
	assert.NoError(err)
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcutil

import (
	"math/big"
	"strings"
)

// alphabet is the modified base58 alphabet used by Bitcoin.
const alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"

var bigRadix = big.NewInt(58)
var bigZero = big.NewInt(0)

// Base58Decode decodes a modified base58 string to a byte slice.
func Base58Decode(b string) []byte {
	answer := big.NewInt(0)
	j := big.NewInt(1)

	for i := len(b) - 1; i >= 0; i-- {
		tmp := strings.IndexAny(alphabet, string(b[i]))
		if tmp == -1 {
			return []byte("")
		}
		idx := big.NewInt(int64(tmp))
		tmp1 := big.NewInt(0)
		tmp1.Mul(j, idx)

		answer.Add(answer, tmp1)
		j.Mul(j, bigRadix)
	}

	tmpval := answer.Bytes()

	var numZeros int
	for numZeros = 0; numZeros < len(b); numZeros++ {
		if b[numZeros] != alphabet[0] {
			break
		}
	}
	flen := numZeros + len(tmpval)
	val := make([]byte, flen, flen)
	copy(val[numZeros:], tmpval)

	return val
}

// Base58Encode encodes a byte slice to a modified base58 string.
func Base58Encode(b []byte) string {
	x := new(big.Int)
	x.SetBytes(b)

	answer := make([]byte, 0)
	for x.Cmp(bigZero) > 0 {
		mod := new(big.Int)
		x.DivMod(x, bigRadix, mod)
		answer = append(answer, alphabet[mod.Int64()])
	}

	// leading zero bytes
	for _, i := range b {
		if i != 0 {
			break
		}
		answer = append(answer, alphabet[0])
	}

	// reverse
	alen := len(answer)
	for i := 0; i < alen/2; i++ {
		answer[i], answer[alen-1-i] = answer[alen-1-i], answer[i]
	}

	return string(answer)
}
<commit_msg>Preallocate space for the answer in Base58Encode.<commit_after>// Copyright (c) 2013-2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcutil

import (
	"math/big"
	"strings"
)

// alphabet is the modified base58 alphabet used by Bitcoin.
const alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"

var bigRadix = big.NewInt(58)
var bigZero = big.NewInt(0)

// Base58Decode decodes a modified base58 string to a byte slice.
func Base58Decode(b string) []byte {
	answer := big.NewInt(0)
	j := big.NewInt(1)

	for i := len(b) - 1; i >= 0; i-- {
		tmp := strings.IndexAny(alphabet, string(b[i]))
		if tmp == -1 {
			return []byte("")
		}
		idx := big.NewInt(int64(tmp))
		tmp1 := big.NewInt(0)
		tmp1.Mul(j, idx)

		answer.Add(answer, tmp1)
		j.Mul(j, bigRadix)
	}

	tmpval := answer.Bytes()

	var numZeros int
	for numZeros = 0; numZeros < len(b); numZeros++ {
		if b[numZeros] != alphabet[0] {
			break
		}
	}
	flen := numZeros + len(tmpval)
	val := make([]byte, flen, flen)
	copy(val[numZeros:], tmpval)

	return val
}

// Base58Encode encodes a byte slice to a modified base58 string.
func Base58Encode(b []byte) string {
	x := new(big.Int)
	x.SetBytes(b)

	answer := make([]byte, 0, len(b)*136/100)
	for x.Cmp(bigZero) > 0 {
		mod := new(big.Int)
		x.DivMod(x, bigRadix, mod)
		answer = append(answer, alphabet[mod.Int64()])
	}

	// leading zero bytes
	for _, i := range b {
		if i != 0 {
			break
		}
		answer = append(answer, alphabet[0])
	}

	// reverse
	alen := len(answer)
	for i := 0; i < alen/2; i++ {
		answer[i], answer[alen-1-i] = answer[alen-1-i], answer[i]
	}

	return string(answer)
}
<|endoftext|>