<commit_before>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"time"

	"github.com/square/keywhiz-fs/log"
)

// SecretBackend represents an interface for storing secrets.
type SecretBackend interface {
	Secret(string) (secret *Secret, err error)
	SecretList() (secretList []Secret, ok bool)
}

// Timeouts contains configuration for timeouts:
// timeout_backend_deadline: optimistic timeout to wait for cache
// timeout_max_wait: timeout for client to get data from server
type Timeouts struct {
	// FUSE may make many lookups in quick succession. If cached data is recent within the threshold,
	// a backend request is not attempted.
	Fresh time.Duration
	// BackendDeadline is distinct from the backend timeout. It is an optimistic timeout to wait
	// until resorting to cached data.
	BackendDeadline time.Duration
	MaxWait         time.Duration
	// Controls how long to keep a deleted entry before purging it.
	DeletionDelay time.Duration
}

// Cache contains necessary state to return secrets, using previously cached content or retrieving
// from a server if necessary.
type Cache struct {
	*log.Logger
	secretMap *SecretMap
	backend   SecretBackend
	timeouts  Timeouts
	now       func() time.Time
}

type secretResult struct {
	secret *Secret
	err    error
}

// NewCache initializes a Cache.
func NewCache(backend SecretBackend, timeouts Timeouts, logConfig log.Config, now func() time.Time) *Cache {
	logger := log.New("kwfs_cache", logConfig)
	return &Cache{logger, NewSecretMap(timeouts, now), backend, timeouts, now}
}

// Warmup reads the secret list from the backend to prime the cache.
// Should only be called after creating a new cache on startup.
func (c *Cache) Warmup() {
	// Attempt to warmup cache
	secrets, ok := c.backend.SecretList()
	if ok {
		for _, backendSecret := range secrets {
			c.secretMap.Put(backendSecret.Name, backendSecret)
		}
	} else {
		c.Warnf("Failed to warmup cache on startup")
	}
}

// Clear empties the internal cache. This function does not honor the
// delayed deletion contract. The function is called when the user deletes
// .clear_cache.
func (c *Cache) Clear() {
	c.Infof("Cache cleared")
	c.secretMap = NewSecretMap(c.timeouts, c.now)
}

// Secret retrieves a Secret by name from cache or a server.
//
// Cache logic:
//  1. Check cache for secret.
//			* If entry is fresh, return cache entry.
//			* If entry is not fresh, call backend.
//  2. Ask backend for secret (with timeout).
//			* If backend returns success: update cache, return.
//			* If backend returns deleted: set delayed deletion, return data from cache.
//  3. If timeout backend deadline hit return whatever we have.
func (c *Cache) Secret(name string) (*Secret, bool) {
	// Perform cache lookup first
	cacheResult := c.cacheSecret(name)

	var secret *Secret
	var success bool

	if cacheResult != nil {
		secret = &cacheResult.Secret
		success = !cacheResult.deleted

		// immediately return fresh cache result
		if time.Since(cacheResult.Time) < c.timeouts.Fresh {
			return secret, success
		}
	}

	backendDeadline := time.After(c.timeouts.BackendDeadline)
	backendDone := c.backendSecret(name)

	select {
	case s := <-backendDone:
		if s.err == nil {
			secret = s.secret
			success = true
		} else if _, ok := s.err.(SecretDeleted); ok {
			c.secretMap.Delete(name)
		}
	case <-backendDeadline:
		c.Errorf("Backend timeout on secret fetch for '%s'", name)
	}

	return secret, success
}

// SecretList returns a listing of Secrets from cache or a server.
//
// Cache logic:
//  * If backend returns fast: update cache, return.
//  * If timeout backend deadline: return cache entries, background update cache.
//  * If timeout max wait: return cache version.
// TODO: why do we prefer the backend to the cache? is cache intended only to be local backup?
// TODO: could we track list freshness and prefer a fresh cache list to avoid pulling from backend?
func (c *Cache) SecretList() []Secret {
	backendDeadline := time.After(c.timeouts.BackendDeadline)
	backendDone := c.backendSecretList()

	for {
		select {
		case backendResult := <-backendDone:
			return backendResult
		case <-backendDeadline:
			c.Errorf("Backend timeout for secret list")
			return c.cacheSecretList()
		}
	}
}

// Add inserts a secret into the cache. If a secret is already in the cache with a matching
// identifier, it will be overridden  This method is most useful for testing since lookups
// may add data to the cache.
// only used by tests
func (c *Cache) Add(s Secret) {
	c.secretMap.Put(s.Name, s)
}

// Len returns the number of values stored in the cache. This method is most useful for testing.
// only used by tests
func (c *Cache) Len() int {
	return c.secretMap.Len()
}

// cacheSecret retrieves a secret from the cache.
func (c *Cache) cacheSecret(name string) *SecretTime {
	secret, ok := c.secretMap.Get(name)
	if ok && (len(secret.Secret.Content) > 0 || secret.deleted) {
		c.Debugf("Cache hit: %v", name)
		return &secret
	}
	c.Debugf("Cache miss: %v", name)
	return nil
}

// cacheSecretList retrieves a secret listing from the cache.
func (c *Cache) cacheSecretList() []Secret {
	values := c.secretMap.Values()
	secrets := make([]Secret, len(values))
	// TODO: is it necessary to gather the list in secretmap.Values() and then copy it one by one to another
	// TODO: list with no change/processing/etc? seems inefficient, can it be improved?
	for i, v := range values {
		secrets[i] = v.Secret
	}
	return secrets
}

// backendSecret retrieves a secret from the backend and updates the cache.
//
// Retrieval is concurrent, so a channel is returned to communicate a successful value.
// The channel will not be fulfilled on error.
func (c *Cache) backendSecret(name string) chan secretResult {
	secretc := make(chan secretResult)
	go func() {
		defer close(secretc)
		secret, err := c.backend.Secret(name)
		secretc <- secretResult{secret, err}
		if err == nil {
			c.secretMap.Put(name, *secret)
		}
	}()
	return secretc
}

// backendSecretList retrieves a secret listing from the backend and updates the cache.
//
// Retrieval is concurrent, so a channel is returned to communicate successful values. The channel
// will not be fulfilled on error.
func (c *Cache) backendSecretList() chan []Secret {
	secretsc := make(chan []Secret, 1)
	go func() {
		secrets, ok := c.backend.SecretList()
		if !ok {
			// Don't close the channel so that we use the result from the cache.
			return
		}

		newMap := NewSecretMap(c.timeouts, c.now)
		for _, backendSecret := range secrets {
			if len(backendSecret.Content) == 0 {
				// The backend didn't return any content. The cache might contain a secret with content, in
				// which case we want to keep the cache's value (and not schedule it for delayed deletion).
				if s, ok := c.secretMap.Get(backendSecret.Name); ok && len(s.Secret.Content) > 0 {
					newMap.Put(backendSecret.Name, s.Secret)
				} else {
					// We don't have content for this secret. This happens when the cache has never seen a given secret
					// (at startup or when a new secret is added).
					// can happen.
					newMap.Put(backendSecret.Name, backendSecret)
				}
			} else {
				// TODO: explain why this case can happen. It doesn't seem like it can,
				// listing secrets always returns just the names.
				// Cache the latest info.
				newMap.Put(backendSecret.Name, backendSecret)
			}
		}
		c.secretMap.Replace(newMap)

		// TODO: copy-pasta from cacheSecretList(), should refactor.
		values := c.secretMap.Values()
		secrets = make([]Secret, len(values))
		for i, v := range values {
			secrets[i] = v.Secret
		}
		secretsc <- secrets
		close(secretsc)
	}()
	return secretsc
}
<commit_msg>Removed code from backendSecretList which checked if secret content in the listing was 0 length or not. The backend cannot return secrets with content in the secret list, so this is unnecessary.<commit_after>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"time"

	"github.com/square/keywhiz-fs/log"
)

// SecretBackend represents an interface for storing secrets.
type SecretBackend interface {
	Secret(string) (secret *Secret, err error)
	SecretList() (secretList []Secret, ok bool)
}

// Timeouts contains configuration for timeouts:
// timeout_backend_deadline: optimistic timeout to wait for cache
// timeout_max_wait: timeout for client to get data from server
type Timeouts struct {
	// FUSE may make many lookups in quick succession. If cached data is recent within the threshold,
	// a backend request is not attempted.
	Fresh time.Duration
	// BackendDeadline is distinct from the backend timeout. It is an optimistic timeout to wait
	// until resorting to cached data.
	BackendDeadline time.Duration
	MaxWait         time.Duration
	// Controls how long to keep a deleted entry before purging it.
	DeletionDelay time.Duration
}

// Cache contains necessary state to return secrets, using previously cached content or retrieving
// from a server if necessary.
type Cache struct {
	*log.Logger
	secretMap *SecretMap
	backend   SecretBackend
	timeouts  Timeouts
	now       func() time.Time
}

type secretResult struct {
	secret *Secret
	err    error
}

// NewCache initializes a Cache.
func NewCache(backend SecretBackend, timeouts Timeouts, logConfig log.Config, now func() time.Time) *Cache {
	logger := log.New("kwfs_cache", logConfig)
	return &Cache{logger, NewSecretMap(timeouts, now), backend, timeouts, now}
}

// Warmup reads the secret list from the backend to prime the cache.
// Should only be called after creating a new cache on startup.
func (c *Cache) Warmup() {
	// Attempt to warmup cache
	secrets, ok := c.backend.SecretList()
	if ok {
		for _, backendSecret := range secrets {
			c.secretMap.Put(backendSecret.Name, backendSecret)
		}
	} else {
		c.Warnf("Failed to warmup cache on startup")
	}
}

// Clear empties the internal cache. This function does not honor the
// delayed deletion contract. The function is called when the user deletes
// .clear_cache.
func (c *Cache) Clear() {
	c.Infof("Cache cleared")
	c.secretMap = NewSecretMap(c.timeouts, c.now)
}

// Secret retrieves a Secret by name from cache or a server.
//
// Cache logic:
//  1. Check cache for secret.
//			* If entry is fresh, return cache entry.
//			* If entry is not fresh, call backend.
//  2. Ask backend for secret (with timeout).
//			* If backend returns success: update cache, return.
//			* If backend returns deleted: set delayed deletion, return data from cache.
//  3. If timeout backend deadline hit return whatever we have.
func (c *Cache) Secret(name string) (*Secret, bool) {
	// Perform cache lookup first
	cacheResult := c.cacheSecret(name)

	var secret *Secret
	var success bool

	if cacheResult != nil {
		secret = &cacheResult.Secret
		success = !cacheResult.deleted

		// immediately return fresh cache result
		if time.Since(cacheResult.Time) < c.timeouts.Fresh {
			return secret, success
		}
	}

	backendDeadline := time.After(c.timeouts.BackendDeadline)
	backendDone := c.backendSecret(name)

	select {
	case s := <-backendDone:
		if s.err == nil {
			secret = s.secret
			success = true
		} else if _, ok := s.err.(SecretDeleted); ok {
			c.secretMap.Delete(name)
		}
	case <-backendDeadline:
		c.Errorf("Backend timeout on secret fetch for '%s'", name)
	}

	return secret, success
}

// SecretList returns a listing of Secrets from cache or a server.
//
// Cache logic:
//  * If backend returns fast: update cache, return.
//  * If timeout backend deadline: return cache entries, background update cache.
//  * If timeout max wait: return cache version.
// TODO: why do we prefer the backend to the cache? is cache intended only to be local backup?
// TODO: could we track list freshness and prefer a fresh cache list to avoid pulling from backend?
func (c *Cache) SecretList() []Secret {
	backendDeadline := time.After(c.timeouts.BackendDeadline)
	backendDone := c.backendSecretList()

	for {
		select {
		case backendResult := <-backendDone:
			return backendResult
		case <-backendDeadline:
			c.Errorf("Backend timeout for secret list")
			return c.cacheSecretList()
		}
	}
}

// Add inserts a secret into the cache. If a secret is already in the cache with a matching
// identifier, it will be overridden  This method is most useful for testing since lookups
// may add data to the cache.
// only used by tests
func (c *Cache) Add(s Secret) {
	c.secretMap.Put(s.Name, s)
}

// Len returns the number of values stored in the cache. This method is most useful for testing.
// only used by tests
func (c *Cache) Len() int {
	return c.secretMap.Len()
}

// cacheSecret retrieves a secret from the cache.
func (c *Cache) cacheSecret(name string) *SecretTime {
	secret, ok := c.secretMap.Get(name)
	if ok && (len(secret.Secret.Content) > 0 || secret.deleted) {
		c.Debugf("Cache hit: %v", name)
		return &secret
	}
	c.Debugf("Cache miss: %v", name)
	return nil
}

// cacheSecretList retrieves a secret listing from the cache.
func (c *Cache) cacheSecretList() []Secret {
	values := c.secretMap.Values()
	secrets := make([]Secret, len(values))
	// TODO: is it necessary to gather the list in secretmap.Values() and then copy it one by one to another
	// TODO: list with no change/processing/etc? seems inefficient, can it be improved?
	for i, v := range values {
		secrets[i] = v.Secret
	}
	return secrets
}

// backendSecret retrieves a secret from the backend and updates the cache.
//
// Retrieval is concurrent, so a channel is returned to communicate a successful value.
// The channel will not be fulfilled on error.
func (c *Cache) backendSecret(name string) chan secretResult {
	secretc := make(chan secretResult)
	go func() {
		defer close(secretc)
		secret, err := c.backend.Secret(name)
		secretc <- secretResult{secret, err}
		if err == nil {
			c.secretMap.Put(name, *secret)
		}
	}()
	return secretc
}

// backendSecretList retrieves a secret listing from the backend and updates the cache.
//
// Retrieval is concurrent, so a channel is returned to communicate successful values. The channel
// will not be fulfilled on error.
func (c *Cache) backendSecretList() chan []Secret {
	secretsc := make(chan []Secret, 1)
	go func() {
		secrets, ok := c.backend.SecretList()
		if !ok {
			// Don't close the channel so that we use the result from the cache.
			return
		}

		newMap := NewSecretMap(c.timeouts, c.now)
		for _, backendSecret := range secrets {
			// The cache might contain a secret with content, in which case we want to keep the cache's
			// value (and not schedule it for delayed deletion).
			if s, ok := c.secretMap.Get(backendSecret.Name); ok && len(s.Secret.Content) > 0 {
				newMap.Put(backendSecret.Name, s.Secret)
			} else {
				// We don't have content for this secret. This happens when the cache has never seen a given secret
				// (at startup or when a new secret is added).
				// can happen.
				newMap.Put(backendSecret.Name, backendSecret)
			}
		}
		c.secretMap.Replace(newMap)

		// TODO: copy-pasta from cacheSecretList(), should refactor.
		values := c.secretMap.Values()
		secrets = make([]Secret, len(values))
		for i, v := range values {
			secrets[i] = v.Secret
		}
		secretsc <- secrets
		close(secretsc)
	}()
	return secretsc
}
<|endoftext|>


<commit_before>package carbon

import (
	"fmt"
	"time"

	"github.com/Sirupsen/logrus"
)

// CacheQuery request from carbonlink
type CacheQuery struct {
	Metric    string
	ReplyChan chan *CacheReply
}

// NewCacheQuery create CacheQuery instance
func NewCacheQuery(metric string) *CacheQuery {
	return &CacheQuery{
		Metric:    metric,
		ReplyChan: make(chan *CacheReply, 1),
	}
}

// CacheReply response to carbonlink
type CacheReply struct {
	Data []struct {
		Value     float64
		Timestamp int64
	}
}

// NewCacheReply create CacheReply instance
func NewCacheReply() *CacheReply {
	return &CacheReply{}
}

// CacheValues one metric data
type CacheValues struct {
	Metric string
	Data   []struct {
		Value     float64
		Timestamp int64
	}
}

// Append new metric value
func (values *CacheValues) Append(value float64, timestamp int64) {
	values.Data = append(values.Data, struct {
		Value     float64
		Timestamp int64
	}{value, timestamp})
}

// Cache stores and aggregate metrics in memory
type Cache struct {
	data        map[string]*CacheValues
	size        int
	inputChan   chan *Message     // from receivers
	outputChan  chan *CacheValues // to persisters
	queryChan   chan *CacheQuery  // from carbonlink
	exitChan    chan bool         // close for stop worker
	graphPrefix string
	queryCnt    int
}

// NewCache create Cache instance and run in/out goroutine
func NewCache() *Cache {
	cache := &Cache{
		data:        make(map[string]*CacheValues, 0),
		size:        0,
		inputChan:   make(chan *Message, 1024),
		exitChan:    make(chan bool),
		queryChan:   make(chan *CacheQuery, 16),
		graphPrefix: "carbon",
		queryCnt:    0,
	}
	return cache
}

// Get any key/values pair from Cache
func (c Cache) Get() (string, *CacheValues) {
	for key, values := range c.data {
		return key, values
	}
	return "", nil
}

// Remove key from cache
func (c *Cache) Remove(key string) {
	if value, exists := c.data[key]; exists {
		c.size -= len(value.Data)
		delete(c.data, key)
	}
}

// Add value to cache
func (c *Cache) Add(key string, value float64, timestamp int64) {
	if values, exists := c.data[key]; exists {
		values.Append(value, timestamp)
	} else {
		values := &CacheValues{}
		values.Append(value, timestamp)
		c.data[key] = values
	}
	c.size++
}

// SetGraphPrefix for internal cache metrics
func (c *Cache) SetGraphPrefix(prefix string) {
	c.graphPrefix = prefix
}

// Size returns size
func (c *Cache) Size() int {
	return c.size
}

// doCheckoint reorder save queue, add carbon metrics to queue
func (c *Cache) doCheckpoint() {
	start := time.Now()

	c.Add(fmt.Sprintf("%s%s", c.graphPrefix, "cache.size"), float64(c.size), start.Unix())
	c.Add(fmt.Sprintf("%s%s", c.graphPrefix, "cache.metrics"), float64(len(c.data)), start.Unix())
	c.Add(fmt.Sprintf("%s%s", c.graphPrefix, "cache.queries"), float64(c.queryCnt), start.Unix())

	worktime := time.Now().Sub(start)
	logrus.WithFields(logrus.Fields{
		"time":    worktime.String(),
		"size":    c.size,
		"metrics": len(c.data),
		"queries": c.queryCnt,
	}).Info("doCheckpoint()")
	c.queryCnt = 0
}

func (c *Cache) worker() {
	var key string
	var values *CacheValues
	var sendTo chan *CacheValues

	ticker := time.NewTicker(time.Minute)
	defer ticker.Stop()

	for {
		if values == nil {
			key, values = c.Get()

			if values != nil {
				c.Remove(key)
				values.Metric = key
				sendTo = c.outputChan
			} else {
				sendTo = nil
			}
		}

		select {
		case <-ticker.C:
			c.doCheckpoint()
		case query := <-c.queryChan:
			c.queryCnt++
			reply := NewCacheReply()

			if values != nil && values.Metric == query.Metric {
				reply.Data = values.Data
			} else if v, ok := c.data[query.Metric]; ok {
				reply.Data = v.Data
			}

			query.ReplyChan <- reply
		case sendTo <- values:
			values = nil
		case msg := <-c.inputChan:
			c.Add(msg.Name, msg.Value, msg.Timestamp)
		case <-c.exitChan:
			break
		}
	}

}

// In returns input channel
func (c *Cache) In() chan *Message {
	return c.inputChan
}

// Out returns output channel
func (c *Cache) Out() chan *CacheValues {
	return c.outputChan
}

// Query returns carbonlink query channel
func (c *Cache) Query() chan *CacheQuery {
	return c.queryChan
}

// SetOutputChanSize ...
func (c *Cache) SetOutputChanSize(size int) {
	c.outputChan = make(chan *CacheValues, size)
}

// Start worker
func (c *Cache) Start() {
	if c.outputChan == nil {
		c.outputChan = make(chan *CacheValues, 1024)
	}
	go c.worker()
}

// Stop worker
func (c *Cache) Stop() {
	close(c.exitChan)
}
<commit_msg>create and sort queue. don't used yet<commit_after>package carbon

import (
	"fmt"
	"sort"
	"time"

	"github.com/Sirupsen/logrus"
)

// CacheQuery request from carbonlink
type CacheQuery struct {
	Metric    string
	ReplyChan chan *CacheReply
}

// NewCacheQuery create CacheQuery instance
func NewCacheQuery(metric string) *CacheQuery {
	return &CacheQuery{
		Metric:    metric,
		ReplyChan: make(chan *CacheReply, 1),
	}
}

// CacheReply response to carbonlink
type CacheReply struct {
	Data []struct {
		Value     float64
		Timestamp int64
	}
}

// NewCacheReply create CacheReply instance
func NewCacheReply() *CacheReply {
	return &CacheReply{}
}

// CacheValues one metric data
type CacheValues struct {
	Metric string
	Data   []struct {
		Value     float64
		Timestamp int64
	}
}

// Append new metric value
func (values *CacheValues) Append(value float64, timestamp int64) {
	values.Data = append(values.Data, struct {
		Value     float64
		Timestamp int64
	}{value, timestamp})
}

// Cache stores and aggregate metrics in memory
type Cache struct {
	data        map[string]*CacheValues
	size        int
	inputChan   chan *Message     // from receivers
	outputChan  chan *CacheValues // to persisters
	queryChan   chan *CacheQuery  // from carbonlink
	exitChan    chan bool         // close for stop worker
	graphPrefix string
	queryCnt    int
}

// NewCache create Cache instance and run in/out goroutine
func NewCache() *Cache {
	cache := &Cache{
		data:        make(map[string]*CacheValues, 0),
		size:        0,
		inputChan:   make(chan *Message, 1024),
		exitChan:    make(chan bool),
		queryChan:   make(chan *CacheQuery, 16),
		graphPrefix: "carbon",
		queryCnt:    0,
	}
	return cache
}

// Get any key/values pair from Cache
func (c Cache) Get() (string, *CacheValues) {
	for key, values := range c.data {
		return key, values
	}
	return "", nil
}

// Remove key from cache
func (c *Cache) Remove(key string) {
	if value, exists := c.data[key]; exists {
		c.size -= len(value.Data)
		delete(c.data, key)
	}
}

// Add value to cache
func (c *Cache) Add(key string, value float64, timestamp int64) {
	if values, exists := c.data[key]; exists {
		values.Append(value, timestamp)
	} else {
		values := &CacheValues{}
		values.Append(value, timestamp)
		c.data[key] = values
	}
	c.size++
}

// SetGraphPrefix for internal cache metrics
func (c *Cache) SetGraphPrefix(prefix string) {
	c.graphPrefix = prefix
}

// Size returns size
func (c *Cache) Size() int {
	return c.size
}

type cacheQueueItem struct {
	metric string
	count  int
}

type cacheQueue []*cacheQueueItem

func (v cacheQueue) Len() int           { return len(v) }
func (v cacheQueue) Swap(i, j int)      { v[i], v[j] = v[j], v[i] }
func (v cacheQueue) Less(i, j int) bool { return v[i].count < v[j].count }

// doCheckoint reorder save queue, add carbon metrics to queue
func (c *Cache) doCheckpoint() {
	start := time.Now()

	newQueue := make(cacheQueue, 0)

	for key, values := range c.data {
		newQueue = append(newQueue, &cacheQueueItem{key, len(values.Data)})
	}

	sort.Sort(newQueue)

	c.Add(fmt.Sprintf("%s%s", c.graphPrefix, "cache.size"), float64(c.size), start.Unix())
	c.Add(fmt.Sprintf("%s%s", c.graphPrefix, "cache.metrics"), float64(len(c.data)), start.Unix())
	c.Add(fmt.Sprintf("%s%s", c.graphPrefix, "cache.queries"), float64(c.queryCnt), start.Unix())

	worktime := time.Now().Sub(start)
	logrus.WithFields(logrus.Fields{
		"time":    worktime.String(),
		"size":    c.size,
		"metrics": len(c.data),
		"queries": c.queryCnt,
	}).Info("doCheckpoint()")
	c.queryCnt = 0
}

func (c *Cache) worker() {
	var key string
	var values *CacheValues
	var sendTo chan *CacheValues

	ticker := time.NewTicker(time.Minute)
	defer ticker.Stop()

	for {
		if values == nil {
			key, values = c.Get()

			if values != nil {
				c.Remove(key)
				values.Metric = key
				sendTo = c.outputChan
			} else {
				sendTo = nil
			}
		}

		select {
		case <-ticker.C:
			c.doCheckpoint()
		case query := <-c.queryChan:
			c.queryCnt++
			reply := NewCacheReply()

			if values != nil && values.Metric == query.Metric {
				reply.Data = values.Data
			} else if v, ok := c.data[query.Metric]; ok {
				reply.Data = v.Data
			}

			query.ReplyChan <- reply
		case sendTo <- values:
			values = nil
		case msg := <-c.inputChan:
			c.Add(msg.Name, msg.Value, msg.Timestamp)
		case <-c.exitChan:
			break
		}
	}

}

// In returns input channel
func (c *Cache) In() chan *Message {
	return c.inputChan
}

// Out returns output channel
func (c *Cache) Out() chan *CacheValues {
	return c.outputChan
}

// Query returns carbonlink query channel
func (c *Cache) Query() chan *CacheQuery {
	return c.queryChan
}

// SetOutputChanSize ...
func (c *Cache) SetOutputChanSize(size int) {
	c.outputChan = make(chan *CacheValues, size)
}

// Start worker
func (c *Cache) Start() {
	if c.outputChan == nil {
		c.outputChan = make(chan *CacheValues, 1024)
	}
	go c.worker()
}

// Stop worker
func (c *Cache) Stop() {
	close(c.exitChan)
}
<|endoftext|>


<commit_before>// -*- tab-width: 4; -*-

package main

import (
	"bufio"
	"bytes"
	"encoding/gob"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"
)

type Cached struct {
	Tweets       Tweets
	Lastmodified string
}

// key: url
type Cache map[string]Cached

func (cache Cache) Store(configpath string) {
	b := new(bytes.Buffer)
	enc := gob.NewEncoder(b)
	err := enc.Encode(cache)
	if err != nil {
		panic(err)
	}

	f, err := os.OpenFile(fmt.Sprintf("%s/cache", configpath),
		os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		panic(err)
	}

	defer f.Close()

	if _, err = f.Write(b.Bytes()); err != nil {
		panic(err)
	}
}

func LoadCache(configpath string) Cache {
	cache := make(Cache)

	f, err := os.Open(fmt.Sprintf("%s/cache", configpath))
	if err != nil {
		if !os.IsNotExist(err) {
			panic(err)
		}
		return cache
	}
	defer f.Close()

	dec := gob.NewDecoder(f)
	err = dec.Decode(&cache)
	if err != nil {
		panic(err)
	}
	return cache
}

const maxfetchers = 50

func (cache Cache) FetchTweets(sources map[string]string) {
	var mu sync.RWMutex

	// buffered to let goroutines write without blocking before the main thread
	// begins reading
	tweetsch := make(chan Tweets, len(sources))

	var wg sync.WaitGroup
	// max parallel http fetchers
	var fetchers = make(chan struct{}, maxfetchers)

	for nick, url := range sources {
		wg.Add(1)
		fetchers <- struct{}{}
		// anon func takes needed variables as arg, avoiding capture of iterator variables
		go func(nick string, url string) {
			defer func() {
				<-fetchers
				wg.Done()
			}()

			if strings.HasPrefix(url, "file://") {
				err := ReadLocalFile(url, nick, tweetsch, cache, &mu)
				if err != nil {
					if debug {
						log.Printf("%s: Failed to read and cache local file: %s", url, err)
					}
				}
				return
			}

			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				if debug {
					log.Printf("%s: http.NewRequest fail: %s", url, err)
				}
				tweetsch <- nil
				return
			}

			if conf.Nick != "" && conf.Twturl != "" {
				req.Header.Set("User-Agent",
					fmt.Sprintf("%s/%s (+%s; @%s)", progname, GetVersion(),
						conf.Twturl, conf.Nick))
			}

			mu.RLock()
			if cached, ok := cache[url]; ok {
				if cached.Lastmodified != "" {
					req.Header.Set("If-Modified-Since", cached.Lastmodified)
				}
			}
			mu.RUnlock()

			client := http.Client{
				Timeout: time.Second * 15,
			}
			resp, err := client.Do(req)
			if err != nil {
				if debug {
					log.Printf("%s: client.Do fail: %s", url, err)
				}
				tweetsch <- nil
				return
			}
			defer resp.Body.Close()

			actualurl := resp.Request.URL.String()
			if actualurl != url {
				url = actualurl
			}

			var tweets Tweets

			switch resp.StatusCode {
			case http.StatusOK: // 200
				scanner := bufio.NewScanner(resp.Body)
				tweets = ParseFile(scanner, Tweeter{Nick: nick, URL: url})
				lastmodified := resp.Header.Get("Last-Modified")
				mu.Lock()
				cache[url] = Cached{Tweets: tweets, Lastmodified: lastmodified}
				mu.Unlock()
			case http.StatusNotModified: // 304
				mu.RLock()
				tweets = cache[url].Tweets
				mu.RUnlock()
			}

			tweetsch <- tweets

		}(nick, url)

	}

	// close tweets channel when all goroutines are done
	go func() {
		wg.Wait()
		close(tweetsch)
	}()

	if debug {
		log.Print("fetching:\n")
	}

	var n = 0
	// loop until channel closed
	for tweets := range tweetsch {
		n++
		if debug {
			log.Printf("%d ", len(sources)+1-n)
		}
		if debug && len(tweets) > 0 {
			log.Printf("%s\n", tweets[0].Tweeter.URL)
		}
	}
	if debug {
		log.Print("\n")
	}
}

func ReadLocalFile(url string, nick string, tweetsch chan<- Tweets, cache Cache, mu *sync.RWMutex) error {
	path := url[6:]
	file, err := os.Stat(path)
	if err != nil {
		if debug {
			log.Printf("%s: Can't stat local file: %s", path, err)
		}
		return err
	}
	if cached, ok := (cache)[url]; ok {
		if cached.Lastmodified == file.ModTime().String() {
			tweets := (cache)[url].Tweets
			tweetsch <- tweets
			return nil
		}
	}
	data, err := ioutil.ReadFile(path)
	if err != nil {
		if debug {
			log.Printf("%s: Can't read local file: %s", path, err)
		}
		tweetsch <- nil
		return err
	}
	scanner := bufio.NewScanner(bytes.NewReader(data))
	tweets := ParseFile(scanner, Tweeter{Nick: nick, URL: url})
	lastmodified := file.ModTime().String()
	mu.Lock()
	cache[url] = Cached{Tweets: tweets, Lastmodified: lastmodified}
	mu.Unlock()
	tweetsch <- tweets
	return nil
}

func (cache Cache) GetAll() Tweets {
	var alltweets Tweets
	for url, cached := range cache {
		alltweets = append(alltweets, cached.Tweets...)
		if debug {
			log.Printf("%s\n", url)
		}
	}
	return alltweets
}

func (cache Cache) GetByURL(url string) Tweets {
	if cached, ok := cache[url]; ok {
		return cached.Tweets
	}
	return Tweets{}
}
<commit_msg>Shorten same type<commit_after>// -*- tab-width: 4; -*-

package main

import (
	"bufio"
	"bytes"
	"encoding/gob"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"
)

type Cached struct {
	Tweets       Tweets
	Lastmodified string
}

// key: url
type Cache map[string]Cached

func (cache Cache) Store(configpath string) {
	b := new(bytes.Buffer)
	enc := gob.NewEncoder(b)
	err := enc.Encode(cache)
	if err != nil {
		panic(err)
	}

	f, err := os.OpenFile(fmt.Sprintf("%s/cache", configpath),
		os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		panic(err)
	}

	defer f.Close()

	if _, err = f.Write(b.Bytes()); err != nil {
		panic(err)
	}
}

func LoadCache(configpath string) Cache {
	cache := make(Cache)

	f, err := os.Open(fmt.Sprintf("%s/cache", configpath))
	if err != nil {
		if !os.IsNotExist(err) {
			panic(err)
		}
		return cache
	}
	defer f.Close()

	dec := gob.NewDecoder(f)
	err = dec.Decode(&cache)
	if err != nil {
		panic(err)
	}
	return cache
}

const maxfetchers = 50

func (cache Cache) FetchTweets(sources map[string]string) {
	var mu sync.RWMutex

	// buffered to let goroutines write without blocking before the main thread
	// begins reading
	tweetsch := make(chan Tweets, len(sources))

	var wg sync.WaitGroup
	// max parallel http fetchers
	var fetchers = make(chan struct{}, maxfetchers)

	for nick, url := range sources {
		wg.Add(1)
		fetchers <- struct{}{}
		// anon func takes needed variables as arg, avoiding capture of iterator variables
		go func(nick string, url string) {
			defer func() {
				<-fetchers
				wg.Done()
			}()

			if strings.HasPrefix(url, "file://") {
				err := ReadLocalFile(url, nick, tweetsch, cache, &mu)
				if err != nil {
					if debug {
						log.Printf("%s: Failed to read and cache local file: %s", url, err)
					}
				}
				return
			}

			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				if debug {
					log.Printf("%s: http.NewRequest fail: %s", url, err)
				}
				tweetsch <- nil
				return
			}

			if conf.Nick != "" && conf.Twturl != "" {
				req.Header.Set("User-Agent",
					fmt.Sprintf("%s/%s (+%s; @%s)", progname, GetVersion(),
						conf.Twturl, conf.Nick))
			}

			mu.RLock()
			if cached, ok := cache[url]; ok {
				if cached.Lastmodified != "" {
					req.Header.Set("If-Modified-Since", cached.Lastmodified)
				}
			}
			mu.RUnlock()

			client := http.Client{
				Timeout: time.Second * 15,
			}
			resp, err := client.Do(req)
			if err != nil {
				if debug {
					log.Printf("%s: client.Do fail: %s", url, err)
				}
				tweetsch <- nil
				return
			}
			defer resp.Body.Close()

			actualurl := resp.Request.URL.String()
			if actualurl != url {
				url = actualurl
			}

			var tweets Tweets

			switch resp.StatusCode {
			case http.StatusOK: // 200
				scanner := bufio.NewScanner(resp.Body)
				tweets = ParseFile(scanner, Tweeter{Nick: nick, URL: url})
				lastmodified := resp.Header.Get("Last-Modified")
				mu.Lock()
				cache[url] = Cached{Tweets: tweets, Lastmodified: lastmodified}
				mu.Unlock()
			case http.StatusNotModified: // 304
				mu.RLock()
				tweets = cache[url].Tweets
				mu.RUnlock()
			}

			tweetsch <- tweets

		}(nick, url)

	}

	// close tweets channel when all goroutines are done
	go func() {
		wg.Wait()
		close(tweetsch)
	}()

	if debug {
		log.Print("fetching:\n")
	}

	var n = 0
	// loop until channel closed
	for tweets := range tweetsch {
		n++
		if debug {
			log.Printf("%d ", len(sources)+1-n)
		}
		if debug && len(tweets) > 0 {
			log.Printf("%s\n", tweets[0].Tweeter.URL)
		}
	}
	if debug {
		log.Print("\n")
	}
}

func ReadLocalFile(url, nick string, tweetsch chan<- Tweets, cache Cache, mu *sync.RWMutex) error {
	path := url[6:]
	file, err := os.Stat(path)
	if err != nil {
		if debug {
			log.Printf("%s: Can't stat local file: %s", path, err)
		}
		return err
	}
	if cached, ok := (cache)[url]; ok {
		if cached.Lastmodified == file.ModTime().String() {
			tweets := (cache)[url].Tweets
			tweetsch <- tweets
			return nil
		}
	}
	data, err := ioutil.ReadFile(path)
	if err != nil {
		if debug {
			log.Printf("%s: Can't read local file: %s", path, err)
		}
		tweetsch <- nil
		return err
	}
	scanner := bufio.NewScanner(bytes.NewReader(data))
	tweets := ParseFile(scanner, Tweeter{Nick: nick, URL: url})
	lastmodified := file.ModTime().String()
	mu.Lock()
	cache[url] = Cached{Tweets: tweets, Lastmodified: lastmodified}
	mu.Unlock()
	tweetsch <- tweets
	return nil
}

func (cache Cache) GetAll() Tweets {
	var alltweets Tweets
	for url, cached := range cache {
		alltweets = append(alltweets, cached.Tweets...)
		if debug {
			log.Printf("%s\n", url)
		}
	}
	return alltweets
}

func (cache Cache) GetByURL(url string) Tweets {
	if cached, ok := cache[url]; ok {
		return cached.Tweets
	}
	return Tweets{}
}
<|endoftext|>


<commit_before>// errchk $G -e $D/$F.go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import "io"

func whatis(x interface{}) string {
	switch x.(type) {
	case int:
		return "int"
	case int: // ERROR "duplicate"
		return "int8"
	case io.Reader:
		return "Reader1"
	case io.Reader: // ERROR "duplicate"
		return "Reader2"
	case interface { r(); w() }:
		return "rw"
	case interface { w(); r() }:	// ERROR "duplicate"
		return "wr"

	}
	return ""
}
<commit_msg>type switches: test for pathological case<commit_after>// errchk $G -e $D/$F.go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import "io"

func whatis(x interface{}) string {
	switch x.(type) {
	case int:
		return "int"
	case int: // ERROR "duplicate"
		return "int8"
	case io.Reader:
		return "Reader1"
	case io.Reader: // ERROR "duplicate"
		return "Reader2"
	case interface {
		r()
		w()
	}:
		return "rw"
	case interface {
		w()
		r()
	}: // ERROR "duplicate"
		return "wr"

	}
	return ""
}

func notused(x interface{}) {
	// The first t is in a different scope than the 2nd t; it cannot
	// be accessed (=> declared and not used error); but it is legal
	// to declare it.
	switch t := 0; t := x.(type) { // ERROR "declared and not used"
	case int:
		_ = t // this is using the t of "t := x.(type)"
	}
}
<|endoftext|>


<commit_before><commit_msg>port StateMachine from Python code<commit_after><|endoftext|>


<commit_before><commit_msg>add serveral fields to subscription<commit_after><|endoftext|>


<commit_before>package goinsta

import (
	"encoding/json"
	"fmt"
)

// Feed is the object for all feed endpoints.
type Feed struct {
	inst *Instagram
}

// newFeed creates new Feed structure
func newFeed(inst *Instagram) *Feed {
	return &Feed{
		inst: inst,
	}
}

// Feed search by locationID
func (feed *Feed) LocationID(locationID int64) (*FeedLocation, error) {
	insta := feed.inst
	body, err := insta.sendRequest(
		&reqOptions{
			Endpoint: fmt.Sprintf(urlFeedLocationID, locationID),
			Query: map[string]string{
				"rank_token":     insta.rankToken,
				"ranked_content": "true",
			},
		},
	)
	if err != nil {
		return nil, err
	}

	res := &FeedLocation{}
	err = json.Unmarshal(body, res)
	return res, err
}

// FeedLocation is the struct that fits the structure returned by instagram on LocationID search.
type FeedLocation struct {
	RankedItems         []Item   `json:"ranked_items"`
	Items               []Item   `json:"items"`
	NumResults          int      `json:"num_results"`
	NextID              string   `json:"next_max_id"`
	MoreAvailable       bool     `json:"more_available"`
	AutoLoadMoreEnabled bool     `json:"auto_load_more_enabled"`
	MediaCount          int      `json:"media_count"`
	Location            Location `json:"location"`
	Status              string   `json:"status"`
}

// Tags search by Tag in user Feed
//
// (sorry for returning FeedTag. See #FeedTag)
func (feed *Feed) Tags(tag string) (*FeedTag, error) {
	insta := feed.inst
	body, err := insta.sendRequest(
		&reqOptions{
			Endpoint: fmt.Sprintf(urlFeedTag, tag),
			Query: map[string]string{
				"rank_token":     insta.rankToken,
				"ranked_content": "true",
			},
		},
	)
	if err != nil {
		return nil, err
	}
	res := &FeedTag{}
	err = json.Unmarshal(body, res)
	if err != nil {
		return nil, err
	}
	for i := range res.RankedItems {
		res.RankedItems[i].media = &FeedMedia{
			inst: insta,
		}
	}
	return res, nil
}

// FeedTag is the struct that fits the structure returned by instagram on TagSearch.
type FeedTag struct {
	RankedItems         []Item     `json:"ranked_items"`
	Images              []Item     `json:"items"`
	NumResults          int        `json:"num_results"`
	NextID              string     `json:"next_max_id"`
	MoreAvailable       bool       `json:"more_available"`
	AutoLoadMoreEnabled bool       `json:"auto_load_more_enabled"`
	Story               StoryMedia `json:"story"`
	Status              string     `json:"status"`
}
<commit_msg>Fix FeedMedia in FeedTag [ci skip]<commit_after>package goinsta

import (
	"encoding/json"
	"fmt"
)

// Feed is the object for all feed endpoints.
type Feed struct {
	inst *Instagram
}

// newFeed creates new Feed structure
func newFeed(inst *Instagram) *Feed {
	return &Feed{
		inst: inst,
	}
}

// Feed search by locationID
func (feed *Feed) LocationID(locationID int64) (*FeedLocation, error) {
	insta := feed.inst
	body, err := insta.sendRequest(
		&reqOptions{
			Endpoint: fmt.Sprintf(urlFeedLocationID, locationID),
			Query: map[string]string{
				"rank_token":     insta.rankToken,
				"ranked_content": "true",
			},
		},
	)
	if err != nil {
		return nil, err
	}

	res := &FeedLocation{}
	err = json.Unmarshal(body, res)
	return res, err
}

// FeedLocation is the struct that fits the structure returned by instagram on LocationID search.
type FeedLocation struct {
	RankedItems         []Item   `json:"ranked_items"`
	Items               []Item   `json:"items"`
	NumResults          int      `json:"num_results"`
	NextID              string   `json:"next_max_id"`
	MoreAvailable       bool     `json:"more_available"`
	AutoLoadMoreEnabled bool     `json:"auto_load_more_enabled"`
	MediaCount          int      `json:"media_count"`
	Location            Location `json:"location"`
	Status              string   `json:"status"`
}

// Tags search by Tag in user Feed
//
// (sorry for returning FeedTag. See #FeedTag)
func (feed *Feed) Tags(tag string) (*FeedTag, error) {
	insta := feed.inst
	body, err := insta.sendRequest(
		&reqOptions{
			Endpoint: fmt.Sprintf(urlFeedTag, tag),
			Query: map[string]string{
				"rank_token":     insta.rankToken,
				"ranked_content": "true",
			},
		},
	)
	if err != nil {
		return nil, err
	}
	res := &FeedTag{}
	err = json.Unmarshal(body, res)
	if err != nil {
		return nil, err
	}
	for i := range res.RankedItems {
		res.RankedItems[i].media = &FeedMedia{
			inst:   insta,
			NextID: res.RankedItems[i].ID,
		}
	}
	return res, nil
}

// FeedTag is the struct that fits the structure returned by instagram on TagSearch.
type FeedTag struct {
	RankedItems         []Item     `json:"ranked_items"`
	Images              []Item     `json:"items"`
	NumResults          int        `json:"num_results"`
	NextID              string     `json:"next_max_id"`
	MoreAvailable       bool       `json:"more_available"`
	AutoLoadMoreEnabled bool       `json:"auto_load_more_enabled"`
	Story               StoryMedia `json:"story"`
	Status              string     `json:"status"`
}
<|endoftext|>


<commit_before>// +build !luminous,!mimic

package admin

import (
	"errors"
	"fmt"
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var (
	cachedFSAdmin *FSAdmin

	// set debugTrace to true to use tracing in tests
	debugTrace = false
)

func init() {
	dt := os.Getenv("GO_CEPH_TEST_DEBUG_TRACE")
	if dt == "yes" || dt == "true" {
		debugTrace = true
	}
}

// tracingCommander serves two purposes: first, it allows one to trace the
// input and output json when running the tests. It can help with actually
// debugging the tests. Second, it demonstrates the rationale for using an
// interface in FSAdmin. You can layer any sort of debugging, error injection,
// or whatnot between the FSAdmin layer and the RADOS layer.
type tracingCommander struct {
	conn RadosCommander
}

func tracer(c RadosCommander) RadosCommander {
	return &tracingCommander{c}
}

func (t *tracingCommander) MgrCommand(buf [][]byte) ([]byte, string, error) {
	for i := range buf {
		fmt.Println("IN:", string(buf[i]))
	}
	r, s, err := t.conn.MgrCommand(buf)
	fmt.Println("OUT(result):", string(r))
	if s != "" {
		fmt.Println("OUT(status):", s)
	}
	if err != nil {
		fmt.Println("OUT(error):", err.Error())
	}
	return r, s, err
}

func getFSAdmin(t *testing.T) *FSAdmin {
	if cachedFSAdmin != nil {
		return cachedFSAdmin
	}
	cachedFSAdmin, err := New()
	require.NoError(t, err)
	require.NotNil(t, cachedFSAdmin)
	if debugTrace {
		cachedFSAdmin = NewFromConn(tracer(cachedFSAdmin.conn))
	}
	return cachedFSAdmin
}

func TestInvalidFSAdmin(t *testing.T) {
	fsa := &FSAdmin{}
	_, _, err := fsa.rawMgrCommand([]byte("FOOBAR!"))
	assert.Error(t, err)
}

type badMarshalType bool

func (badMarshalType) MarshalJSON() ([]byte, error) {
	return nil, errors.New("Zowie! wow")
}

func TestBadMarshal(t *testing.T) {
	fsa := getFSAdmin(t)

	var bad badMarshalType
	_, _, err := fsa.marshalMgrCommand(bad)
	assert.Error(t, err)
}

func TestParseListNames(t *testing.T) {
	t.Run("error", func(t *testing.T) {
		_, err := parseListNames(nil, "", errors.New("bonk"))
		assert.Error(t, err)
		assert.Equal(t, "bonk", err.Error())
	})
	t.Run("statusSet", func(t *testing.T) {
		_, err := parseListNames(nil, "unexpected!", nil)
		assert.Error(t, err)
	})
	t.Run("badJSON", func(t *testing.T) {
		_, err := parseListNames([]byte("Foo[[["), "", nil)
		assert.Error(t, err)
	})
	t.Run("ok", func(t *testing.T) {
		l, err := parseListNames([]byte(`[{"name":"bob"}]`), "", nil)
		assert.NoError(t, err)
		if assert.Len(t, l, 1) {
			assert.Equal(t, "bob", l[0])
		}
	})
}

func TestCheckEmptyResponseExpected(t *testing.T) {
	t.Run("error", func(t *testing.T) {
		err := checkEmptyResponseExpected(nil, "", errors.New("bonk"))
		assert.Error(t, err)
		assert.Equal(t, "bonk", err.Error())
	})
	t.Run("statusSet", func(t *testing.T) {
		err := checkEmptyResponseExpected(nil, "unexpected!", nil)
		assert.Error(t, err)
	})
	t.Run("someJSON", func(t *testing.T) {
		err := checkEmptyResponseExpected([]byte(`{"trouble": true}`), "", nil)
		assert.Error(t, err)
	})
	t.Run("ok", func(t *testing.T) {
		err := checkEmptyResponseExpected([]byte{}, "", nil)
		assert.NoError(t, err)
	})
}
<commit_msg>cephfs admin: in the test setup code, delay a bit to fetch mgr map<commit_after>// +build !luminous,!mimic

package admin

import (
	"errors"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var (
	cachedFSAdmin *FSAdmin

	// set debugTrace to true to use tracing in tests
	debugTrace = false
)

func init() {
	dt := os.Getenv("GO_CEPH_TEST_DEBUG_TRACE")
	if dt == "yes" || dt == "true" {
		debugTrace = true
	}
}

// tracingCommander serves two purposes: first, it allows one to trace the
// input and output json when running the tests. It can help with actually
// debugging the tests. Second, it demonstrates the rationale for using an
// interface in FSAdmin. You can layer any sort of debugging, error injection,
// or whatnot between the FSAdmin layer and the RADOS layer.
type tracingCommander struct {
	conn RadosCommander
}

func tracer(c RadosCommander) RadosCommander {
	return &tracingCommander{c}
}

func (t *tracingCommander) MgrCommand(buf [][]byte) ([]byte, string, error) {
	for i := range buf {
		fmt.Println("IN:", string(buf[i]))
	}
	r, s, err := t.conn.MgrCommand(buf)
	fmt.Println("OUT(result):", string(r))
	if s != "" {
		fmt.Println("OUT(status):", s)
	}
	if err != nil {
		fmt.Println("OUT(error):", err.Error())
	}
	return r, s, err
}

func getFSAdmin(t *testing.T) *FSAdmin {
	if cachedFSAdmin != nil {
		return cachedFSAdmin
	}
	fsa, err := New()
	require.NoError(t, err)
	require.NotNil(t, fsa)
	// We steal the connection set up by the New() method and wrap it in an
	// optional tracer.
	c := fsa.conn
	if debugTrace {
		c = tracer(c)
	}
	cachedFSAdmin = NewFromConn(c)
	// We sleep briefly before returning in order to ensure we have a mgr map
	// before we start executing the tests.
	time.Sleep(50 * time.Millisecond)
	return cachedFSAdmin
}

func TestInvalidFSAdmin(t *testing.T) {
	fsa := &FSAdmin{}
	_, _, err := fsa.rawMgrCommand([]byte("FOOBAR!"))
	assert.Error(t, err)
}

type badMarshalType bool

func (badMarshalType) MarshalJSON() ([]byte, error) {
	return nil, errors.New("Zowie! wow")
}

func TestBadMarshal(t *testing.T) {
	fsa := getFSAdmin(t)

	var bad badMarshalType
	_, _, err := fsa.marshalMgrCommand(bad)
	assert.Error(t, err)
}

func TestParseListNames(t *testing.T) {
	t.Run("error", func(t *testing.T) {
		_, err := parseListNames(nil, "", errors.New("bonk"))
		assert.Error(t, err)
		assert.Equal(t, "bonk", err.Error())
	})
	t.Run("statusSet", func(t *testing.T) {
		_, err := parseListNames(nil, "unexpected!", nil)
		assert.Error(t, err)
	})
	t.Run("badJSON", func(t *testing.T) {
		_, err := parseListNames([]byte("Foo[[["), "", nil)
		assert.Error(t, err)
	})
	t.Run("ok", func(t *testing.T) {
		l, err := parseListNames([]byte(`[{"name":"bob"}]`), "", nil)
		assert.NoError(t, err)
		if assert.Len(t, l, 1) {
			assert.Equal(t, "bob", l[0])
		}
	})
}

func TestCheckEmptyResponseExpected(t *testing.T) {
	t.Run("error", func(t *testing.T) {
		err := checkEmptyResponseExpected(nil, "", errors.New("bonk"))
		assert.Error(t, err)
		assert.Equal(t, "bonk", err.Error())
	})
	t.Run("statusSet", func(t *testing.T) {
		err := checkEmptyResponseExpected(nil, "unexpected!", nil)
		assert.Error(t, err)
	})
	t.Run("someJSON", func(t *testing.T) {
		err := checkEmptyResponseExpected([]byte(`{"trouble": true}`), "", nil)
		assert.Error(t, err)
	})
	t.Run("ok", func(t *testing.T) {
		err := checkEmptyResponseExpected([]byte{}, "", nil)
		assert.NoError(t, err)
	})
}
<|endoftext|>


<commit_before>package zgok

import (
	"os"
	"os/exec"
	fpath "path/filepath"
	"runtime"
	"testing"
)

const (
	REAL_EXE_SRC_PATH = "testdata/hello.go"
	REAL_EXE_PATH     = "hello.out"
	DUMMY_EXE_PATH    = "testdata/executable"
)

var (
	exePath string = DUMMY_EXE_PATH
)

func TestMain(m *testing.M) {
	setup()
	code := m.Run()
	defer os.Exit(code)
	teardown()
}

func setup() {
	// Erase unused *.out files.
	paths, _ := fpath.Glob("*.out")
	for _, path := range paths {
		os.Remove(path)
	}
	// Compile testdata/hello.go
	cmd := exec.Command("go", "build", "-o", REAL_EXE_PATH, REAL_EXE_SRC_PATH)
	err := cmd.Run()
	if err == nil {
		exePath = REAL_EXE_PATH
	}
}

func teardown() {
	// Erase unused *.out files.
	paths, _ := fpath.Glob("*.out")
	for _, path := range paths {
		os.Remove(path)
	}
}

func TestBuildRestore(t *testing.T) {
	// Build zgok file.
	outPath := "builder_test.out"
	builder := NewZgokBuilder()
	err := builder.SetExePath(exePath)
	if err != nil {
		t.Errorf("SetExePath():error=[%v]", err)
	}
	err = builder.AddZipPath("testdata/foo")
	if err != nil {
		t.Errorf("AddZipPath():error=[%v]", err)
	}
	err = builder.AddZipPath("testdata/dir")
	if err != nil {
		t.Errorf("AddZipPath():error=[%v]", err)
	}
	builder.SetOutPath(outPath)
	err = builder.Build()
	if err != nil {
		t.Errorf("Build():error=[%v]", err)
	}
	// Load zgok filesystem.
	zfs, err := RestoreFileSystem(outPath)
	if err != nil {
		t.Errorf("RestoreFileSystem():error=[%v]", err)
	}
	// Compare exe size.
	exeStat, err := os.Stat(exePath)
	if err != nil {
		t.Errorf("Failed to read stat of [%v].", exePath)
	}
	if exeStat.Size() != zfs.Signature().ExeSize() {
		t.Errorf("Exe size:expected [%v] got [%v].",
			exeStat.Size(), zfs.Signature().ExeSize())
	}
	// Compare included zip paths.
	zipPaths := []string{
		"testdata/dir/bar",
		"testdata/dir/baz",
		"testdata/foo",
	}
	for i, path := range zfs.Paths() {
		if zipPaths[i] != path {
			t.Errorf("Paths():expected [%v] got [%v].", zipPaths[i], path)
		}
	}
	// Check if exe is not broken.
	if exePath == REAL_EXE_PATH {
		// Get parent path.
		_, filename, _, _ := runtime.Caller(0)
		parentPath, err := fpath.Abs(fpath.Join(filename, ".."))
		// Get expected output.
		exBytes, err := exec.Command(fpath.Join(parentPath, exePath)).Output()
		if err != nil {
			t.Errorf("Failed to run test executable:[%v]", err.Error())
		}
		expected := string(exBytes[:])
		// Get output of the built executable file.
		outBytes, err := exec.Command(fpath.Join(parentPath, outPath)).Output()
		if err != nil {
			t.Errorf("Failed to run built executable:[%v]", err.Error())
		}
		output := string(outBytes[:])
		// Compare the results.
		if expected != output {
			t.Errorf("Different output for built executable:expected [%v] got [%v].",
				expected, output)
		}
	}
}

func TestBuilderSetInvalidExePath(t *testing.T) {
	builder := NewZgokBuilder()
	err := builder.SetExePath("invalid")
	if err == nil {
		t.Errorf("Expected error on setting invalid exe path.")
	}
}

func TestBuilderAddInvalidZipPath(t *testing.T) {
	builder := NewZgokBuilder()
	err := builder.AddZipPath("invalid")
	if err == nil {
		t.Errorf("Expected error on adding invalid zip path.")
	}
}

func TestBuilderBuildWithoutExePath(t *testing.T) {
	builder := NewZgokBuilder()
	builder.AddZipPath("testdata/foo")
	builder.AddZipPath("testdata/dir")
	builder.SetOutPath("builder_test.out")
	err := builder.Build()
	if err == nil {
		t.Errorf("Expected error on build without exe path.")
	}
}

func TestBuilderBuildWithoutZipPath(t *testing.T) {
	builder := NewZgokBuilder()
	builder.SetExePath("testdata/executable")
	builder.SetOutPath("builder_test.out")
	err := builder.Build()
	if err == nil {
		t.Errorf("Expected error on build without zip path.")
	}
}
<commit_msg>[Update] Updated builder_test to print which exe is used.<commit_after>package zgok

import (
	"os"
	"os/exec"
	fpath "path/filepath"
	"runtime"
	"testing"
)

const (
	REAL_EXE_SRC_PATH = "testdata/hello.go"
	REAL_EXE_PATH     = "hello.out"
	DUMMY_EXE_PATH    = "testdata/executable"
)

var (
	exePath string = DUMMY_EXE_PATH
)

func TestMain(m *testing.M) {
	setup()
	code := m.Run()
	defer os.Exit(code)
	teardown()
}

func setup() {
	// Erase unused *.out files.
	paths, _ := fpath.Glob("*.out")
	for _, path := range paths {
		os.Remove(path)
	}
	// Compile testdata/hello.go
	cmd := exec.Command("go", "build", "-o", REAL_EXE_PATH, REAL_EXE_SRC_PATH)
	err := cmd.Run()
	if err == nil {
		exePath = REAL_EXE_PATH
	}
}

func teardown() {
	// Erase unused *.out files.
	paths, _ := fpath.Glob("*.out")
	for _, path := range paths {
		os.Remove(path)
	}
}

func TestBuildRestore(t *testing.T) {
	// Show comment on exePath.
	if exePath == REAL_EXE_PATH {
		t.Logf("Testing with golang built exec [%s].", exePath)
	} else {
		t.Logf("Testing with dummy exec [%s].", exePath)
	}
	// Build zgok file.
	outPath := "builder_test.out"
	builder := NewZgokBuilder()
	err := builder.SetExePath(exePath)
	if err != nil {
		t.Errorf("SetExePath():error=[%v]", err)
	}
	err = builder.AddZipPath("testdata/foo")
	if err != nil {
		t.Errorf("AddZipPath():error=[%v]", err)
	}
	err = builder.AddZipPath("testdata/dir")
	if err != nil {
		t.Errorf("AddZipPath():error=[%v]", err)
	}
	builder.SetOutPath(outPath)
	err = builder.Build()
	if err != nil {
		t.Errorf("Build():error=[%v]", err)
	}
	// Load zgok filesystem.
	zfs, err := RestoreFileSystem(outPath)
	if err != nil {
		t.Errorf("RestoreFileSystem():error=[%v]", err)
	}
	// Compare exe size.
	exeStat, err := os.Stat(exePath)
	if err != nil {
		t.Errorf("Failed to read stat of [%v].", exePath)
	}
	if exeStat.Size() != zfs.Signature().ExeSize() {
		t.Errorf("Exe size:expected [%v] got [%v].",
			exeStat.Size(), zfs.Signature().ExeSize())
	}
	// Compare included zip paths.
	zipPaths := []string{
		"testdata/dir/bar",
		"testdata/dir/baz",
		"testdata/foo",
	}
	for i, path := range zfs.Paths() {
		if zipPaths[i] != path {
			t.Errorf("Paths():expected [%v] got [%v].", zipPaths[i], path)
		}
	}
	// Check if exe is not broken.
	if exePath == REAL_EXE_PATH {
		// Get parent path.
		_, filename, _, _ := runtime.Caller(0)
		parentPath, err := fpath.Abs(fpath.Join(filename, ".."))
		// Get expected output.
		exBytes, err := exec.Command(fpath.Join(parentPath, exePath)).Output()
		if err != nil {
			t.Errorf("Failed to run test executable:[%v]", err.Error())
		}
		expected := string(exBytes[:])
		// Get output of the built executable file.
		outBytes, err := exec.Command(fpath.Join(parentPath, outPath)).Output()
		if err != nil {
			t.Errorf("Failed to run built executable:[%v]", err.Error())
		}
		output := string(outBytes[:])
		// Compare the results.
		if expected != output {
			t.Errorf("Different output for built executable:expected [%v] got [%v].",
				expected, output)
		}
	}
}

func TestBuilderSetInvalidExePath(t *testing.T) {
	builder := NewZgokBuilder()
	err := builder.SetExePath("invalid")
	if err == nil {
		t.Errorf("Expected error on setting invalid exe path.")
	}
}

func TestBuilderAddInvalidZipPath(t *testing.T) {
	builder := NewZgokBuilder()
	err := builder.AddZipPath("invalid")
	if err == nil {
		t.Errorf("Expected error on adding invalid zip path.")
	}
}

func TestBuilderBuildWithoutExePath(t *testing.T) {
	builder := NewZgokBuilder()
	builder.AddZipPath("testdata/foo")
	builder.AddZipPath("testdata/dir")
	builder.SetOutPath("builder_test.out")
	err := builder.Build()
	if err == nil {
		t.Errorf("Expected error on build without exe path.")
	}
}

func TestBuilderBuildWithoutZipPath(t *testing.T) {
	builder := NewZgokBuilder()
	builder.SetExePath("testdata/executable")
	builder.SetOutPath("builder_test.out")
	err := builder.Build()
	if err == nil {
		t.Errorf("Expected error on build without zip path.")
	}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vellum

import (
	"bufio"
	"io/ioutil"
	"math/rand"
	"os"
	"sort"
	"testing"
)

func init() {
	thousandTestWords, _ = loadWords("data/words-1000.txt")
}

// this simple test case only has a shared final state
// it also tests out of order insert
func TestBuilderSimple(t *testing.T) {
	b, err := New(ioutil.Discard, nil)
	if err != nil {
		t.Fatalf("error creating builder: %v", err)
	}

	// add our first string
	err = b.Insert([]byte("jul"), 0)
	if err != nil {
		t.Errorf("got error inserting string: %v", err)
	}
	// expect len to be 1
	if b.len != 1 {
		t.Errorf("expected node count to be 1, got %v", b.len)
	}

	// try to add a value out of order (not allowed)
	err = b.Insert([]byte("abc"), 0)
	if err == nil {
		t.Errorf("expected err, got nil")
	}

	// add a second string
	err = b.Insert([]byte("mar"), 0)
	if err != nil {
		t.Errorf("got error inserting string: %v", err)
	}
	// expect len to grow by 1
	if b.len != 2 {
		t.Errorf("expected node count to be 2, got %v", b.len)
	}

	// now close the builder
	err = b.Close()
	if err != nil {
		t.Errorf("got error closing set builder: %v", err)
	}
}

func TestBuilderSharedPrefix(t *testing.T) {
	b, err := New(ioutil.Discard, nil)
	if err != nil {
		t.Fatalf("error creating builder: %v", err)
	}

	// add our first string
	err = b.Insert([]byte("car"), 0)
	if err != nil {
		t.Errorf("got error inserting string: %v", err)
	}
	// expect len to be 1
	if b.len != 1 {
		t.Errorf("expected node count to be 1, got %v", b.len)
	}

	// add a second string
	err = b.Insert([]byte("cat"), 0)
	if err != nil {
		t.Errorf("got error inserting string: %v", err)
	}
	// expect len to be 2
	if b.len != 2 {
		t.Errorf("expected node count to be 2, got %v", b.len)
	}

	// now close the builder
	err = b.Close()
	if err != nil {
		t.Errorf("got error closing set builder: %v", err)
	}
}

func randomValues(list []string) []uint64 {
	rv := make([]uint64, len(list))
	for i := range list {
		rv[i] = uint64(rand.Uint64())
	}
	return rv
}

func insertStrings(b *Builder, list []string, vals []uint64) error {
	for i, item := range list {
		err := b.Insert([]byte(item), vals[i])
		if err != nil {
			return err
		}
	}
	return nil
}

var smallSample = map[string]uint64{
	"mon":   2,
	"tues":  3,
	"thurs": 5,
	"tye":   99,
}

func insertStringMap(b *Builder, m map[string]uint64) error {
	// make list of keys
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	// sort it
	sort.Strings(keys)
	// insert in sorted order
	for _, k := range keys {
		err := b.Insert([]byte(k), m[k])
		if err != nil {
			return err
		}
	}
	return nil
}

func TestBuilderNodeEquiv(t *testing.T) {
	tests := []struct {
		desc string
		a    *builderNode
		b    *builderNode
		want bool
	}{
		{
			"both states final",
			&builderNode{
				final: true,
			},
			&builderNode{
				final: true,
			},
			true,
		},
		{
			"both states final, different final val",
			&builderNode{
				final:       true,
				finalOutput: 7,
			},
			&builderNode{
				final:       true,
				finalOutput: 9,
			},
			false,
		},
		{
			"both states final, same transitions, but different trans val",
			&builderNode{
				final: true,
				trans: []*transition{
					{in: 'a', out: 7},
				},
			},
			&builderNode{
				final: true,
				trans: []*transition{
					{in: 'a', out: 9},
				},
			},
			false,
		},
	}

	for _, test := range tests {
		t.Run(test.desc, func(t *testing.T) {
			got := test.a.equiv(test.b)
			if got != test.want {
				t.Errorf("wanted: %t, got: %t", test.want, got)
			}
		})
	}
}

func loadWords(path string) ([]string, error) {
	var rv []string

	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		word := append([]byte(nil), scanner.Bytes()...)
		rv = append(rv, string(word))
		if err != nil {
			return nil, err
		}
	}

	if err = scanner.Err(); err != nil {
		return nil, err
	}

	return rv, nil
}

var thousandTestWords []string

func BenchmarkBuilder(b *testing.B) {
	dataset := thousandTestWords
	randomThousandVals := randomValues(dataset)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {

		builder, err := New(ioutil.Discard, nil)
		if err != nil {
			b.Fatalf("error creating builder: %v", err)
		}
		err = insertStrings(builder, dataset, randomThousandVals)
		if err != nil {
			b.Fatalf("error inserting thousand words: %v", err)
		}
		err = builder.Close()
		if err != nil {
			b.Fatalf("error closing builder: %v", err)
		}
	}
}
<commit_msg>fix unchecked err<commit_after>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vellum

import (
	"bufio"
	"io/ioutil"
	"math/rand"
	"os"
	"sort"
	"testing"
)

func init() {
	thousandTestWords, _ = loadWords("data/words-1000.txt")
}

// this simple test case only has a shared final state
// it also tests out of order insert
func TestBuilderSimple(t *testing.T) {
	b, err := New(ioutil.Discard, nil)
	if err != nil {
		t.Fatalf("error creating builder: %v", err)
	}

	// add our first string
	err = b.Insert([]byte("jul"), 0)
	if err != nil {
		t.Errorf("got error inserting string: %v", err)
	}
	// expect len to be 1
	if b.len != 1 {
		t.Errorf("expected node count to be 1, got %v", b.len)
	}

	// try to add a value out of order (not allowed)
	err = b.Insert([]byte("abc"), 0)
	if err == nil {
		t.Errorf("expected err, got nil")
	}

	// add a second string
	err = b.Insert([]byte("mar"), 0)
	if err != nil {
		t.Errorf("got error inserting string: %v", err)
	}
	// expect len to grow by 1
	if b.len != 2 {
		t.Errorf("expected node count to be 2, got %v", b.len)
	}

	// now close the builder
	err = b.Close()
	if err != nil {
		t.Errorf("got error closing set builder: %v", err)
	}
}

func TestBuilderSharedPrefix(t *testing.T) {
	b, err := New(ioutil.Discard, nil)
	if err != nil {
		t.Fatalf("error creating builder: %v", err)
	}

	// add our first string
	err = b.Insert([]byte("car"), 0)
	if err != nil {
		t.Errorf("got error inserting string: %v", err)
	}
	// expect len to be 1
	if b.len != 1 {
		t.Errorf("expected node count to be 1, got %v", b.len)
	}

	// add a second string
	err = b.Insert([]byte("cat"), 0)
	if err != nil {
		t.Errorf("got error inserting string: %v", err)
	}
	// expect len to be 2
	if b.len != 2 {
		t.Errorf("expected node count to be 2, got %v", b.len)
	}

	// now close the builder
	err = b.Close()
	if err != nil {
		t.Errorf("got error closing set builder: %v", err)
	}
}

func randomValues(list []string) []uint64 {
	rv := make([]uint64, len(list))
	for i := range list {
		rv[i] = uint64(rand.Uint64())
	}
	return rv
}

func insertStrings(b *Builder, list []string, vals []uint64) error {
	for i, item := range list {
		err := b.Insert([]byte(item), vals[i])
		if err != nil {
			return err
		}
	}
	return nil
}

var smallSample = map[string]uint64{
	"mon":   2,
	"tues":  3,
	"thurs": 5,
	"tye":   99,
}

func insertStringMap(b *Builder, m map[string]uint64) error {
	// make list of keys
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	// sort it
	sort.Strings(keys)
	// insert in sorted order
	for _, k := range keys {
		err := b.Insert([]byte(k), m[k])
		if err != nil {
			return err
		}
	}
	return nil
}

func TestBuilderNodeEquiv(t *testing.T) {
	tests := []struct {
		desc string
		a    *builderNode
		b    *builderNode
		want bool
	}{
		{
			"both states final",
			&builderNode{
				final: true,
			},
			&builderNode{
				final: true,
			},
			true,
		},
		{
			"both states final, different final val",
			&builderNode{
				final:       true,
				finalOutput: 7,
			},
			&builderNode{
				final:       true,
				finalOutput: 9,
			},
			false,
		},
		{
			"both states final, same transitions, but different trans val",
			&builderNode{
				final: true,
				trans: []*transition{
					{in: 'a', out: 7},
				},
			},
			&builderNode{
				final: true,
				trans: []*transition{
					{in: 'a', out: 9},
				},
			},
			false,
		},
	}

	for _, test := range tests {
		t.Run(test.desc, func(t *testing.T) {
			got := test.a.equiv(test.b)
			if got != test.want {
				t.Errorf("wanted: %t, got: %t", test.want, got)
			}
		})
	}
}

func loadWords(path string) ([]string, error) {
	var rv []string

	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		word := append([]byte(nil), scanner.Bytes()...)
		rv = append(rv, string(word))
		if err != nil {
			return nil, err
		}
	}

	if err = scanner.Err(); err != nil {
		return nil, err
	}

	err = file.Close()
	if err != nil {
		return nil, err
	}

	return rv, nil
}

var thousandTestWords []string

func BenchmarkBuilder(b *testing.B) {
	dataset := thousandTestWords
	randomThousandVals := randomValues(dataset)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {

		builder, err := New(ioutil.Discard, nil)
		if err != nil {
			b.Fatalf("error creating builder: %v", err)
		}
		err = insertStrings(builder, dataset, randomThousandVals)
		if err != nil {
			b.Fatalf("error inserting thousand words: %v", err)
		}
		err = builder.Close()
		if err != nil {
			b.Fatalf("error closing builder: %v", err)
		}
	}
}
<|endoftext|>


<commit_before>// Package http implements functionality common to HTTP uploading pipelines.
package http

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"html/template"
	"io"
	h "net/http"
	"os"
	"runtime"
	"strings"

	"github.com/apex/log"
	"github.com/pkg/errors"

	"github.com/goreleaser/goreleaser/internal/artifact"
	"github.com/goreleaser/goreleaser/internal/pipe"
	"github.com/goreleaser/goreleaser/internal/semerrgroup"
	"github.com/goreleaser/goreleaser/pkg/config"
	"github.com/goreleaser/goreleaser/pkg/context"
)

const (
	// ModeBinary uploads only compiled binaries
	ModeBinary = "binary"
	// ModeArchive uploads release archives
	ModeArchive = "archive"
)

type asset struct {
	ReadCloser io.ReadCloser
	Size       int64
}

type assetOpenFunc func(string, *artifact.Artifact) (*asset, error)

// nolint: gochecknoglobals
var assetOpen assetOpenFunc

// TODO: fix this.
// nolint: gochecknoinits
func init() {
	assetOpenReset()
}

func assetOpenReset() {
	assetOpen = assetOpenDefault
}

func assetOpenDefault(kind string, a *artifact.Artifact) (*asset, error) {
	f, err := os.Open(a.Path)
	if err != nil {
		return nil, err
	}
	s, err := f.Stat()
	if err != nil {
		return nil, err
	}
	if s.IsDir() {
		return nil, errors.Errorf("%s: upload failed: the asset to upload can't be a directory", kind)
	}
	return &asset{
		ReadCloser: f,
		Size:       s.Size(),
	}, nil
}

// Defaults sets default configuration options on Put structs
func Defaults(puts []config.Put) error {
	for i := range puts {
		defaults(&puts[i])
	}
	return nil
}

func defaults(put *config.Put) {
	if put.Mode == "" {
		put.Mode = ModeArchive
	}
}

// CheckConfig validates a Put configuration returning a descriptive error when appropriate
func CheckConfig(ctx *context.Context, put *config.Put, kind string) error {

	if put.Target == "" {
		return misconfigured(kind, put, "missing target")
	}

	if put.Name == "" {
		return misconfigured(kind, put, "missing name")
	}

	if put.Mode != ModeArchive && put.Mode != ModeBinary {
		return misconfigured(kind, put, "mode must be 'binary' or 'archive'")
	}

	envName := fmt.Sprintf("%s_%s_SECRET", strings.ToUpper(kind), strings.ToUpper(put.Name))
	if _, ok := ctx.Env[envName]; !ok {
		return misconfigured(kind, put, fmt.Sprintf("missing %s environment variable", envName))
	}

	if put.TrustedCerts != "" && !x509.NewCertPool().AppendCertsFromPEM([]byte(put.TrustedCerts)) {
		return misconfigured(kind, put, "no certificate could be added from the specified trusted_certificates configuration")
	}

	return nil

}

func misconfigured(kind string, upload *config.Put, reason string) error {
	return pipe.Skip(fmt.Sprintf("%s section '%s' is not configured properly (%s)", kind, upload.Name, reason))
}

// ResponseChecker is a function capable of validating an http server response.
// It must return and error when the response must be considered a failure.
type ResponseChecker func(*h.Response) error

// Upload does the actual uploading work
func Upload(ctx *context.Context, puts []config.Put, kind string, check ResponseChecker) error {
	if ctx.SkipPublish {
		return pipe.ErrSkipPublishEnabled
	}

	// Handle every configured put
	for _, put := range puts {
		put := put
		filters := []artifact.Filter{}
		if put.Checksum {
			filters = append(filters, artifact.ByType(artifact.Checksum))
		}
		if put.Signature {
			filters = append(filters, artifact.ByType(artifact.Signature))
		}
		// We support two different modes
		//	- "archive": Upload all artifacts
		//	- "binary": Upload only the raw binaries
		switch v := strings.ToLower(put.Mode); v {
		case ModeArchive:
			filters = append(filters,
				artifact.ByType(artifact.UploadableArchive),
				artifact.ByType(artifact.LinuxPackage),
			)
		case ModeBinary:
			filters = append(filters, artifact.ByType(artifact.UploadableBinary))
		default:
			err := fmt.Errorf("%s: mode \"%s\" not supported", kind, v)
			log.WithFields(log.Fields{
				kind:   put.Name,
				"mode": v,
			}).Error(err.Error())
			return err
		}

		var filter = artifact.Or(filters...)
		if len(put.IDs) > 0 {
			filter = artifact.And(filter, artifact.ByIDs(put.IDs...))
		}
		if err := uploadWithFilter(ctx, &put, filter, kind, check); err != nil {
			return err
		}
	}

	return nil
}

func uploadWithFilter(ctx *context.Context, put *config.Put, filter artifact.Filter, kind string, check ResponseChecker) error {
	var artifacts = ctx.Artifacts.Filter(filter).List()
	log.Debugf("will upload %d artifacts", len(artifacts))
	var g = semerrgroup.New(ctx.Parallelism)
	for _, artifact := range artifacts {
		artifact := artifact
		g.Go(func() error {
			return uploadAsset(ctx, put, artifact, kind, check)
		})
	}
	return g.Wait()
}

// uploadAsset uploads file to target and logs all actions
func uploadAsset(ctx *context.Context, put *config.Put, artifact artifact.Artifact, kind string, check ResponseChecker) error {
	envBase := fmt.Sprintf("%s_%s_", strings.ToUpper(kind), strings.ToUpper(put.Name))
	username := put.Username
	if username == "" {
		// username not configured: using env
		username = ctx.Env[envBase+"USERNAME"]
	}
	secret := ctx.Env[envBase+"SECRET"]

	// Generate the target url
	targetURL, err := resolveTargetTemplate(ctx, put, artifact)
	if err != nil {
		msg := fmt.Sprintf("%s: error while building the target url", kind)
		log.WithField("instance", put.Name).WithError(err).Error(msg)
		return errors.Wrap(err, msg)
	}

	// Handle the artifact
	asset, err := assetOpen(kind, &artifact)
	if err != nil {
		return err
	}
	defer asset.ReadCloser.Close() // nolint: errcheck

	// The target url needs to contain the artifact name
	if !strings.HasSuffix(targetURL, "/") {
		targetURL += "/"
	}
	targetURL += artifact.Name

	var headers = map[string]string{}
	if put.ChecksumHeader != "" {
		sum, err := artifact.Checksum("sha256")
		if err != nil {
			return err
		}
		headers[put.ChecksumHeader] = sum
	}

	_, err = uploadAssetToServer(ctx, put, targetURL, username, secret, headers, asset, check)
	if err != nil {
		msg := fmt.Sprintf("%s: upload failed", kind)
		log.WithError(err).WithFields(log.Fields{
			"instance": put.Name,
			"username": username,
		}).Error(msg)
		return errors.Wrap(err, msg)
	}

	log.WithFields(log.Fields{
		"instance": put.Name,
		"mode":     put.Mode,
	}).Info("uploaded successful")

	return nil
}

// uploadAssetToServer uploads the asset file to target
func uploadAssetToServer(ctx *context.Context, put *config.Put, target, username, secret string, headers map[string]string, a *asset, check ResponseChecker) (*h.Response, error) {
	req, err := newUploadRequest(target, username, secret, headers, a)
	if err != nil {
		return nil, err
	}

	return executeHTTPRequest(ctx, put, req, check)
}

// newUploadRequest creates a new h.Request for uploading
func newUploadRequest(target, username, secret string, headers map[string]string, a *asset) (*h.Request, error) {
	req, err := h.NewRequest(h.MethodPut, target, a.ReadCloser)
	if err != nil {
		return nil, err
	}
	req.ContentLength = a.Size
	req.SetBasicAuth(username, secret)

	for k, v := range headers {
		req.Header.Add(k, v)
	}

	return req, err
}

func getHTTPClient(put *config.Put) (*h.Client, error) {
	if put.TrustedCerts == "" {
		return h.DefaultClient, nil
	}
	pool, err := x509.SystemCertPool()
	if err != nil {
		if runtime.GOOS == "windows" {
			// on windows ignore errors until golang issues #16736 & #18609 get fixed
			pool = x509.NewCertPool()
		} else {
			return nil, err
		}
	}
	pool.AppendCertsFromPEM([]byte(put.TrustedCerts)) // already validated certs checked by CheckConfig
	return &h.Client{
		Transport: &h.Transport{
			TLSClientConfig: &tls.Config{
				RootCAs: pool,
			},
		},
	}, nil
}

// executeHTTPRequest processes the http call with respect of context ctx
func executeHTTPRequest(ctx *context.Context, put *config.Put, req *h.Request, check ResponseChecker) (*h.Response, error) {
	client, err := getHTTPClient(put)
	if err != nil {
		return nil, err
	}
	log.Debugf("executing request: %s %s (headers: %v)", req.Method, req.URL, req.Header)
	resp, err := client.Do(req)
	if err != nil {
		// If we got an error, and the context has been canceled,
		// the context's error is probably more useful.
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
		}
		return nil, err
	}

	defer resp.Body.Close() // nolint: errcheck

	err = check(resp)
	if err != nil {
		// even though there was an error, we still return the response
		// in case the caller wants to inspect it further
		return resp, err
	}

	return resp, err
}

// targetData is used as a template struct for
// Artifactory.Target
type targetData struct {
	Version     string
	Tag         string
	ProjectName string

	// Only supported in mode binary
	Os   string
	Arch string
	Arm  string
}

// resolveTargetTemplate returns the resolved target template with replaced variables
// Those variables can be replaced by the given context, goos, goarch, goarm and more
// TODO: replace this with our internal template pkg
func resolveTargetTemplate(ctx *context.Context, put *config.Put, artifact artifact.Artifact) (string, error) {
	data := targetData{
		Version:     ctx.Version,
		Tag:         ctx.Git.CurrentTag,
		ProjectName: ctx.Config.ProjectName,
	}

	if put.Mode == ModeBinary {
		data.Os = replace(ctx.Config.Archive.Replacements, artifact.Goos)
		data.Arch = replace(ctx.Config.Archive.Replacements, artifact.Goarch)
		data.Arm = replace(ctx.Config.Archive.Replacements, artifact.Goarm)
	}

	var out bytes.Buffer
	t, err := template.New(ctx.Config.ProjectName).Parse(put.Target)
	if err != nil {
		return "", err
	}
	err = t.Execute(&out, data)
	return out.String(), err
}

func replace(replacements map[string]string, original string) string {
	result := replacements[original]
	if result == "" {
		return original
	}
	return result
}
<commit_msg>chore: added todo<commit_after>// Package http implements functionality common to HTTP uploading pipelines.
package http

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"html/template"
	"io"
	h "net/http"
	"os"
	"runtime"
	"strings"

	"github.com/apex/log"
	"github.com/pkg/errors"

	"github.com/goreleaser/goreleaser/internal/artifact"
	"github.com/goreleaser/goreleaser/internal/pipe"
	"github.com/goreleaser/goreleaser/internal/semerrgroup"
	"github.com/goreleaser/goreleaser/pkg/config"
	"github.com/goreleaser/goreleaser/pkg/context"
)

const (
	// ModeBinary uploads only compiled binaries
	ModeBinary = "binary"
	// ModeArchive uploads release archives
	ModeArchive = "archive"
)

type asset struct {
	ReadCloser io.ReadCloser
	Size       int64
}

type assetOpenFunc func(string, *artifact.Artifact) (*asset, error)

// nolint: gochecknoglobals
var assetOpen assetOpenFunc

// TODO: fix this.
// nolint: gochecknoinits
func init() {
	assetOpenReset()
}

func assetOpenReset() {
	assetOpen = assetOpenDefault
}

func assetOpenDefault(kind string, a *artifact.Artifact) (*asset, error) {
	f, err := os.Open(a.Path)
	if err != nil {
		return nil, err
	}
	s, err := f.Stat()
	if err != nil {
		return nil, err
	}
	if s.IsDir() {
		return nil, errors.Errorf("%s: upload failed: the asset to upload can't be a directory", kind)
	}
	return &asset{
		ReadCloser: f,
		Size:       s.Size(),
	}, nil
}

// Defaults sets default configuration options on Put structs
func Defaults(puts []config.Put) error {
	for i := range puts {
		defaults(&puts[i])
	}
	return nil
}

func defaults(put *config.Put) {
	if put.Mode == "" {
		put.Mode = ModeArchive
	}
}

// CheckConfig validates a Put configuration returning a descriptive error when appropriate
func CheckConfig(ctx *context.Context, put *config.Put, kind string) error {
	if put.Target == "" {
		return misconfigured(kind, put, "missing target")
	}

	if put.Name == "" {
		return misconfigured(kind, put, "missing name")
	}

	if put.Mode != ModeArchive && put.Mode != ModeBinary {
		return misconfigured(kind, put, "mode must be 'binary' or 'archive'")
	}

	envName := fmt.Sprintf("%s_%s_SECRET", strings.ToUpper(kind), strings.ToUpper(put.Name))
	if _, ok := ctx.Env[envName]; !ok {
		return misconfigured(kind, put, fmt.Sprintf("missing %s environment variable", envName))
	}

	if put.TrustedCerts != "" && !x509.NewCertPool().AppendCertsFromPEM([]byte(put.TrustedCerts)) {
		return misconfigured(kind, put, "no certificate could be added from the specified trusted_certificates configuration")
	}

	return nil

}

func misconfigured(kind string, upload *config.Put, reason string) error {
	return pipe.Skip(fmt.Sprintf("%s section '%s' is not configured properly (%s)", kind, upload.Name, reason))
}

// ResponseChecker is a function capable of validating an http server response.
// It must return and error when the response must be considered a failure.
type ResponseChecker func(*h.Response) error

// Upload does the actual uploading work
func Upload(ctx *context.Context, puts []config.Put, kind string, check ResponseChecker) error {
	if ctx.SkipPublish {
		return pipe.ErrSkipPublishEnabled
	}

	// Handle every configured put
	for _, put := range puts {
		put := put
		filters := []artifact.Filter{}
		if put.Checksum {
			filters = append(filters, artifact.ByType(artifact.Checksum))
		}
		if put.Signature {
			filters = append(filters, artifact.ByType(artifact.Signature))
		}
		// We support two different modes
		//	- "archive": Upload all artifacts
		//	- "binary": Upload only the raw binaries
		switch v := strings.ToLower(put.Mode); v {
		case ModeArchive:
			filters = append(filters,
				artifact.ByType(artifact.UploadableArchive),
				artifact.ByType(artifact.LinuxPackage),
			)
		case ModeBinary:
			filters = append(filters, artifact.ByType(artifact.UploadableBinary))
		default:
			err := fmt.Errorf("%s: mode \"%s\" not supported", kind, v)
			log.WithFields(log.Fields{
				kind:   put.Name,
				"mode": v,
			}).Error(err.Error())
			return err
		}

		var filter = artifact.Or(filters...)
		if len(put.IDs) > 0 {
			filter = artifact.And(filter, artifact.ByIDs(put.IDs...))
		}
		if err := uploadWithFilter(ctx, &put, filter, kind, check); err != nil {
			return err
		}
	}

	return nil
}

func uploadWithFilter(ctx *context.Context, put *config.Put, filter artifact.Filter, kind string, check ResponseChecker) error {
	var artifacts = ctx.Artifacts.Filter(filter).List()
	log.Debugf("will upload %d artifacts", len(artifacts))
	var g = semerrgroup.New(ctx.Parallelism)
	for _, artifact := range artifacts {
		artifact := artifact
		g.Go(func() error {
			return uploadAsset(ctx, put, artifact, kind, check)
		})
	}
	return g.Wait()
}

// uploadAsset uploads file to target and logs all actions
func uploadAsset(ctx *context.Context, put *config.Put, artifact artifact.Artifact, kind string, check ResponseChecker) error {
	envBase := fmt.Sprintf("%s_%s_", strings.ToUpper(kind), strings.ToUpper(put.Name))
	username := put.Username
	if username == "" {
		// username not configured: using env
		username = ctx.Env[envBase+"USERNAME"]
	}
	secret := ctx.Env[envBase+"SECRET"]

	// Generate the target url
	targetURL, err := resolveTargetTemplate(ctx, put, artifact)
	if err != nil {
		msg := fmt.Sprintf("%s: error while building the target url", kind)
		log.WithField("instance", put.Name).WithError(err).Error(msg)
		return errors.Wrap(err, msg)
	}

	// Handle the artifact
	asset, err := assetOpen(kind, &artifact)
	if err != nil {
		return err
	}
	defer asset.ReadCloser.Close() // nolint: errcheck

	// The target url needs to contain the artifact name
	if !strings.HasSuffix(targetURL, "/") {
		targetURL += "/"
	}
	targetURL += artifact.Name

	var headers = map[string]string{}
	if put.ChecksumHeader != "" {
		sum, err := artifact.Checksum("sha256")
		if err != nil {
			return err
		}
		headers[put.ChecksumHeader] = sum
	}

	_, err = uploadAssetToServer(ctx, put, targetURL, username, secret, headers, asset, check)
	if err != nil {
		msg := fmt.Sprintf("%s: upload failed", kind)
		log.WithError(err).WithFields(log.Fields{
			"instance": put.Name,
			"username": username,
		}).Error(msg)
		return errors.Wrap(err, msg)
	}

	log.WithFields(log.Fields{
		"instance": put.Name,
		"mode":     put.Mode,
	}).Info("uploaded successful")

	return nil
}

// uploadAssetToServer uploads the asset file to target
func uploadAssetToServer(ctx *context.Context, put *config.Put, target, username, secret string, headers map[string]string, a *asset, check ResponseChecker) (*h.Response, error) {
	req, err := newUploadRequest(target, username, secret, headers, a)
	if err != nil {
		return nil, err
	}

	return executeHTTPRequest(ctx, put, req, check)
}

// newUploadRequest creates a new h.Request for uploading
func newUploadRequest(target, username, secret string, headers map[string]string, a *asset) (*h.Request, error) {
	req, err := h.NewRequest(h.MethodPut, target, a.ReadCloser)
	if err != nil {
		return nil, err
	}
	req.ContentLength = a.Size
	req.SetBasicAuth(username, secret)

	for k, v := range headers {
		req.Header.Add(k, v)
	}

	return req, err
}

func getHTTPClient(put *config.Put) (*h.Client, error) {
	if put.TrustedCerts == "" {
		return h.DefaultClient, nil
	}
	pool, err := x509.SystemCertPool()
	if err != nil {
		if runtime.GOOS == "windows" {
			// on windows ignore errors until golang issues #16736 & #18609 get fixed
			pool = x509.NewCertPool()
		} else {
			return nil, err
		}
	}
	pool.AppendCertsFromPEM([]byte(put.TrustedCerts)) // already validated certs checked by CheckConfig
	return &h.Client{
		Transport: &h.Transport{
			TLSClientConfig: &tls.Config{
				RootCAs: pool,
			},
		},
	}, nil
}

// executeHTTPRequest processes the http call with respect of context ctx
func executeHTTPRequest(ctx *context.Context, put *config.Put, req *h.Request, check ResponseChecker) (*h.Response, error) {
	client, err := getHTTPClient(put)
	if err != nil {
		return nil, err
	}
	log.Debugf("executing request: %s %s (headers: %v)", req.Method, req.URL, req.Header)
	resp, err := client.Do(req)
	if err != nil {
		// If we got an error, and the context has been canceled,
		// the context's error is probably more useful.
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
		}
		return nil, err
	}

	defer resp.Body.Close() // nolint: errcheck

	err = check(resp)
	if err != nil {
		// even though there was an error, we still return the response
		// in case the caller wants to inspect it further
		return resp, err
	}

	return resp, err
}

// targetData is used as a template struct for
// Artifactory.Target
type targetData struct {
	Version     string
	Tag         string
	ProjectName string

	// Only supported in mode binary
	Os   string
	Arch string
	Arm  string
}

// resolveTargetTemplate returns the resolved target template with replaced variables
// Those variables can be replaced by the given context, goos, goarch, goarm and more
// TODO: replace this with our internal template pkg
func resolveTargetTemplate(ctx *context.Context, put *config.Put, artifact artifact.Artifact) (string, error) {
	data := targetData{
		Version:     ctx.Version,
		Tag:         ctx.Git.CurrentTag,
		ProjectName: ctx.Config.ProjectName,
	}

	if put.Mode == ModeBinary {
		// TODO: multiple archives here
		data.Os = replace(ctx.Config.Archive.Replacements, artifact.Goos)
		data.Arch = replace(ctx.Config.Archive.Replacements, artifact.Goarch)
		data.Arm = replace(ctx.Config.Archive.Replacements, artifact.Goarm)
	}

	var out bytes.Buffer
	t, err := template.New(ctx.Config.ProjectName).Parse(put.Target)
	if err != nil {
		return "", err
	}
	err = t.Execute(&out, data)
	return out.String(), err
}

func replace(replacements map[string]string, original string) string {
	result := replacements[original]
	if result == "" {
		return original
	}
	return result
}
<|endoftext|>