<commit_before>package memory

import (
	"sort"
	"sync/atomic"
	"unsafe"

	"github.com/grafana/metrictank/stats"

	"github.com/grafana/metrictank/errors"
	"github.com/grafana/metrictank/expr/tagquery"
	"github.com/grafana/metrictank/schema"
	lru "github.com/hashicorp/golang-lru"
)

var (
	// the collision avoidance window defines how many times we try to find a higher
	// slot that's free if two record hashes collide
	collisionAvoidanceWindow = uint32(1024)

	// metric idx.memory.meta-tags.enrichment-cache.ops.hit is a counter of enrichment cache hits
	enrichmentCacheHits = stats.NewCounter32("idx.memory.meta-tags.enrichment-cache.ops.hit")
	// metric idx.memory.meta-tags.enrichment-cache.ops.miss is a counter of enrichment cache misses
	enrichmentCacheMisses = stats.NewCounter32("idx.memory.meta-tags.enrichment-cache.ops.miss")
	// metric idx.memory.meta-tags.enrichment-cache.entries is a the number of entries in the enrichment cache
	enrichmentCacheEntries = stats.NewGauge32("idx.memory.meta-tags.enrichment-cache.entries")
)

type recordId uint32

// list of meta records keyed by a unique identifier used as ID
type metaTagRecords struct {
	records  map[recordId]tagquery.MetaTagRecord
	enricher unsafe.Pointer
}

func newMetaTagRecords() *metaTagRecords {
	return &metaTagRecords{
		records: make(map[recordId]tagquery.MetaTagRecord),
	}
}

// upsert inserts or updates a meta tag record according to the given specifications
// it uses the set of tag query expressions as the identity of the record, if a record with the
// same identity is already present then its meta tags get updated to the specified ones.
// If the new record contains no meta tags, then the update is equivalent to a delete.
// Those are the return values:
// 1) The id at which the new record got inserted
// 2) Pointer to the inserted metaTagRecord
// 3) The id of the record that has been replaced if an update was performed
// 4) Pointer to the metaTagRecord that has been replaced if an update was performed, otherwise nil
// 5) Error if an error occurred, otherwise it's nil
func (m *metaTagRecords) upsert(record tagquery.MetaTagRecord) (recordId, *tagquery.MetaTagRecord, recordId, *tagquery.MetaTagRecord, error) {
	// after altering meta records we need to reinstantiate the enricher the next time we want to use it
	defer atomic.StorePointer(&m.enricher, nil)

	record.Expressions.Sort()

	id := recordId(record.HashExpressions())

	var oldRecord *tagquery.MetaTagRecord
	var oldId recordId

	// loop over existing records, starting from id, trying to find one that has
	// the exact same queries as the one we're upserting
	for i := uint32(0); i < collisionAvoidanceWindow; i++ {
		if existingRecord, ok := m.records[id+recordId(i)]; ok {
			if record.EqualExpressions(&existingRecord) {
				oldRecord = &existingRecord
				oldId = id + recordId(i)
				delete(m.records, oldId)
				break
			}
		}
	}

	if !record.HasMetaTags() {
		return 0, &record, oldId, oldRecord, nil
	}

	record.MetaTags.Sort()

	// now find the best position to insert the new/updated record, starting from id
	for i := uint32(0); i < collisionAvoidanceWindow; i++ {
		// if we find a free slot, then insert the new record there
		if _, ok := m.records[id]; !ok {
			m.records[id] = record

			return id, &record, oldId, oldRecord, nil
		}

		id++
	}

	return 0, nil, 0, nil, errors.NewInternal("Could not find a free ID to insert record")
}

func (m *metaTagRecords) getEnricher(lookup tagquery.IdTagLookup) *enricher {
	res := (*enricher)(atomic.LoadPointer(&m.enricher))
	if res != nil {
		return res
	}

	// if no enricher is present yet, then we instantiate one and store its reference
	// to reuse it later.
	// there is an unlikely race condition where we receive multiple calls to FindByTag
	// from multiple requests at the same time, which would result in multiple enrichers
	// getting instantiated. in such a case the last one would overwrite the previous
	// ones and there shouldn't be any issues. this case is very unlikely, so it doesn't
	// seem to be worth it to add a lock for that.

	// if provided size is valid, it's not possible for lru.New to return an error
	cache, _ := lru.New(enrichmentCacheSize)
	res = &enricher{
		filters: make([]tagquery.MetricDefinitionFilter, len(m.records)),
		tags:    make([]tagquery.Tags, len(m.records)),
		cache:   cache,
	}

	i := 0
	for _, record := range m.records {
		res.filters[i] = record.GetMetricDefinitionFilter(lookup)
		res.tags[i] = record.MetaTags
		i++
	}

	// to atomically store a pointer, unfortunately we need to use unsafe.Pointer
	atomic.StorePointer(&m.enricher, (unsafe.Pointer)(res))

	return res
}

func (m *metaTagRecords) hashRecords() uint32 {
	i := 0
	recordIds := make([]recordId, len(m.records))
	for recordId := range m.records {
		recordIds[i] = recordId
		i++
	}

	sort.Slice(recordIds, func(i, j int) bool {
		return recordIds[i] < recordIds[j]
	})

	h := tagquery.QueryHash()
	var record tagquery.MetaTagRecord
	var recordHash uint64
	for _, recordId := range recordIds {
		record = m.records[recordId]
		recordHash = record.HashRecord()

		h.Write([]byte{
			byte(recordHash),
			byte(recordHash >> 8),
			byte(recordHash >> 16),
			byte(recordHash >> 24),
			byte(recordHash >> 32),
			byte(recordHash >> 40),
			byte(recordHash >> 48),
			byte(recordHash >> 56),
		})
	}

	return h.Sum32()
}

type enricher struct {
	filters []tagquery.MetricDefinitionFilter
	tags    []tagquery.Tags
	cache   *lru.Cache
}

func (e *enricher) reportStats() {
	enrichmentCacheEntries.Set(e.cache.Len())
}

// enrich takes a metric id, the name string and its tags as strings and evaluates the
// expressions of all meta records to determine which meta tags need to be associated
// with that given metric.
// it is heavily cached, the cache key is org agnostic because the enricher needs to
// be instantiated once per org.
func (e *enricher) enrich(id schema.MKey, name string, tags []string) tagquery.Tags {
	cachedRes, ok := e.cache.Get(id.Key)
	if ok {
		enrichmentCacheHits.Inc()
		return cachedRes.(tagquery.Tags)
	}
	enrichmentCacheMisses.Inc()

	var res tagquery.Tags
	var matches []int
	for i := range e.filters {
		if e.filters[i](id, name, tags) == tagquery.Pass {
			res = append(res, e.tags[i]...)
			matches = append(matches, i)
		}
	}

	e.cache.Add(id.Key, res)

	return res
}

// index structure keyed by tag -> value -> list of meta record IDs
type metaTagValue map[string][]recordId
type metaTagIndex map[string]metaTagValue

func (m metaTagIndex) deleteRecord(keyValue tagquery.Tag, id recordId) {
	if values, ok := m[keyValue.Key]; ok {
		if ids, ok := values[keyValue.Value]; ok {
			for i := 0; i < len(ids); i++ {
				if ids[i] == id {
					// no need to keep the order
					ids[i] = ids[len(ids)-1]
					values[keyValue.Value] = ids[:len(ids)-1]

					// no id should ever be present more than once
					return
				}
			}
		}
	}
}

func (m metaTagIndex) insertRecord(keyValue tagquery.Tag, id recordId) {
	var values metaTagValue
	var ok bool

	if values, ok = m[keyValue.Key]; !ok {
		values = make(metaTagValue)
		m[keyValue.Key] = values
	}

	values[keyValue.Value] = append(values[keyValue.Value], id)
}

// getMetaRecordIdsByExpression takes an expression and a bool, it returns all meta record
// ids of the records which match it. the bool indicates whether the result set should be
// inverted. In some cases inverting the result set can minimize its size.
// Minimizing the size of the returned result set will lead to a faster expression evaluation,
// because less meta records will need to be checked against a given MetricDefinition.
// The caller, after receiving the result set, needs to be aware of whether the result set
// is inverted and interpret it accordingly.
func (m metaTagIndex) getMetaRecordIdsByExpression(expr tagquery.Expression, invertSetOfMetaRecords bool) []recordId {
	if expr.OperatesOnTag() {
		return m.getByTag(expr, invertSetOfMetaRecords)
	}
	return m.getByTagValue(expr, invertSetOfMetaRecords)
}

func (m metaTagIndex) getByTag(expr tagquery.Expression, invertSetOfMetaRecords bool) []recordId {
	var res []recordId

	for key := range m {

		if invertSetOfMetaRecords {
			if expr.Matches(key) {
				continue
			}
		} else {
			if !expr.Matches(key) {
				continue
			}
		}

		for _, ids := range m[key] {
			res = append(res, ids...)
		}
	}

	return res
}

func (m metaTagIndex) getByTagValue(expr tagquery.Expression, invertSetOfMetaRecords bool) []recordId {
	if expr.MatchesExactly() {
		return m[expr.GetKey()][expr.GetValue()]
	}

	var res []recordId
	for value, ids := range m[expr.GetKey()] {
		passes := expr.Matches(value)

		if invertSetOfMetaRecords {
			passes = !passes
		}

		if !passes {
			continue
		}

		res = append(res, ids...)
	}

	return res
}
<commit_msg>remove dead code<commit_after>package memory

import (
	"sort"
	"sync/atomic"
	"unsafe"

	"github.com/grafana/metrictank/stats"

	"github.com/grafana/metrictank/errors"
	"github.com/grafana/metrictank/expr/tagquery"
	"github.com/grafana/metrictank/schema"
	lru "github.com/hashicorp/golang-lru"
)

var (
	// the collision avoidance window defines how many times we try to find a higher
	// slot that's free if two record hashes collide
	collisionAvoidanceWindow = uint32(1024)

	// metric idx.memory.meta-tags.enrichment-cache.ops.hit is a counter of enrichment cache hits
	enrichmentCacheHits = stats.NewCounter32("idx.memory.meta-tags.enrichment-cache.ops.hit")
	// metric idx.memory.meta-tags.enrichment-cache.ops.miss is a counter of enrichment cache misses
	enrichmentCacheMisses = stats.NewCounter32("idx.memory.meta-tags.enrichment-cache.ops.miss")
	// metric idx.memory.meta-tags.enrichment-cache.entries is a the number of entries in the enrichment cache
	enrichmentCacheEntries = stats.NewGauge32("idx.memory.meta-tags.enrichment-cache.entries")
)

type recordId uint32

// list of meta records keyed by a unique identifier used as ID
type metaTagRecords struct {
	records  map[recordId]tagquery.MetaTagRecord
	enricher unsafe.Pointer
}

func newMetaTagRecords() *metaTagRecords {
	return &metaTagRecords{
		records: make(map[recordId]tagquery.MetaTagRecord),
	}
}

// upsert inserts or updates a meta tag record according to the given specifications
// it uses the set of tag query expressions as the identity of the record, if a record with the
// same identity is already present then its meta tags get updated to the specified ones.
// If the new record contains no meta tags, then the update is equivalent to a delete.
// Those are the return values:
// 1) The id at which the new record got inserted
// 2) Pointer to the inserted metaTagRecord
// 3) The id of the record that has been replaced if an update was performed
// 4) Pointer to the metaTagRecord that has been replaced if an update was performed, otherwise nil
// 5) Error if an error occurred, otherwise it's nil
func (m *metaTagRecords) upsert(record tagquery.MetaTagRecord) (recordId, *tagquery.MetaTagRecord, recordId, *tagquery.MetaTagRecord, error) {
	// after altering meta records we need to reinstantiate the enricher the next time we want to use it
	defer atomic.StorePointer(&m.enricher, nil)

	record.Expressions.Sort()

	id := recordId(record.HashExpressions())

	var oldRecord *tagquery.MetaTagRecord
	var oldId recordId

	// loop over existing records, starting from id, trying to find one that has
	// the exact same queries as the one we're upserting
	for i := uint32(0); i < collisionAvoidanceWindow; i++ {
		if existingRecord, ok := m.records[id+recordId(i)]; ok {
			if record.EqualExpressions(&existingRecord) {
				oldRecord = &existingRecord
				oldId = id + recordId(i)
				delete(m.records, oldId)
				break
			}
		}
	}

	if !record.HasMetaTags() {
		return 0, &record, oldId, oldRecord, nil
	}

	record.MetaTags.Sort()

	// now find the best position to insert the new/updated record, starting from id
	for i := uint32(0); i < collisionAvoidanceWindow; i++ {
		// if we find a free slot, then insert the new record there
		if _, ok := m.records[id]; !ok {
			m.records[id] = record

			return id, &record, oldId, oldRecord, nil
		}

		id++
	}

	return 0, nil, 0, nil, errors.NewInternal("Could not find a free ID to insert record")
}

func (m *metaTagRecords) getEnricher(lookup tagquery.IdTagLookup) *enricher {
	res := (*enricher)(atomic.LoadPointer(&m.enricher))
	if res != nil {
		return res
	}

	// if no enricher is present yet, then we instantiate one and store its reference
	// to reuse it later.
	// there is an unlikely race condition where we receive multiple calls to FindByTag
	// from multiple requests at the same time, which would result in multiple enrichers
	// getting instantiated. in such a case the last one would overwrite the previous
	// ones and there shouldn't be any issues. this case is very unlikely, so it doesn't
	// seem to be worth it to add a lock for that.

	// if provided size is valid, it's not possible for lru.New to return an error
	cache, _ := lru.New(enrichmentCacheSize)
	res = &enricher{
		filters: make([]tagquery.MetricDefinitionFilter, len(m.records)),
		tags:    make([]tagquery.Tags, len(m.records)),
		cache:   cache,
	}

	i := 0
	for _, record := range m.records {
		res.filters[i] = record.GetMetricDefinitionFilter(lookup)
		res.tags[i] = record.MetaTags
		i++
	}

	// to atomically store a pointer, unfortunately we need to use unsafe.Pointer
	atomic.StorePointer(&m.enricher, (unsafe.Pointer)(res))

	return res
}

func (m *metaTagRecords) hashRecords() uint32 {
	i := 0
	recordIds := make([]recordId, len(m.records))
	for recordId := range m.records {
		recordIds[i] = recordId
		i++
	}

	sort.Slice(recordIds, func(i, j int) bool {
		return recordIds[i] < recordIds[j]
	})

	h := tagquery.QueryHash()
	var record tagquery.MetaTagRecord
	var recordHash uint64
	for _, recordId := range recordIds {
		record = m.records[recordId]
		recordHash = record.HashRecord()

		h.Write([]byte{
			byte(recordHash),
			byte(recordHash >> 8),
			byte(recordHash >> 16),
			byte(recordHash >> 24),
			byte(recordHash >> 32),
			byte(recordHash >> 40),
			byte(recordHash >> 48),
			byte(recordHash >> 56),
		})
	}

	return h.Sum32()
}

type enricher struct {
	filters []tagquery.MetricDefinitionFilter
	tags    []tagquery.Tags
	cache   *lru.Cache
}

func (e *enricher) reportStats() {
	enrichmentCacheEntries.Set(e.cache.Len())
}

// enrich takes a metric id, the name string and its tags as strings and evaluates the
// expressions of all meta records to determine which meta tags need to be associated
// with that given metric.
// it is heavily cached, the cache key is org agnostic because the enricher needs to
// be instantiated once per org.
func (e *enricher) enrich(id schema.MKey, name string, tags []string) tagquery.Tags {
	cachedRes, ok := e.cache.Get(id.Key)
	if ok {
		enrichmentCacheHits.Inc()
		return cachedRes.(tagquery.Tags)
	}
	enrichmentCacheMisses.Inc()

	var res tagquery.Tags
	for i := range e.filters {
		if e.filters[i](id, name, tags) == tagquery.Pass {
			res = append(res, e.tags[i]...)
		}
	}

	e.cache.Add(id.Key, res)

	return res
}

// index structure keyed by tag -> value -> list of meta record IDs
type metaTagValue map[string][]recordId
type metaTagIndex map[string]metaTagValue

func (m metaTagIndex) deleteRecord(keyValue tagquery.Tag, id recordId) {
	if values, ok := m[keyValue.Key]; ok {
		if ids, ok := values[keyValue.Value]; ok {
			for i := 0; i < len(ids); i++ {
				if ids[i] == id {
					// no need to keep the order
					ids[i] = ids[len(ids)-1]
					values[keyValue.Value] = ids[:len(ids)-1]

					// no id should ever be present more than once
					return
				}
			}
		}
	}
}

func (m metaTagIndex) insertRecord(keyValue tagquery.Tag, id recordId) {
	var values metaTagValue
	var ok bool

	if values, ok = m[keyValue.Key]; !ok {
		values = make(metaTagValue)
		m[keyValue.Key] = values
	}

	values[keyValue.Value] = append(values[keyValue.Value], id)
}

// getMetaRecordIdsByExpression takes an expression and a bool, it returns all meta record
// ids of the records which match it. the bool indicates whether the result set should be
// inverted. In some cases inverting the result set can minimize its size.
// Minimizing the size of the returned result set will lead to a faster expression evaluation,
// because less meta records will need to be checked against a given MetricDefinition.
// The caller, after receiving the result set, needs to be aware of whether the result set
// is inverted and interpret it accordingly.
func (m metaTagIndex) getMetaRecordIdsByExpression(expr tagquery.Expression, invertSetOfMetaRecords bool) []recordId {
	if expr.OperatesOnTag() {
		return m.getByTag(expr, invertSetOfMetaRecords)
	}
	return m.getByTagValue(expr, invertSetOfMetaRecords)
}

func (m metaTagIndex) getByTag(expr tagquery.Expression, invertSetOfMetaRecords bool) []recordId {
	var res []recordId

	for key := range m {

		if invertSetOfMetaRecords {
			if expr.Matches(key) {
				continue
			}
		} else {
			if !expr.Matches(key) {
				continue
			}
		}

		for _, ids := range m[key] {
			res = append(res, ids...)
		}
	}

	return res
}

func (m metaTagIndex) getByTagValue(expr tagquery.Expression, invertSetOfMetaRecords bool) []recordId {
	if expr.MatchesExactly() {
		return m[expr.GetKey()][expr.GetValue()]
	}

	var res []recordId
	for value, ids := range m[expr.GetKey()] {
		passes := expr.Matches(value)

		if invertSetOfMetaRecords {
			passes = !passes
		}

		if !passes {
			continue
		}

		res = append(res, ids...)
	}

	return res
}
<|endoftext|>


<commit_before>// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blob

import (
	"bufio"
	"io"
	"os"
	"path"
)

type Manager struct {
	blobPath string
}

func New(blobPath string) *Manager {
	return &Manager{blobPath: blobPath}
}

func (f *Manager) Save(id string, checksum string, rc io.ReadCloser) error {
	// TODO(burcud): Remove old versions of the same file
	file, err := os.OpenFile(f.getBlobPath(id, checksum), os.O_CREATE|os.O_RDWR, 0750)
	if file == nil && err != nil {
		return err
	}
	defer file.Close()

	reader := bufio.NewReader(rc)
	writer := bufio.NewWriter(file)
	p := make([]byte, 4096)
	for {
		n, err := reader.Read(p)
		if err == io.EOF {
			break
		}
		_, err = writer.Write(p[:n])
		if err != nil {
			return err
		}
	}
	return nil
}

func (f *Manager) Read(id string, checksum string, seek int64, l int) (blob []byte, size int64, err error) {
	var file *os.File
	file, err = os.Open(f.getBlobPath(id, checksum))
	if err != nil {
		return
	}
	blob = make([]byte, l)
	file.Seek(seek, 0)
	var s int
	s, err = file.Read(blob)
	return blob, int64(s), err
}

func (f *Manager) getBlobPath(id string, checksum string) string {
	// TODO: shard the files, fs perf issue here
	return path.Join(f.blobPath, id+"=="+checksum)
}
<commit_msg>Clean up old blob revisions before saving the new blob.<commit_after>// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blob

import (
	"bufio"
	"io"
	"io/ioutil"
	"logger"
	"os"
	"path"
	"strings"
)

type Manager struct {
	blobPath string
}

func New(blobPath string) *Manager {
	return &Manager{blobPath: blobPath}
}

func (f *Manager) Save(id string, checksum string, rc io.ReadCloser) error {
	f.cleanup(id, checksum)
	file, err := os.OpenFile(f.getBlobPath(id, checksum), os.O_CREATE|os.O_RDWR, 0750)
	if file == nil && err != nil {
		return err
	}
	defer file.Close()

	reader := bufio.NewReader(rc)
	writer := bufio.NewWriter(file)
	p := make([]byte, 4096)
	for {
		n, err := reader.Read(p)
		if err == io.EOF {
			break
		}
		_, err = writer.Write(p[:n])
		if err != nil {
			return err
		}
	}
	return nil
}

func (f *Manager) Read(id string, checksum string, seek int64, l int) (blob []byte, size int64, err error) {
	var file *os.File
	file, err = os.Open(f.getBlobPath(id, checksum))
	if err != nil {
		return
	}
	blob = make([]byte, l)
	file.Seek(seek, 0)
	var s int
	s, err = file.Read(blob)
	return blob, int64(s), err
}

func (f *Manager) cleanup(id string, checksum string) (err error) {
	var blobs []os.FileInfo
	if blobs, err = ioutil.ReadDir(f.getBlobDir(id)); err != nil {
		return
	}
	for _, file := range blobs {
		if file.Name() != f.getBlobName(id, checksum) && strings.Contains(file.Name(), f.getBlobName(id, "")) {
			logger.V("Deleting", file.Name())
			// errors are not show stoppers here, they will cost additional disk space
			// we can get rid of on the next removal try.
			if rmErr := os.Remove(path.Join(f.getBlobDir(id), file.Name())); rmErr != nil {
				logger.V(rmErr)
			}
		}
	}
	return nil
}

func (f *Manager) getBlobDir(id string) string {
	// TODO: shard the files, fs perf issue here
	return f.blobPath
}

func (f *Manager) getBlobName(id string, checksum string) string {
	return id + "==" + checksum
}

func (f *Manager) getBlobPath(id string, checksum string) string {
	return path.Join(f.getBlobDir(id), f.getBlobName(id, checksum))
}
<|endoftext|>


<commit_before>// Package replicat is a server for n way synchronization of content (rsync for the cloud).
// More information at: http://replic.at
// Copyright 2016 Jacob Taylor jacob@ablox.io
// License: Apache2 - http://www.apache.org/licenses/LICENSE-2.0
package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/goji/httpauth"
	"io"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"sync"
	"time"
)

// ReplicatServer is a structure that contains the definition of the servers in a cluster. Each node has a name and this
// node (as determined by globalSettings.name at the moment) also has a StorageTracker interface.
type ReplicatServer struct {
	ClusterKey    string
	Name          string
	Address       string
	Status        string
	CurrentState  DirTreeMap
	PreviousState DirTreeMap
	Lock          sync.Mutex
	storage       StorageTracker
}

// GetStatus - get the current status of the server
func (server *ReplicatServer) GetStatus() string {
	return server.Status
}

// SetStatus - set the current status of the server
func (server *ReplicatServer) SetStatus(status string) {
	// adding an inline check to make sure the server is valid
	// if the server is being shut down or at the end of a unit test)
	// the assignment can crash without this check
	if server != nil {
		server.Status = status
		sendConfigToServer()
	}
}

const (
	// REPLICAT_STATUS_INITIAL_SCAN - The server is scanning its local storage
	REPLICAT_STATUS_INITIAL_SCAN = "Initial Scan"
	// REPLICAT_STATUS_JOINING_CLUSTER - The server has started to join the cluster and get up to date
	REPLICAT_STATUS_JOINING_CLUSTER = "Joining Cluster"
	// REPLICAT_STATUS_ONLINE - The server is up to date and part of the cluster
	REPLICAT_STATUS_ONLINE = "Online"
)

var serverMap = make(map[string]*ReplicatServer)
var serverMapLock sync.RWMutex

var lastConfigPing = time.Now()

// BootstrapAndServe - Start the server
func BootstrapAndServe(address string) {
	//trackerTestDual()
	//trackerTestSmallFileInSubfolder()
	//trackerTestEmptyDirectoryMovesInOutAround()
	//trackerTestFileChangeTrackerAddFolders()
	//trackerTestSmallFileCreationAndRename()
	//trackerTestSmallFileCreationAndUpdate()
	//trackerTestSmallFileMovesInOutAround()
	//trackerTestDirectoryCreation()
	//trackerTestNestedDirectoryCreation()
	//trackerTestDirectoryStorage()
	//trackerTestFileChangeTrackerAutoCreateFolderAndCleanup()
	//trackerTestNestedFastDirectoryCreation()

	// testing code to enable debugger use
	http.Handle("/event/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(eventHandler)))
	http.Handle("/tree/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(folderTreeHandler)))
	http.Handle("/config/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(configHandler)))
	http.Handle("/upload/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(uploadHandler)))

	lsnr, err := net.Listen("tcp4", address)
	if err != nil {
		panic(fmt.Sprintf("Error listening: %v\nAddress: %s\n", err, address))
	}
	fmt.Println("Listening on:", lsnr.Addr().String())

	logOnlyHandler := LogOnlyChangeHandler{}
	tracker := FilesystemTracker{}
	fmt.Printf("Looking up settings for node: %s\n", globalSettings.Name)

	directory := globalSettings.Directory

	fmt.Printf("GlobalSettings directory retrieved for this node: %s\n", directory)
	server := &ReplicatServer{Name: globalSettings.Name, ClusterKey: globalSettings.ClusterKey, Address: lsnr.Addr().String(), storage: &tracker, Status: REPLICAT_STATUS_INITIAL_SCAN}
	serverMap[globalSettings.Name] = server
	tracker.init(directory, server)

	go func(tracker StorageTracker) {
		for true {
			tracker.GetStatistics()
			time.Sleep(30 * time.Second)
		}
	}(&tracker)

	var c ChangeHandler
	c = &logOnlyHandler
	tracker.watchDirectory(&c)

	go func(listener net.Listener) {
		err = http.Serve(listener, nil)
		if err != nil {
			panic(err)
		}
	}(lsnr)

	fmt.Println("Starting config update processor")
	go configUpdateProcessor(configUpdateChannel)
	go keepConfigCurrent()

	if globalSettings.ManagerAddress != "" {
		fmt.Printf("about to send config to server (%s)\nOur address is: (%s)", globalSettings.ManagerAddress, lsnr.Addr())
	}
}

const (
	managerOverdueSeconds = 30
	managerCheckSleepTime = 45
)

func keepConfigCurrent() {
	for {
		if time.Since(lastConfigPing) > managerOverdueSeconds*time.Second {
			log.Printf("Manager Contact Overdue, attempting to contact: %s\n", globalSettings.ManagerAddress)
			sendConfigToServer()
		} else {
			fmt.Println("No Update Required")
		}
		time.Sleep((rand.Intn(10) - 5 + managerCheckSleepTime) * time.Second)
	}
}

func sendConfigToServer() {
	// This field will be empty during testing
	if globalSettings.ManagerAddress == "" {
		return
	}

	url := "http://" + globalSettings.ManagerAddress + "/config/"
	fmt.Printf("Manager location: %s\n", url)

	jsonStr, _ := json.Marshal(serverMap[globalSettings.Name])
	fmt.Printf("jsonStr: %s\n", jsonStr)

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
	req.Header.Set("Content-Type", "application/json")

	data := []byte(globalSettings.ManagerCredentials)
	authHash := base64.StdEncoding.EncodeToString(data)
	req.Header.Add("Authorization", "Basic "+authHash)

	client := &http.Client{}
	_, err = client.Do(req)
	if err != nil {
		panic(err)
	}
}

func uploadHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		r.ParseMultipartForm(32 << 20)
		file, handler, err := r.FormFile("uploadfile")
		if err != nil {
			fmt.Println(err)
			return
		}
		defer file.Close()
		fmt.Fprint(w, handler.Header)

		hash := []byte(r.Form.Get("HASH"))

		storage := serverMap[globalSettings.Name].storage
		local, _ := storage.getEntryJSON(handler.Filename)

		if !bytes.Equal(hash, local.Hash) {
			fullPath := globalSettings.Directory + "/" + handler.Filename
			f, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE, 0666)

			if err != nil {
				fmt.Println(err)
				return
			}

			bytesWritten, err := io.Copy(f, file)
			if err != nil {
				log.Printf("Error copying file: %s, error(%#v)\n", handler.Filename, err)
				w.WriteHeader(http.StatusInternalServerError)
				w.Write([]byte("500 - Error copying file"))
				f.Close()
				return
			}
			fmt.Printf("Wrote out (%s) bytes (%d)\n", handler.Filename, bytesWritten)
			f.Close()

			// Get the old entry
			entryString := r.Form.Get("entryJSON")
			if entryString != "" {
				var entry EntryJSON
				err := json.Unmarshal([]byte(entryString), &entry)
				if err != nil {
					log.Fatalf("Error copying file (Entry handling): %s, error(%#v)\n", handler.Filename, err)
					w.WriteHeader(http.StatusInternalServerError)
					w.Write([]byte("500 - Error copying file"))
					return
				}

				err = os.Chtimes(fullPath, time.Now(), entry.ModTime)
				if err != nil {
					log.Fatalf("Error copying file (Changing times): %s, error(%#v)\n", handler.Filename, err)
					w.WriteHeader(http.StatusInternalServerError)
					w.Write([]byte("500 - Error copying file"))
					return
				}
			}

		}
	}
}

var configUpdateChannel = make(chan *map[string]*ReplicatServer, 100)

func configHandler(_ http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "POST":
		lastConfigPing = time.Now()
		fmt.Println("configHandler: 3")
		decoder := json.NewDecoder(r.Body)
		fmt.Println("configHandler: 4")
		var newServerMap map[string]*ReplicatServer
		err := decoder.Decode(&newServerMap)
		fmt.Println("configHandler: 5")
		if err != nil {
			fmt.Println(err)
		}

		fmt.Println("configHandler: 6")
		configUpdateChannel <- &newServerMap
		fmt.Println("configHandler: 8")
	}
}

func configUpdateProcessor(c chan *map[string]*ReplicatServer) {
	for {
		newServerMap := <-c
		serverMapLock.Lock()

		sendData := false

		// find any nodes that have been deleted
		for name, serverData := range serverMap {
			newServerData, exists := (*newServerMap)[name]
			if !exists {
				fmt.Printf("No longer found config for: %s deleting\n", name)
				delete(serverMap, name)
				continue
			}

			if serverData.Address != newServerData.Address || serverData.Name != newServerData.Name || serverData.ClusterKey != newServerData.ClusterKey || serverData.Status != serverData.Status {
				fmt.Printf("Server data is changed. Replacing.\nold: %v\nnew: %v\n", &serverData, &newServerData)
				if serverData.Status != REPLICAT_STATUS_JOINING_CLUSTER && newServerData.Status == REPLICAT_STATUS_JOINING_CLUSTER {
					fmt.Printf("Decided to send data to: %s\n", serverData.Name)
					sendData = true
				}
				serverMap[name] = newServerData
				fmt.Println("Server data replaced with new server data")
			}
		}

		// find any new nodes
		for name, newServerData := range *newServerMap {
			_, exists := serverMap[name]
			if !exists {
				fmt.Printf("New server configuration for %s: %v\n", name, newServerData)

				// If this server map is for ourselves, build a list of folder if needed and notify others
				if name == globalSettings.Name {
					listOfFileInfo, err := scanDirectoryContents()
					if err != nil {
						log.Fatal(err)
					}
					newServerData.CurrentState = listOfFileInfo
					// Tell all of our friends that we exist and our current state for them to compare against.
					go func(tree DirTreeMap) {
						sendFolderTree(tree)
					}(listOfFileInfo)
				} else {
					fmt.Printf("New Node Decided to send data to: %s\n", name)
					sendData = true
				}

				fmt.Printf("New server configuration provided. Copying: %s\n", name)
				serverMap[name] = newServerData
			}
		}

		if sendData {
			server := serverMap[globalSettings.Name]
			fmt.Println("about to send existing files")
			server.storage.SendCatalog()
			fmt.Println("done sending existing files")
		}

		serverMapLock.Unlock()
	}
}
<commit_msg>adding more debugging output on a sendConfigToServer to figure out what the error I am encountering is (when the manager goes down at the wrong time). Changing duration of sleep message to be type cast<commit_after>// Package replicat is a server for n way synchronization of content (rsync for the cloud).
// More information at: http://replic.at
// Copyright 2016 Jacob Taylor jacob@ablox.io
// License: Apache2 - http://www.apache.org/licenses/LICENSE-2.0
package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/goji/httpauth"
	"io"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"sync"
	"time"
)

// ReplicatServer is a structure that contains the definition of the servers in a cluster. Each node has a name and this
// node (as determined by globalSettings.name at the moment) also has a StorageTracker interface.
type ReplicatServer struct {
	ClusterKey    string
	Name          string
	Address       string
	Status        string
	CurrentState  DirTreeMap
	PreviousState DirTreeMap
	Lock          sync.Mutex
	storage       StorageTracker
}

// GetStatus - get the current status of the server
func (server *ReplicatServer) GetStatus() string {
	return server.Status
}

// SetStatus - set the current status of the server
func (server *ReplicatServer) SetStatus(status string) {
	// adding an inline check to make sure the server is valid
	// if the server is being shut down or at the end of a unit test)
	// the assignment can crash without this check
	if server != nil {
		server.Status = status
		sendConfigToServer()
	}
}

const (
	// REPLICAT_STATUS_INITIAL_SCAN - The server is scanning its local storage
	REPLICAT_STATUS_INITIAL_SCAN = "Initial Scan"
	// REPLICAT_STATUS_JOINING_CLUSTER - The server has started to join the cluster and get up to date
	REPLICAT_STATUS_JOINING_CLUSTER = "Joining Cluster"
	// REPLICAT_STATUS_ONLINE - The server is up to date and part of the cluster
	REPLICAT_STATUS_ONLINE = "Online"
)

var serverMap = make(map[string]*ReplicatServer)
var serverMapLock sync.RWMutex

var lastConfigPing = time.Now()

// BootstrapAndServe - Start the server
func BootstrapAndServe(address string) {
	//trackerTestDual()
	//trackerTestSmallFileInSubfolder()
	//trackerTestEmptyDirectoryMovesInOutAround()
	//trackerTestFileChangeTrackerAddFolders()
	//trackerTestSmallFileCreationAndRename()
	//trackerTestSmallFileCreationAndUpdate()
	//trackerTestSmallFileMovesInOutAround()
	//trackerTestDirectoryCreation()
	//trackerTestNestedDirectoryCreation()
	//trackerTestDirectoryStorage()
	//trackerTestFileChangeTrackerAutoCreateFolderAndCleanup()
	//trackerTestNestedFastDirectoryCreation()

	// testing code to enable debugger use
	http.Handle("/event/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(eventHandler)))
	http.Handle("/tree/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(folderTreeHandler)))
	http.Handle("/config/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(configHandler)))
	http.Handle("/upload/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(uploadHandler)))

	lsnr, err := net.Listen("tcp4", address)
	if err != nil {
		panic(fmt.Sprintf("Error listening: %v\nAddress: %s\n", err, address))
	}
	fmt.Println("Listening on:", lsnr.Addr().String())

	logOnlyHandler := LogOnlyChangeHandler{}
	tracker := FilesystemTracker{}
	fmt.Printf("Looking up settings for node: %s\n", globalSettings.Name)

	directory := globalSettings.Directory

	fmt.Printf("GlobalSettings directory retrieved for this node: %s\n", directory)
	server := &ReplicatServer{Name: globalSettings.Name, ClusterKey: globalSettings.ClusterKey, Address: lsnr.Addr().String(), storage: &tracker, Status: REPLICAT_STATUS_INITIAL_SCAN}
	serverMap[globalSettings.Name] = server
	tracker.init(directory, server)

	go func(tracker StorageTracker) {
		for true {
			tracker.GetStatistics()
			time.Sleep(30 * time.Second)
		}
	}(&tracker)

	var c ChangeHandler
	c = &logOnlyHandler
	tracker.watchDirectory(&c)

	go func(listener net.Listener) {
		err = http.Serve(listener, nil)
		if err != nil {
			panic(err)
		}
	}(lsnr)

	fmt.Println("Starting config update processor")
	go configUpdateProcessor(configUpdateChannel)
	go keepConfigCurrent()

	if globalSettings.ManagerAddress != "" {
		fmt.Printf("about to send config to server (%s)\nOur address is: (%s)", globalSettings.ManagerAddress, lsnr.Addr())
	}
}

const (
	managerOverdueSeconds = 30
	managerCheckSleepTime = 45
)

func keepConfigCurrent() {
	for {
		if time.Since(lastConfigPing) > managerOverdueSeconds*time.Second {
			log.Printf("Manager Contact Overdue, attempting to contact: %s\n", globalSettings.ManagerAddress)
			sendConfigToServer()
		} else {
			fmt.Println("No Update Required")
		}
		time.Sleep(time.Duration(rand.Intn(10)-5+managerCheckSleepTime) * time.Second)
	}
}

func sendConfigToServer() {
	// This field will be empty during testing
	if globalSettings.ManagerAddress == "" {
		return
	}

	url := "http://" + globalSettings.ManagerAddress + "/config/"
	fmt.Printf("Manager location: %s\n", url)

	jsonStr, _ := json.Marshal(serverMap[globalSettings.Name])
	fmt.Printf("jsonStr: %s\n", jsonStr)

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
	req.Header.Set("Content-Type", "application/json")

	data := []byte(globalSettings.ManagerCredentials)
	authHash := base64.StdEncoding.EncodeToString(data)
	req.Header.Add("Authorization", "Basic "+authHash)

	client := &http.Client{}
	_, err = client.Do(req)
	if err != nil {
		log.Fatalf("Alert, going down in flames: %s\n", err)
		panic(err)
	}
}

func uploadHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		r.ParseMultipartForm(32 << 20)
		file, handler, err := r.FormFile("uploadfile")
		if err != nil {
			fmt.Println(err)
			return
		}
		defer file.Close()
		fmt.Fprint(w, handler.Header)

		hash := []byte(r.Form.Get("HASH"))

		storage := serverMap[globalSettings.Name].storage
		local, _ := storage.getEntryJSON(handler.Filename)

		if !bytes.Equal(hash, local.Hash) {
			fullPath := globalSettings.Directory + "/" + handler.Filename
			f, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE, 0666)

			if err != nil {
				fmt.Println(err)
				return
			}

			bytesWritten, err := io.Copy(f, file)
			if err != nil {
				log.Printf("Error copying file: %s, error(%#v)\n", handler.Filename, err)
				w.WriteHeader(http.StatusInternalServerError)
				w.Write([]byte("500 - Error copying file"))
				f.Close()
				return
			}
			fmt.Printf("Wrote out (%s) bytes (%d)\n", handler.Filename, bytesWritten)
			f.Close()

			// Get the old entry
			entryString := r.Form.Get("entryJSON")
			if entryString != "" {
				var entry EntryJSON
				err := json.Unmarshal([]byte(entryString), &entry)
				if err != nil {
					log.Fatalf("Error copying file (Entry handling): %s, error(%#v)\n", handler.Filename, err)
					w.WriteHeader(http.StatusInternalServerError)
					w.Write([]byte("500 - Error copying file"))
					return
				}

				err = os.Chtimes(fullPath, time.Now(), entry.ModTime)
				if err != nil {
					log.Fatalf("Error copying file (Changing times): %s, error(%#v)\n", handler.Filename, err)
					w.WriteHeader(http.StatusInternalServerError)
					w.Write([]byte("500 - Error copying file"))
					return
				}
			}

		}
	}
}

var configUpdateChannel = make(chan *map[string]*ReplicatServer, 100)

func configHandler(_ http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "POST":
		lastConfigPing = time.Now()
		fmt.Println("configHandler: 3")
		decoder := json.NewDecoder(r.Body)
		fmt.Println("configHandler: 4")
		var newServerMap map[string]*ReplicatServer
		err := decoder.Decode(&newServerMap)
		fmt.Println("configHandler: 5")
		if err != nil {
			fmt.Println(err)
		}

		fmt.Println("configHandler: 6")
		configUpdateChannel <- &newServerMap
		fmt.Println("configHandler: 8")
	}
}

func configUpdateProcessor(c chan *map[string]*ReplicatServer) {
	for {
		newServerMap := <-c
		serverMapLock.Lock()

		sendData := false

		// find any nodes that have been deleted
		for name, serverData := range serverMap {
			newServerData, exists := (*newServerMap)[name]
			if !exists {
				fmt.Printf("No longer found config for: %s deleting\n", name)
				delete(serverMap, name)
				continue
			}

			if serverData.Address != newServerData.Address || serverData.Name != newServerData.Name || serverData.ClusterKey != newServerData.ClusterKey || serverData.Status != serverData.Status {
				fmt.Printf("Server data is changed. Replacing.\nold: %v\nnew: %v\n", &serverData, &newServerData)
				if serverData.Status != REPLICAT_STATUS_JOINING_CLUSTER && newServerData.Status == REPLICAT_STATUS_JOINING_CLUSTER {
					fmt.Printf("Decided to send data to: %s\n", serverData.Name)
					sendData = true
				}
				serverMap[name] = newServerData
				fmt.Println("Server data replaced with new server data")
			}
		}

		// find any new nodes
		for name, newServerData := range *newServerMap {
			_, exists := serverMap[name]
			if !exists {
				fmt.Printf("New server configuration for %s: %v\n", name, newServerData)

				// If this server map is for ourselves, build a list of folder if needed and notify others
				if name == globalSettings.Name {
					listOfFileInfo, err := scanDirectoryContents()
					if err != nil {
						log.Fatal(err)
					}
					newServerData.CurrentState = listOfFileInfo
					// Tell all of our friends that we exist and our current state for them to compare against.
					go func(tree DirTreeMap) {
						sendFolderTree(tree)
					}(listOfFileInfo)
				} else {
					fmt.Printf("New Node Decided to send data to: %s\n", name)
					sendData = true
				}

				fmt.Printf("New server configuration provided. Copying: %s\n", name)
				serverMap[name] = newServerData
			}
		}

		if sendData {
			server := serverMap[globalSettings.Name]
			fmt.Println("about to send existing files")
			server.storage.SendCatalog()
			fmt.Println("done sending existing files")
		}

		serverMapLock.Unlock()
	}
}
<|endoftext|>


<commit_before>package imagick

/*
#include <wand/MagickWand.h>
*/
import "C"

type BooleanType int

const(
	BOOLEAN_TYPE_FALSE BooleanType = C.MagickFalse
	BOOLEAN_TYPE_TRUE BooleanType = C.MagickTrue
)<commit_msg>Command shift f<commit_after>package imagick

/*
#include <wand/MagickWand.h>
*/
import "C"

type BooleanType int

const (
	BOOLEAN_TYPE_FALSE BooleanType = C.MagickFalse
	BOOLEAN_TYPE_TRUE  BooleanType = C.MagickTrue
)
<|endoftext|>


<commit_before>package api

import (
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
)

const (
	ResourceKindPostgres = "Postgres"
	ResourceNamePostgres = "postgres"
	ResourceTypePostgres = "postgreses"
)

// Postgres defines a Postgres database.
type Postgres struct {
	unversioned.TypeMeta `json:",inline,omitempty"`
	api.ObjectMeta       `json:"metadata,omitempty"`
	Spec                 PostgresSpec    `json:"spec,omitempty"`
	Status               *PostgresStatus `json:"status,omitempty"`
}

type PostgresSpec struct {
	// Version of Postgres to be deployed.
	Version string `json:"version,omitempty"`
	// Number of instances to deploy for a Postgres database.
	Replicas int32 `json:"replicas,omitempty"`
	// Storage spec to specify how storage shall be used.
	Storage *StorageSpec `json:"storage,omitempty"`
	// ServiceAccountName is the name of the ServiceAccount to use to run the
	// Prometheus Pods.
	ServiceAccountName string `json:"serviceAccountName,omitempty"`
	// Database authentication secret
	DatabaseSecret *api.SecretVolumeSource `json:"databaseSecret,omitempty"`
	// NodeSelector is a selector which must be true for the pod to fit on a node
	// +optional
	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
	// Run initial script when starting Postgres master
	InitialScript *InitialScriptSpec `json:"initialScript,omitempty"`
	// BackupSchedule spec to specify how database backup will be taken
	// +optional
	BackupSchedule *BackupScheduleSpec `json:"backupSchedule,omitempty"`
	// If DoNotDelete is true, controller will prevent to delete this Postgres object.
	// Controller will create same Postgres object and ignore other process.
	// +optional
	DoNotDelete *bool `json:"doNotDelete,omitempty"`
}

type PostgresStatus struct {
	// Total number of non-terminated pods targeted by this Postgres TPR
	Replicas int32 `json:"replicas"`
	// Total number of available pods targeted by this Postgres TPR.
	AvailableReplicas int32 `json:"availableReplicas"`
}

type PostgresList struct {
	unversioned.TypeMeta `json:",inline"`
	unversioned.ListMeta `json:"metadata,omitempty"`
	// Items is a list of Postgres TPR objects
	Items []*Postgres `json:"items,omitempty"`
}
<commit_msg>change type of DoNotDelete from *bool to bool<commit_after>package api

import (
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
)

const (
	ResourceKindPostgres = "Postgres"
	ResourceNamePostgres = "postgres"
	ResourceTypePostgres = "postgreses"
)

// Postgres defines a Postgres database.
type Postgres struct {
	unversioned.TypeMeta `json:",inline,omitempty"`
	api.ObjectMeta       `json:"metadata,omitempty"`
	Spec                 PostgresSpec    `json:"spec,omitempty"`
	Status               *PostgresStatus `json:"status,omitempty"`
}

type PostgresSpec struct {
	// Version of Postgres to be deployed.
	Version string `json:"version,omitempty"`
	// Number of instances to deploy for a Postgres database.
	Replicas int32 `json:"replicas,omitempty"`
	// Storage spec to specify how storage shall be used.
	Storage *StorageSpec `json:"storage,omitempty"`
	// ServiceAccountName is the name of the ServiceAccount to use to run the
	// Prometheus Pods.
	ServiceAccountName string `json:"serviceAccountName,omitempty"`
	// Database authentication secret
	DatabaseSecret *api.SecretVolumeSource `json:"databaseSecret,omitempty"`
	// NodeSelector is a selector which must be true for the pod to fit on a node
	// +optional
	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
	// Run initial script when starting Postgres master
	InitialScript *InitialScriptSpec `json:"initialScript,omitempty"`
	// BackupSchedule spec to specify how database backup will be taken
	// +optional
	BackupSchedule *BackupScheduleSpec `json:"backupSchedule,omitempty"`
	// If DoNotDelete is true, controller will prevent to delete this Postgres object.
	// Controller will create same Postgres object and ignore other process.
	// +optional
	DoNotDelete bool `json:"doNotDelete,omitempty"`
}

type PostgresStatus struct {
	// Total number of non-terminated pods targeted by this Postgres TPR
	Replicas int32 `json:"replicas"`
	// Total number of available pods targeted by this Postgres TPR.
	AvailableReplicas int32 `json:"availableReplicas"`
}

type PostgresList struct {
	unversioned.TypeMeta `json:",inline"`
	unversioned.ListMeta `json:"metadata,omitempty"`
	// Items is a list of Postgres TPR objects
	Items []*Postgres `json:"items,omitempty"`
}
<|endoftext|>


<commit_before>package provider

import (
	. "github.com/smartystreets/goconvey/convey"
	"io/ioutil"
	"os"
	"testing"
)

func TestResolve(t *testing.T) {
	Convey("Passing no provider should return an error", t, func() {
		_, err := Resolve("")

		So(err, ShouldNotBeNil)
	})

	Convey("Passing the provider via CLI should return it without error", t, func() {
		expected := "/usr/bin/myprovider"
		provider, err := Resolve(expected)

		So(provider, ShouldEqual, expected)
		So(err, ShouldBeNil)
	})

	Convey("Setting the provider via environment variable works", t, func() {
		expected := "/opt/providers/custom"
		os.Setenv("CAULDRON_PROVIDER", expected)
		provider, err := Resolve("")
		os.Unsetenv("CAULDRON_PROVIDER")

		So(provider, ShouldEqual, expected)
		So(err, ShouldBeNil)
	})

	Convey("Given a provider path", t, func() {
		tempDir, _ := ioutil.TempDir("", "cauldrontest")
		defer os.RemoveAll(tempDir)
		DefaultPath = tempDir

		Convey("If there is 1 executable, return it as the provider", func() {
			f, err := ioutil.TempFile(DefaultPath, "")
			provider, err := Resolve("")

			So(provider, ShouldEqual, f.Name())
			So(err, ShouldBeNil)
		})

		Convey("If there are > 1 executables, return an error to user", func() {
			// Create 2 exes in provider path
			ioutil.TempFile(DefaultPath, "")
			ioutil.TempFile(DefaultPath, "")
			_, err := Resolve("")

			So(err, ShouldNotBeNil)
		})
	})
}

func TestCall(t *testing.T) {
	Convey("When I call a provider", t, func() {
		Convey("If it returns exit code 0, return stdout", func() {
			arg := "provider.go"
			out, err := Call("ls", arg)

			So(out, ShouldEqual, arg)
			So(err, ShouldBeNil)
		})
		Convey("If it returns exit code > 0, return stderr", func() {
			out, err := Call("ls", "README.notafile")

			So(out, ShouldContainSubstring, "No such file or directory")
			So(err, ShouldNotBeNil)
		})
	})
}
<commit_msg>Isolate provider test from dev env<commit_after>package provider

import (
	. "github.com/smartystreets/goconvey/convey"
	"io/ioutil"
	"os"
	"testing"
)

func TestResolve(t *testing.T) {
	Convey("Passing no provider should return an error", t, func() {
		// Point to a tempdir to avoid pollution from dev env
		tempDir, _ := ioutil.TempDir("", "cauldrontest")
		defer os.RemoveAll(tempDir)
		DefaultPath = tempDir

		_, err := Resolve("")

		So(err, ShouldNotBeNil)
	})

	Convey("Passing the provider via CLI should return it without error", t, func() {
		expected := "/usr/bin/myprovider"
		provider, err := Resolve(expected)

		So(provider, ShouldEqual, expected)
		So(err, ShouldBeNil)
	})

	Convey("Setting the provider via environment variable works", t, func() {
		expected := "/opt/providers/custom"
		os.Setenv("CAULDRON_PROVIDER", expected)
		provider, err := Resolve("")
		os.Unsetenv("CAULDRON_PROVIDER")

		So(provider, ShouldEqual, expected)
		So(err, ShouldBeNil)
	})

	Convey("Given a provider path", t, func() {
		tempDir, _ := ioutil.TempDir("", "cauldrontest")
		defer os.RemoveAll(tempDir)
		DefaultPath = tempDir

		Convey("If there is 1 executable, return it as the provider", func() {
			f, err := ioutil.TempFile(DefaultPath, "")
			provider, err := Resolve("")

			So(provider, ShouldEqual, f.Name())
			So(err, ShouldBeNil)
		})

		Convey("If there are > 1 executables, return an error to user", func() {
			// Create 2 exes in provider path
			ioutil.TempFile(DefaultPath, "")
			ioutil.TempFile(DefaultPath, "")
			_, err := Resolve("")

			So(err, ShouldNotBeNil)
		})
	})
}

func TestCall(t *testing.T) {
	Convey("When I call a provider", t, func() {
		Convey("If it returns exit code 0, return stdout", func() {
			arg := "provider.go"
			out, err := Call("ls", arg)

			So(out, ShouldEqual, arg)
			So(err, ShouldBeNil)
		})
		Convey("If it returns exit code > 0, return stderr", func() {
			out, err := Call("ls", "README.notafile")

			So(out, ShouldContainSubstring, "No such file or directory")
			So(err, ShouldNotBeNil)
		})
	})
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package s3

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"path/filepath"
	"strings"
	"time"

	"github.com/alecthomas/template"
	"github.com/blang/semver"
	"github.com/goamz/goamz/aws"
	"github.com/goamz/goamz/s3"
)

type Section struct {
	Header   string
	Releases []Release
}

type Release struct {
	Name    string
	URL     string
	Version string
	Date    string
	Commit  string
}

func WriteHTML(path string, bucketName string, prefixes string, suffix string) error {
	auth, err := aws.EnvAuth()
	if err != nil {
		return err
	}
	client := s3.New(auth, aws.USEast)
	bucket := client.Bucket(bucketName)
	if bucket == nil {
		return fmt.Errorf("Bucket %s not found", bucketName)
	}

	var sections []Section
	for _, prefix := range strings.Split(prefixes, ",") {
		resp, err := bucket.List(prefix, "", "", 0)
		if err != nil {
			return err
		}

		var releases []Release
		for _, k := range resp.Contents {
			if strings.HasSuffix(k.Key, suffix) {
				key := k.Key
				name := key[len(prefix):]
				urlString := fmt.Sprintf("https://s3.amazonaws.com/%s/%s%s", bucketName, prefix, url.QueryEscape(name))
				version, date, commit, err := parseName(name, prefix)
				if err != nil {
					log.Printf("Couldn't get version from name: %s\n", name)
				}
				releases = append(releases,
					Release{
						Name:    name,
						URL:     urlString,
						Version: version,
						Date:    date.Format("Mon Jan _2 15:04:05 MST 2006"),
						Commit:  commit,
					})
			}
		}
		sections = append(sections, Section{
			Header:   prefix,
			Releases: reverse(releases),
		})
	}

	return WriteHTMLForLinks(path, bucketName, sections)
}

func removeExt(name string) string {
	suffix := filepath.Ext(name)
	if len(suffix) > 0 {
		name = name[0 : len(name)-len(suffix)]
	}
	return name
}

func parseName(name string, prefix string) (version string, t time.Time, commit string, err error) {
	t = time.Unix(0, 0)

	start := strings.IndexAny(name, "123456789")
	if start == -1 {
		return
	}
	name = name[start:len(name)]
	verstr := removeExt(name)
	sversion, err := semver.Make(verstr)
	if err != nil {
		return
	}
	version = fmt.Sprintf("%d.%d.%d", sversion.Major, sversion.Minor, sversion.Patch)

	if len(sversion.Pre) != 1 {
		err = fmt.Errorf("Invalid prerelease")
		return
	}

	commit = strings.Join(sversion.Build, " ")
	// Detect if really sha commit
	if len(commit) != 7 {
		commit = ""
	}

	d := fmt.Sprintf("%d", sversion.Pre[0].VersionNum)
	t, err = time.Parse("20060102150405", d)
	if err != nil {
		return
	}

	return
}

func reverse(a []Release) []Release {
	for left, right := 0, len(a)-1; left < right; left, right = left+1, right-1 {
		a[left], a[right] = a[right], a[left]
	}
	return a
}

var htmlTemplate = `
<!doctype html>

  <title>{{ .Title }}</title>

	{{ range $index, $sec := .Sections }}

{{ $sec.Header }}

		{{ range $index2, $rel := $sec.Releases }}

{{ $rel.Name }} {{ $rel.Version }} {{ $rel.Date }} {{ $rel.Commit }}

		{{ end }}

	{{ end }}

`

func WriteHTMLForLinks(path string, title string, sections []Section) error {
	vars := map[string]interface{}{
		"Title":    title,
		"Sections": sections,
	}

	t, err := template.New("t").Parse(htmlTemplate)
	if err != nil {
		return err
	}

	if path != "" {
		var data bytes.Buffer
		err = t.Execute(&data, vars)
		if err != nil {
			return err
		}
		return ioutil.WriteFile(path, data.Bytes(), 0644)
	}
	return nil
}
<commit_msg>Use eastern time for html<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package s3

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"path/filepath"
	"strings"
	"time"

	"github.com/alecthomas/template"
	"github.com/blang/semver"
	"github.com/goamz/goamz/aws"
	"github.com/goamz/goamz/s3"
)

type Section struct {
	Header   string
	Releases []Release
}

type Release struct {
	Name    string
	URL     string
	Version string
	Date    string
	Commit  string
}

func WriteHTML(path string, bucketName string, prefixes string, suffix string) error {
	auth, err := aws.EnvAuth()
	if err != nil {
		return err
	}
	client := s3.New(auth, aws.USEast)
	bucket := client.Bucket(bucketName)
	if bucket == nil {
		return fmt.Errorf("Bucket %s not found", bucketName)
	}

	var sections []Section
	for _, prefix := range strings.Split(prefixes, ",") {
		resp, err := bucket.List(prefix, "", "", 0)
		if err != nil {
			return err
		}

		var releases []Release
		for _, k := range resp.Contents {
			if strings.HasSuffix(k.Key, suffix) {
				key := k.Key
				name := key[len(prefix):]
				urlString := fmt.Sprintf("https://s3.amazonaws.com/%s/%s%s", bucketName, prefix, url.QueryEscape(name))
				version, date, commit, err := parseName(name, prefix)
				if err != nil {
					log.Printf("Couldn't get version from name: %s\n", name)
				}

				// Convert to Eastern
				locationNewYork, err := time.LoadLocation("America/New_York")
				if err != nil {
					log.Printf("Couldn't load location: %s", err)
				}
				date = date.In(locationNewYork)

				releases = append(releases,
					Release{
						Name:    name,
						URL:     urlString,
						Version: version,
						Date:    date.Format("Mon Jan _2 15:04:05 MST 2006"),
						Commit:  commit,
					})
			}
		}
		sections = append(sections, Section{
			Header:   prefix,
			Releases: reverse(releases),
		})
	}

	return WriteHTMLForLinks(path, bucketName, sections)
}

func removeExt(name string) string {
	suffix := filepath.Ext(name)
	if len(suffix) > 0 {
		name = name[0 : len(name)-len(suffix)]
	}
	return name
}

func parseName(name string, prefix string) (version string, t time.Time, commit string, err error) {
	t = time.Unix(0, 0)

	start := strings.IndexAny(name, "123456789")
	if start == -1 {
		return
	}
	name = name[start:len(name)]
	verstr := removeExt(name)
	sversion, err := semver.Make(verstr)
	if err != nil {
		return
	}
	version = fmt.Sprintf("%d.%d.%d", sversion.Major, sversion.Minor, sversion.Patch)

	if len(sversion.Pre) != 1 {
		err = fmt.Errorf("Invalid prerelease")
		return
	}

	commit = strings.Join(sversion.Build, " ")
	// Detect if really sha commit
	if len(commit) != 7 {
		commit = ""
	}

	d := fmt.Sprintf("%d", sversion.Pre[0].VersionNum)
	t, err = time.Parse("20060102150405", d)
	if err != nil {
		return
	}

	return
}

func reverse(a []Release) []Release {
	for left, right := 0, len(a)-1; left < right; left, right = left+1, right-1 {
		a[left], a[right] = a[right], a[left]
	}
	return a
}

var htmlTemplate = `
<!doctype html>

  <title>{{ .Title }}</title>

	{{ range $index, $sec := .Sections }}

{{ $sec.Header }}

		{{ range $index2, $rel := $sec.Releases }}

{{ $rel.Name }} {{ $rel.Version }} {{ $rel.Date }} {{ $rel.Commit }}

		{{ end }}

	{{ end }}

`

func WriteHTMLForLinks(path string, title string, sections []Section) error {
	vars := map[string]interface{}{
		"Title":    title,
		"Sections": sections,
	}

	t, err := template.New("t").Parse(htmlTemplate)
	if err != nil {
		return err
	}

	if path != "" {
		var data bytes.Buffer
		err = t.Execute(&data, vars)
		if err != nil {
			return err
		}
		return ioutil.WriteFile(path, data.Bytes(), 0644)
	}
	return nil
}
<|endoftext|>


<commit_before>package client

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"sync"
	"time"
)

var (
	httpCache = make(map[string]*http.Client)
	cacheLock sync.Mutex
)

// NewClient returns a new REST client for specified server.
func NewClient(host string, version string) (*Client, error) {
	baseURL, err := url.Parse(host)
	if err != nil {
		return nil, err
	}
	if baseURL.Path == "" {
		baseURL.Path = "/"
	}
	unix2HTTP(baseURL)
	c := &Client{
		base:       baseURL,
		version:    version,
		httpClient: getHTTPClient(host),
	}
	return c, nil
}

// GetUnixServerPath returns a unix domain socket prepended with the
// provided path.
func GetUnixServerPath(socketName string, paths ...string) string {
	serverPath := "unix://"
	for _, path := range paths {
		serverPath = serverPath + path
	}
	serverPath = serverPath + socketName + ".sock"
	return serverPath
}

// Client is an HTTP REST wrapper. Use one of Get/Post/Put/Delete to get a request
// object.
type Client struct {
	base       *url.URL
	version    string
	httpClient *http.Client
}

// Status sends a Status request at the /status REST endpoint.
func (c *Client) Status() (*Status, error) {
	status := &Status{}
	err := c.Get().UsePath("/status").Do().Unmarshal(status)
	return status, err
}

// Versions send a request at the /versions REST endpoint.
func (c *Client) Versions(endpoint string) ([]string, error) {
	versions := []string{}
	err := c.Get().Resource(endpoint + "/versions").Do().Unmarshal(&versions)
	return versions, err
}

// Get returns a Request object setup for GET call.
func (c *Client) Get() *Request {
	return NewRequest(c.httpClient, c.base, "GET", c.version)
}

// Post returns a Request object setup for POST call.
func (c *Client) Post() *Request {
	return NewRequest(c.httpClient, c.base, "POST", c.version)
}

// Put returns a Request object setup for PUT call.
func (c *Client) Put() *Request {
	return NewRequest(c.httpClient, c.base, "PUT", c.version)
}

// Delete returns a Request object setup for DELETE call.
func (c *Client) Delete() *Request {
	return NewRequest(c.httpClient, c.base, "DELETE", c.version)
}

func unix2HTTP(u *url.URL) {
	if u.Scheme == "unix" {
		// Override the main URL object so the HTTP lib won't complain
		u.Scheme = "http"
		u.Host = "unix.sock"
		u.Path = ""
	}
}

func newHTTPClient(u *url.URL, tlsConfig *tls.Config, timeout time.Duration) *http.Client {
	httpTransport := &http.Transport{
		TLSClientConfig: tlsConfig,
	}

	switch u.Scheme {
	case "unix":
		socketPath := u.Path
		unixDial := func(proto, addr string) (net.Conn, error) {
			ret, err := net.DialTimeout("unix", socketPath, timeout)
			return ret, err
		}
		httpTransport.Dial = unixDial
		unix2HTTP(u)
	default:
		httpTransport.Dial = func(proto, addr string) (net.Conn, error) {
			return net.DialTimeout(proto, addr, timeout)
		}
	}

	return &http.Client{Transport: httpTransport}
}

func getHTTPClient(host string) *http.Client {
	c, ok := httpCache[host]
	if !ok {
		cacheLock.Lock()
		defer cacheLock.Unlock()
		c, ok = httpCache[host]
		if !ok {
			u, err := url.Parse(host)
			if err != nil {
				// TODO(pedge): clean up
				fmt.Println("Failed to parse into url", host)
				return nil
			}
			if u.Path == "" {
				u.Path = "/"
			}
			c = newHTTPClient(u, nil, 10*time.Second)
			httpCache[host] = c
		}
	}
	return c
}
<commit_msg>In api client take a lock before looking into httpCache map<commit_after>package client

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"sync"
	"time"
)

var (
	httpCache = make(map[string]*http.Client)
	cacheLock sync.Mutex
)

// NewClient returns a new REST client for specified server.
func NewClient(host string, version string) (*Client, error) {
	baseURL, err := url.Parse(host)
	if err != nil {
		return nil, err
	}
	if baseURL.Path == "" {
		baseURL.Path = "/"
	}
	unix2HTTP(baseURL)
	hClient := getHTTPClient(host)
	if hClient == nil {
		return nil, fmt.Errorf("Unable to parse provided url: %v", host)
	}
	c := &Client{
		base:       baseURL,
		version:    version,
		httpClient: hClient,
	}
	return c, nil
}

// GetUnixServerPath returns a unix domain socket prepended with the
// provided path.
func GetUnixServerPath(socketName string, paths ...string) string {
	serverPath := "unix://"
	for _, path := range paths {
		serverPath = serverPath + path
	}
	serverPath = serverPath + socketName + ".sock"
	return serverPath
}

// Client is an HTTP REST wrapper. Use one of Get/Post/Put/Delete to get a request
// object.
type Client struct {
	base       *url.URL
	version    string
	httpClient *http.Client
}

// Status sends a Status request at the /status REST endpoint.
func (c *Client) Status() (*Status, error) {
	status := &Status{}
	err := c.Get().UsePath("/status").Do().Unmarshal(status)
	return status, err
}

// Versions send a request at the /versions REST endpoint.
func (c *Client) Versions(endpoint string) ([]string, error) {
	versions := []string{}
	err := c.Get().Resource(endpoint + "/versions").Do().Unmarshal(&versions)
	return versions, err
}

// Get returns a Request object setup for GET call.
func (c *Client) Get() *Request {
	return NewRequest(c.httpClient, c.base, "GET", c.version)
}

// Post returns a Request object setup for POST call.
func (c *Client) Post() *Request {
	return NewRequest(c.httpClient, c.base, "POST", c.version)
}

// Put returns a Request object setup for PUT call.
func (c *Client) Put() *Request {
	return NewRequest(c.httpClient, c.base, "PUT", c.version)
}

// Delete returns a Request object setup for DELETE call.
func (c *Client) Delete() *Request {
	return NewRequest(c.httpClient, c.base, "DELETE", c.version)
}

func unix2HTTP(u *url.URL) {
	if u.Scheme == "unix" {
		// Override the main URL object so the HTTP lib won't complain
		u.Scheme = "http"
		u.Host = "unix.sock"
		u.Path = ""
	}
}

func newHTTPClient(u *url.URL, tlsConfig *tls.Config, timeout time.Duration) *http.Client {
	httpTransport := &http.Transport{
		TLSClientConfig: tlsConfig,
	}

	switch u.Scheme {
	case "unix":
		socketPath := u.Path
		unixDial := func(proto, addr string) (net.Conn, error) {
			ret, err := net.DialTimeout("unix", socketPath, timeout)
			return ret, err
		}
		httpTransport.Dial = unixDial
		unix2HTTP(u)
	default:
		httpTransport.Dial = func(proto, addr string) (net.Conn, error) {
			return net.DialTimeout(proto, addr, timeout)
		}
	}

	return &http.Client{Transport: httpTransport}
}

func getHTTPClient(host string) *http.Client {
	cacheLock.Lock()
	defer cacheLock.Unlock()
	c, ok := httpCache[host]
	if !ok {
		u, err := url.Parse(host)
		if err != nil {
			return nil
		}
		if u.Path == "" {
			u.Path = "/"
		}
		c = newHTTPClient(u, nil, 10*time.Second)
		httpCache[host] = c
	}

	return c
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"strconv"
)

var inputfile = flag.String("i", "", "The data input file to read from")

func getdata(filename string) ([]int, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer func() {
		file.Close() /// \todo handle error
	}()

	scanner := bufio.NewScanner(file)
	scanner.Scan() // scan the header line
	scanner.Scan() // scan the length of the file
	len, err := strconv.Atoi(scanner.Text())
	if err != nil {
		return nil, err
	}

	data := make([]int, len, len)

	for i := 0; scanner.Scan(); i++ {
		val, err := strconv.Atoi(scanner.Text())
		if err != nil {
			return nil, err
		}
		data[i] = val
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return data, nil
}

/// This is a subset of a true radix sort, for input values from [0..99].
/// A true radix sort for arbitrary integers is significantly slower, having to check many more digits.
func selectionSort(data []int) []int {

	/// DO NOT replace this with the XOR method. It is slow.
	swap := func(a *int, b *int) {
		temp := *a
		*a = *b
		*b = temp
	};

	// i is the current position, before which all elements are sorted
	for i := 0; i != len(data); i++ {

		nextLowest := i ///< the index of the lowest value in the array, which needs to be 'selected' and swapped with data[i]
		// j is the position of the iterator, which finds the next lowest value in the array
		for j := i; j != len(data); j++ {
			if data[j] < data[nextLowest] {
				nextLowest = j
			}
		}

//		fmt.Printf("swapping %d and %d\n", i, nextLowest) // debug

		swap(&data[i], &data[nextLowest])
	}

	return data
}

func main() {
	flag.Parse()

	if *inputfile == "" {
		fmt.Println("Usage:")
		flag.PrintDefaults()
		return
	}

	data, err := getdata(*inputfile)
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Println(data)

	fmt.Println("sorting...")
	data = selectionSort(data)

	fmt.Println(data)
}
<commit_msg>fixed erroneous comment<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"strconv"
)

var inputfile = flag.String("i", "", "The data input file to read from")

func getdata(filename string) ([]int, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer func() {
		file.Close() /// \todo handle error
	}()

	scanner := bufio.NewScanner(file)
	scanner.Scan() // scan the header line
	scanner.Scan() // scan the length of the file
	len, err := strconv.Atoi(scanner.Text())
	if err != nil {
		return nil, err
	}

	data := make([]int, len, len)

	for i := 0; scanner.Scan(); i++ {
		val, err := strconv.Atoi(scanner.Text())
		if err != nil {
			return nil, err
		}
		data[i] = val
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return data, nil
}

func selectionsort(data []int) []int {

	/// DO NOT replace this with the XOR method. It is slow.
	swap := func(a *int, b *int) {
		temp := *a
		*a = *b
		*b = temp
	};

	// i is the current position, before which all elements are sorted
	for i := 0; i != len(data); i++ {

		nextLowest := i ///< the index of the lowest value in the array, which needs to be 'selected' and swapped with data[i]
		// j is the position of the iterator, which finds the next lowest value in the array
		for j := i; j != len(data); j++ {
			if data[j] < data[nextLowest] {
				nextLowest = j
			}
		}

//		fmt.Printf("swapping %d and %d\n", i, nextLowest) // debug

		swap(&data[i], &data[nextLowest])
	}

	return data
}

func main() {
	flag.Parse()

	if *inputfile == "" {
		fmt.Println("Usage:")
		flag.PrintDefaults()
		return
	}

	data, err := getdata(*inputfile)
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Println(data)

	fmt.Println("sorting...")
	data = selectionsort(data)

	fmt.Println(data)
}
<|endoftext|>


<commit_before>package shell_test

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"testing"
	"time"

	"github.com/buildkite/agent/bootstrap/shell"
	"github.com/buildkite/bintest"
)

func TestRunAndCaptureWithTTY(t *testing.T) {
	sshKeygen, err := bintest.NewMock("ssh-keygen")
	if err != nil {
		t.Fatal(err)
	}
	defer sshKeygen.Close()

	sh := newShellForTest(t)
	sh.PTY = true

	go func() {
		call := <-sshKeygen.Ch
		fmt.Fprintln(call.Stdout, "Llama party! 🎉")
		call.Exit(0)
	}()

	actual, err := sh.RunAndCapture(sshKeygen.Path, "-f", "my_hosts", "-F", "llamas.com")
	if err != nil {
		t.Error(err)
	}

	if expected := "Llama party! 🎉"; string(actual) != expected {
		t.Fatalf("Expected %q, got %q", expected, actual)
	}
}

func TestRun(t *testing.T) {
	sshKeygen, err := bintest.NewMock("ssh-keygen")
	if err != nil {
		t.Fatal(err)
	}
	defer sshKeygen.Close()

	out := &bytes.Buffer{}

	sh := newShellForTest(t)
	sh.PTY = false
	sh.Writer = out
	sh.Logger = &shell.WriterLogger{Writer: out, Ansi: false}

	go func() {
		call := <-sshKeygen.Ch
		fmt.Fprintln(call.Stdout, "Llama party! 🎉")
		call.Exit(0)
	}()

	if err = sh.Run(sshKeygen.Path, "-f", "my_hosts", "-F", "llamas.com"); err != nil {
		t.Fatal(err)
	}

	actual := out.String()

	promptPrefix := "$"
	if runtime.GOOS == "windows" {
		promptPrefix = ">"
	}

	if expected := promptPrefix + " " + sshKeygen.Path + " -f my_hosts -F llamas.com\nLlama party! 🎉\n"; actual != expected {
		t.Fatalf("Expected %q, got %q", expected, actual)
	}
}

func TestDefaultWorkingDirFromSystem(t *testing.T) {
	sh, err := shell.New()
	if err != nil {
		t.Fatal(err)
	}

	currentWd, _ := os.Getwd()
	if actual := sh.Getwd(); actual != currentWd {
		t.Fatalf("Expected working dir %q, got %q", currentWd, actual)
	}
}

func TestWorkingDir(t *testing.T) {
	tempDir, err := ioutil.TempDir("", "shelltest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tempDir)

	// macos has a symlinked temp dir
	if runtime.GOOS == "darwin" {
		tempDir, _ = filepath.EvalSymlinks(tempDir)
	}

	dirs := []string{tempDir, "my", "test", "dirs"}

	if err := os.MkdirAll(filepath.Join(dirs...), 0700); err != nil {
		t.Fatal(err)
	}

	currentWd, _ := os.Getwd()

	sh, err := shell.New()
	sh.Logger = shell.DiscardLogger

	if err != nil {
		t.Fatal(err)
	}

	for idx := range dirs {
		dir := filepath.Join(dirs[0 : idx+1]...)

		if err := sh.Chdir(dir); err != nil {
			t.Fatal(err)
		}

		if actual := sh.Getwd(); actual != dir {
			t.Fatalf("Expected working dir %q, got %q", dir, actual)
		}

		var out string

		// there is no pwd for windows, and getting it requires using a shell builtin
		if runtime.GOOS == "windows" {
			out, err = sh.RunAndCapture("cmd", "/c", "echo", "%cd%")
			if err != nil {
				t.Fatal(err)
			}
		} else {
			out, err = sh.RunAndCapture("pwd")
			if err != nil {
				t.Fatal(err)
			}
		}

		if actual := out; actual != dir {
			t.Fatalf("Expected working dir (from pwd command) %q, got %q", dir, actual)
		}
	}

	afterWd, _ := os.Getwd()
	if afterWd != currentWd {
		t.Fatalf("Expected working dir to be the same as before shell commands ran")
	}
}

func TestLockFileRetriesAndTimesOut(t *testing.T) {
	dir, err := ioutil.TempDir("", "shelltest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	sh := newShellForTest(t)
	sh.Logger = shell.DiscardLogger

	lockPath := filepath.Join(dir, "my.lock")

	// acquire a lock in another process
	cmd, err := acquireLockInOtherProcess(lockPath)
	if err != nil {
		t.Fatal(err)
	}

	defer cmd.Process.Kill()

	// acquire lock
	_, err = sh.LockFile(lockPath, time.Second*2)
	if err != context.DeadlineExceeded {
		t.Fatalf("Expected DeadlineExceeded error, got %v", err)
	}
}

func acquireLockInOtherProcess(lockfile string) (*exec.Cmd, error) {
	cmd := exec.Command(os.Args[0], "-test.run=TestAcquiringLockHelperProcess", "--", lockfile)
	cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}

	err := cmd.Start()
	if err != nil {
		return cmd, err
	}

	// wait for the above process to get a lock
	for {
		if _, err = os.Stat(lockfile); os.IsNotExist(err) {
			time.Sleep(time.Millisecond * 10)
			continue
		}
		break
	}

	return cmd, nil
}

// TestAcquiringLockHelperProcess isn't a real test. It's used as a helper process
func TestAcquiringLockHelperProcess(t *testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
		return
	}

	fileName := os.Args[len(os.Args)-1]
	sh := newShellForTest(t)

	log.Printf("Locking %s", fileName)
	if _, err := sh.LockFile(fileName, time.Second*10); err != nil {
		os.Exit(1)
	}

	log.Printf("Acquired lock %s", fileName)
	c := make(chan struct{})
	<-c
}

func newShellForTest(t *testing.T) *shell.Shell {
	sh, err := shell.New()
	if err != nil {
		t.Fatal(err)
	}
	sh.Logger = shell.DiscardLogger
	return sh
}
<commit_msg>Use proxies, not mocks for ssh test<commit_after>package shell_test

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"testing"
	"time"

	"github.com/buildkite/agent/bootstrap/shell"
	"github.com/buildkite/bintest"
)

func TestRunAndCaptureWithTTY(t *testing.T) {
	sshKeygen, err := bintest.CompileProxy("ssh-keygen")
	if err != nil {
		t.Fatal(err)
	}
	defer sshKeygen.Close()

	sh := newShellForTest(t)
	sh.PTY = true

	go func() {
		call := <-sshKeygen.Ch
		fmt.Fprintln(call.Stdout, "Llama party! 🎉")
		call.Exit(0)
	}()

	actual, err := sh.RunAndCapture(sshKeygen.Path, "-f", "my_hosts", "-F", "llamas.com")
	if err != nil {
		t.Error(err)
	}

	if expected := "Llama party! 🎉"; string(actual) != expected {
		t.Fatalf("Expected %q, got %q", expected, actual)
	}
}

func TestRun(t *testing.T) {
	sshKeygen, err := bintest.CompileProxy("ssh-keygen")
	if err != nil {
		t.Fatal(err)
	}
	defer sshKeygen.Close()

	out := &bytes.Buffer{}

	sh := newShellForTest(t)
	sh.PTY = false
	sh.Writer = out
	sh.Logger = &shell.WriterLogger{Writer: out, Ansi: false}

	go func() {
		call := <-sshKeygen.Ch
		fmt.Fprintln(call.Stdout, "Llama party! 🎉")
		call.Exit(0)
	}()

	if err = sh.Run(sshKeygen.Path, "-f", "my_hosts", "-F", "llamas.com"); err != nil {
		t.Fatal(err)
	}

	actual := out.String()

	promptPrefix := "$"
	if runtime.GOOS == "windows" {
		promptPrefix = ">"
	}

	if expected := promptPrefix + " " + sshKeygen.Path + " -f my_hosts -F llamas.com\nLlama party! 🎉\n"; actual != expected {
		t.Fatalf("Expected %q, got %q", expected, actual)
	}
}

func TestDefaultWorkingDirFromSystem(t *testing.T) {
	sh, err := shell.New()
	if err != nil {
		t.Fatal(err)
	}

	currentWd, _ := os.Getwd()
	if actual := sh.Getwd(); actual != currentWd {
		t.Fatalf("Expected working dir %q, got %q", currentWd, actual)
	}
}

func TestWorkingDir(t *testing.T) {
	tempDir, err := ioutil.TempDir("", "shelltest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tempDir)

	// macos has a symlinked temp dir
	if runtime.GOOS == "darwin" {
		tempDir, _ = filepath.EvalSymlinks(tempDir)
	}

	dirs := []string{tempDir, "my", "test", "dirs"}

	if err := os.MkdirAll(filepath.Join(dirs...), 0700); err != nil {
		t.Fatal(err)
	}

	currentWd, _ := os.Getwd()

	sh, err := shell.New()
	sh.Logger = shell.DiscardLogger

	if err != nil {
		t.Fatal(err)
	}

	for idx := range dirs {
		dir := filepath.Join(dirs[0 : idx+1]...)

		if err := sh.Chdir(dir); err != nil {
			t.Fatal(err)
		}

		if actual := sh.Getwd(); actual != dir {
			t.Fatalf("Expected working dir %q, got %q", dir, actual)
		}

		var out string

		// there is no pwd for windows, and getting it requires using a shell builtin
		if runtime.GOOS == "windows" {
			out, err = sh.RunAndCapture("cmd", "/c", "echo", "%cd%")
			if err != nil {
				t.Fatal(err)
			}
		} else {
			out, err = sh.RunAndCapture("pwd")
			if err != nil {
				t.Fatal(err)
			}
		}

		if actual := out; actual != dir {
			t.Fatalf("Expected working dir (from pwd command) %q, got %q", dir, actual)
		}
	}

	afterWd, _ := os.Getwd()
	if afterWd != currentWd {
		t.Fatalf("Expected working dir to be the same as before shell commands ran")
	}
}

func TestLockFileRetriesAndTimesOut(t *testing.T) {
	dir, err := ioutil.TempDir("", "shelltest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	sh := newShellForTest(t)
	sh.Logger = shell.DiscardLogger

	lockPath := filepath.Join(dir, "my.lock")

	// acquire a lock in another process
	cmd, err := acquireLockInOtherProcess(lockPath)
	if err != nil {
		t.Fatal(err)
	}

	defer cmd.Process.Kill()

	// acquire lock
	_, err = sh.LockFile(lockPath, time.Second*2)
	if err != context.DeadlineExceeded {
		t.Fatalf("Expected DeadlineExceeded error, got %v", err)
	}
}

func acquireLockInOtherProcess(lockfile string) (*exec.Cmd, error) {
	cmd := exec.Command(os.Args[0], "-test.run=TestAcquiringLockHelperProcess", "--", lockfile)
	cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}

	err := cmd.Start()
	if err != nil {
		return cmd, err
	}

	// wait for the above process to get a lock
	for {
		if _, err = os.Stat(lockfile); os.IsNotExist(err) {
			time.Sleep(time.Millisecond * 10)
			continue
		}
		break
	}

	return cmd, nil
}

// TestAcquiringLockHelperProcess isn't a real test. It's used as a helper process
func TestAcquiringLockHelperProcess(t *testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
		return
	}

	fileName := os.Args[len(os.Args)-1]
	sh := newShellForTest(t)

	log.Printf("Locking %s", fileName)
	if _, err := sh.LockFile(fileName, time.Second*10); err != nil {
		os.Exit(1)
	}

	log.Printf("Acquired lock %s", fileName)
	c := make(chan struct{})
	<-c
}

func newShellForTest(t *testing.T) *shell.Shell {
	sh, err := shell.New()
	if err != nil {
		t.Fatal(err)
	}
	sh.Logger = shell.DiscardLogger
	return sh
}
<|endoftext|>


<commit_before>package helpers

// CleanMapDeep - Cleans map deeply. Converting all map[interface{}]interface{} to map[string]interface{}.
// OK is false when obtained invalid type in map key.
func CleanMapDeep(v interface{}) (_ interface{}, ok bool) {
	switch m := v.(type) {
	case map[string]interface{}:
		for k, v := range m {
			m[k], ok = CleanMapDeep(v)
			if !ok {
				return
			}
		}
		return m, true
	case map[interface{}]interface{}:
		res := make(map[string]interface{})
		for k, v := range m {
			var key string
			key, ok = k.(string)
			if !ok {
				return
			}
			res[key], ok = CleanMapDeep(v)
			if !ok {
				return
			}
		}
		return res, true
	case []interface{}:
		for n, v := range m {
			m[n], ok = CleanMapDeep(v)
			if !ok {
				return
			}
		}
		return m, true
	}
	return v, true
}
<commit_msg>cleanmap: helper for template.Context<commit_after>package helpers

import "tower.pro/renderer/template"

// CleanMapDeep - Cleans map deeply. Converting all map[interface{}]interface{} to map[string]interface{}.
// OK is false when obtained invalid type in map key.
func CleanMapDeep(v interface{}) (_ interface{}, ok bool) {
	switch m := v.(type) {
	case template.Context:
		return CleanMapDeep(map[string]interface{}(m))
	case map[string]interface{}:
		for k, v := range m {
			m[k], ok = CleanMapDeep(v)
			if !ok {
				return
			}
		}
		return m, true
	case map[interface{}]interface{}:
		res := make(map[string]interface{})
		for k, v := range m {
			var key string
			key, ok = k.(string)
			if !ok {
				return
			}
			res[key], ok = CleanMapDeep(v)
			if !ok {
				return
			}
		}
		return res, true
	case []interface{}:
		for n, v := range m {
			m[n], ok = CleanMapDeep(v)
			if !ok {
				return
			}
		}
		return m, true
	}
	return v, true
}
<|endoftext|>


<commit_before>package tasks

import (
	"bufio"
	"encoding/json"
	"fmt"
	"os/exec"
	"time"

	"github.com/ansible-semaphore/semaphore/api/sockets"
	"github.com/ansible-semaphore/semaphore/db"
)

func (t *task) log(msg string) {
	now := time.Now()

	for _, user := range t.users {
		b, err := json.Marshal(&map[string]interface{}{
			"type":       "log",
			"output":     msg,
			"time":       now,
			"task_id":    t.task.ID,
			"project_id": t.projectID,
		})

		if err != nil {
			panic(err)
		}

		sockets.Message(user, b)
	}

	go func() {
		_, err := db.Mysql.Exec("insert into task__output (task_id, task, output, time) VALUES (?, '', ?, ?)", t.task.ID, msg, now)
		if err != nil {
			fmt.Printf("Failed to insert task output: %s\n", err.Error())
			panic(err)
		}
	}()
}

func (t *task) updateStatus() {
	for _, user := range t.users {
		b, err := json.Marshal(&map[string]interface{}{
			"type":       "update",
			"start":      t.task.Start,
			"end":        t.task.End,
			"status":     t.task.Status,
			"task_id":    t.task.ID,
			"project_id": t.projectID,
		})

		if err != nil {
			panic(err)
		}

		sockets.Message(user, b)
	}

	if _, err := db.Mysql.Exec("update task set status=?, start=?, end=? where id=?", t.task.Status, t.task.Start, t.task.End, t.task.ID); err != nil {
		fmt.Printf("Failed to update task status: %s\n", err.Error())
		t.log("Fatal error with database!")
		panic(err)
	}
}

func Readln(r *bufio.Reader) (string, error) {
	var (
		isPrefix bool  = true
		err      error = nil
		line, ln []byte
	)
	for isPrefix && err == nil {
		line, isPrefix, err = r.ReadLine()
		ln = append(ln, line...)
	}
	return string(ln), err
}

func (t *task) logPipe(reader *bufio.Reader) {

	s, e := Readln(reader)
	for e == nil {
		t.log(s)
		s, e = Readln(reader)
	}

	/*for reader.Scan() {
		t.log(reader.Text())
	}

	if err := reader.Err(); err != nil {
		t.log("Error scanning input!")
		panic(err)
	}*/
}

func (t *task) logCmd(cmd *exec.Cmd) {
	stderr, _ := cmd.StderrPipe()
	stdout, _ := cmd.StdoutPipe()

	go t.logPipe(bufio.NewReader(stderr))
	go t.logPipe(bufio.NewReader(stdout))
}
<commit_msg>add error handling, name variables more clear<commit_after>package tasks

import (
	"bufio"
	"encoding/json"
	"fmt"
	"os/exec"
	"time"

	"github.com/ansible-semaphore/semaphore/api/sockets"
	"github.com/ansible-semaphore/semaphore/db"
)

func (t *task) log(msg string) {
	now := time.Now()

	for _, user := range t.users {
		b, err := json.Marshal(&map[string]interface{}{
			"type":       "log",
			"output":     msg,
			"time":       now,
			"task_id":    t.task.ID,
			"project_id": t.projectID,
		})

		if err != nil {
			panic(err)
		}

		sockets.Message(user, b)
	}

	go func() {
		_, err := db.Mysql.Exec("insert into task__output (task_id, task, output, time) VALUES (?, '', ?, ?)", t.task.ID, msg, now)
		if err != nil {
			fmt.Printf("Failed to insert task output: %s\n", err.Error())
			panic(err)
		}
	}()
}

func (t *task) updateStatus() {
	for _, user := range t.users {
		b, err := json.Marshal(&map[string]interface{}{
			"type":       "update",
			"start":      t.task.Start,
			"end":        t.task.End,
			"status":     t.task.Status,
			"task_id":    t.task.ID,
			"project_id": t.projectID,
		})

		if err != nil {
			panic(err)
		}

		sockets.Message(user, b)
	}

	if _, err := db.Mysql.Exec("update task set status=?, start=?, end=? where id=?", t.task.Status, t.task.Start, t.task.End, t.task.ID); err != nil {
		fmt.Printf("Failed to update task status: %s\n", err.Error())
		t.log("Fatal error with database!")
		panic(err)
	}
}

func Readln(r *bufio.Reader) (string, error) {
	var (
		isPrefix bool  = true
		err      error = nil
		line, ln []byte
	)
	for isPrefix && err == nil {
		line, isPrefix, err = r.ReadLine()
		ln = append(ln, line...)
	}
	return string(ln), err
}

func (t *task) logPipe(reader *bufio.Reader) {

	line, err := Readln(reader)
	for err == nil {
		t.log(line)
		line, err = Readln(reader)
	}

	if err != nil && err.Error() != "EOF" {
		fmt.Printf("Failed to read output of command: %s\n", err.Error())
		panic(err)
	}

}

func (t *task) logCmd(cmd *exec.Cmd) {
	stderr, _ := cmd.StderrPipe()
	stdout, _ := cmd.StdoutPipe()

	go t.logPipe(bufio.NewReader(stderr))
	go t.logPipe(bufio.NewReader(stdout))
}
<|endoftext|>


<commit_before>package types

type Script struct {
	Id          string   `json:"id" header:"ID"`
	Name        string   `json:"name" header:"NAME"`
	Description string   `json:"description" header:"DESCRIPTION"`
	Code        string   `json:"code" header:"CODE"`
	Parameters  []string `json:"parameters" header:"PARAMETERS"`
}
<commit_msg>avoid showing script code in blueprint script lists<commit_after>package types

type Script struct {
	Id          string   `json:"id" header:"ID"`
	Name        string   `json:"name" header:"NAME"`
	Description string   `json:"description" header:"DESCRIPTION"`
	Code        string   `json:"code" header:"CODE" show:"nolist"`
	Parameters  []string `json:"parameters" header:"PARAMETERS"`
}
<|endoftext|>


<commit_before>package types

type Server struct {
	ID                string `json:"id" header:"ID"`
	Name              string `json:"name" header:"NAME"`
	Fqdn              string `json:"fqdn" header:"FQDN"`
	State             string `json:"state" header:"STATE"`
	PublicIP          string `json:"public_ip" header:"PUBLIC_IP"`
	TemplateID        string `json:"template_id" header:"TEMPLATE_ID"`
	ServerPlanID      string `json:"server_plan_id" header:"SERVER_PLAN_ID"`
	CloudAccountID    string `json:"cloud_account_id" header:"CLOUD_ACCOUNT_ID"`
	SSHProfileID      string `json:"ssh_profile_id" header:"SSH_PROFILE_ID"`
	FirewallProfileID string `json:"firewall_profile_id" header:"FIREWALL_PROFILE_ID"`
	SubnetID          string `json:"subnet_id,omitempty"`
	VpcID             string `json:"vpc_id,omitempty"`
	ResourceType      string `json:"resource_type" header:"RESOURCE_TYPE" show:"nolist"`
	LabelableFields
}

type ScriptChar struct {
	ResourceType    string                 `json:"resource_type" header:"RESOURCE_TYPE" show:"noshow,nolist"`
	ID              string                 `json:"id" header:"ID"`
	Type            string                 `json:"type" header:"TYPE"`
	ParameterValues map[string]interface{} `json:"parameter_values" header:"PARAMETER_VALUES"`
	TemplateID      string                 `json:"template_id" header:"TEMPLATE_ID"`
	ScriptID        string                 `json:"script_id" header:"SCRIPT_ID"`
	ExecutionOrder  int                    `json:"execution_order" header:"EXECUTION_ORDER" show:"noshow,nolist"`
}
<commit_msg>Fix VPC and Subnet identifiers in Server resource (issue #81)<commit_after>package types

type Server struct {
	ID                string `json:"id" header:"ID"`
	Name              string `json:"name" header:"NAME"`
	Fqdn              string `json:"fqdn" header:"FQDN"`
	State             string `json:"state" header:"STATE"`
	PublicIP          string `json:"public_ip" header:"PUBLIC_IP"`
	TemplateID        string `json:"template_id" header:"TEMPLATE_ID"`
	ServerPlanID      string `json:"server_plan_id" header:"SERVER_PLAN_ID"`
	CloudAccountID    string `json:"cloud_account_id" header:"CLOUD_ACCOUNT_ID"`
	SSHProfileID      string `json:"ssh_profile_id" header:"SSH_PROFILE_ID"`
	FirewallProfileID string `json:"firewall_profile_id" header:"FIREWALL_PROFILE_ID"`
	SubnetID          string `json:"subnet_id,omitempty" header:"SUBNET_ID" show:"nolist"`
	VpcID             string `json:"vpc_id,omitempty" header:"VPC_ID" show:"nolist"`
	ResourceType      string `json:"resource_type" header:"RESOURCE_TYPE" show:"nolist"`
	LabelableFields
}

type ScriptChar struct {
	ResourceType    string                 `json:"resource_type" header:"RESOURCE_TYPE" show:"noshow,nolist"`
	ID              string                 `json:"id" header:"ID"`
	Type            string                 `json:"type" header:"TYPE"`
	ParameterValues map[string]interface{} `json:"parameter_values" header:"PARAMETER_VALUES"`
	TemplateID      string                 `json:"template_id" header:"TEMPLATE_ID"`
	ScriptID        string                 `json:"script_id" header:"SCRIPT_ID"`
	ExecutionOrder  int                    `json:"execution_order" header:"EXECUTION_ORDER" show:"noshow,nolist"`
}
<|endoftext|>