<commit_before>package radar

import (
	"sync"
	"time"

	"github.com/concourse/atc/builds"
	"github.com/concourse/atc/config"

	"github.com/pivotal-golang/lager"
)

type VersionDB interface {
	SaveVersionedResource(builds.VersionedResource) error
	GetLatestVersionedResource(string) (builds.VersionedResource, error)
}

type ResourceChecker interface {
	CheckResource(config.Resource, builds.Version) ([]builds.Version, error)
}

type Radar struct {
	logger lager.Logger

	checker  ResourceChecker
	tracker  VersionDB
	interval time.Duration

	stop     chan struct{}
	scanning *sync.WaitGroup
}

func NewRadar(
	logger lager.Logger,
	checker ResourceChecker,
	tracker VersionDB,
	interval time.Duration,
) *Radar {
	return &Radar{
		logger: logger,

		checker:  checker,
		tracker:  tracker,
		interval: interval,

		stop:     make(chan struct{}),
		scanning: new(sync.WaitGroup),
	}
}

func (radar *Radar) Scan(resource config.Resource) {
	radar.scanning.Add(1)

	go func() {
		defer radar.scanning.Done()

		ticker := time.NewTicker(radar.interval)

		for {
			select {
			case <-radar.stop:
				return

			case <-ticker.C:
				var from builds.Version

				if vr, err := radar.tracker.GetLatestVersionedResource(resource.Name); err == nil {
					from = vr.Version
				}

				log := radar.logger.Session("radar", lager.Data{
					"resource": resource.Name,
					"from":     from,
				})

				log.Debug("check")

				newVersions, err := radar.checker.CheckResource(resource, from)
				if err != nil {
					log.Error("failed-to-check", err)
					break
				}

				if len(newVersions) == 0 {
					break
				}

				log.Info("versions-found", lager.Data{
					"versions": newVersions,
					"total":    len(newVersions),
				})

				for _, version := range newVersions {
					err = radar.tracker.SaveVersionedResource(builds.VersionedResource{
						Name:    resource.Name,
						Type:    resource.Type,
						Source:  resource.Source,
						Version: version,
					})
					if err != nil {
						log.Error("failed-to-save-current-version", err, lager.Data{
							"version": version,
						})
					}
				}
			}
		}
	}()
}

func (radar *Radar) Stop() {
	close(radar.stop)
	radar.scanning.Wait()
}
<commit_msg>log resource type when checking<commit_after>package radar

import (
	"sync"
	"time"

	"github.com/concourse/atc/builds"
	"github.com/concourse/atc/config"

	"github.com/pivotal-golang/lager"
)

type VersionDB interface {
	SaveVersionedResource(builds.VersionedResource) error
	GetLatestVersionedResource(string) (builds.VersionedResource, error)
}

type ResourceChecker interface {
	CheckResource(config.Resource, builds.Version) ([]builds.Version, error)
}

type Radar struct {
	logger lager.Logger

	checker  ResourceChecker
	tracker  VersionDB
	interval time.Duration

	stop     chan struct{}
	scanning *sync.WaitGroup
}

func NewRadar(
	logger lager.Logger,
	checker ResourceChecker,
	tracker VersionDB,
	interval time.Duration,
) *Radar {
	return &Radar{
		logger: logger,

		checker:  checker,
		tracker:  tracker,
		interval: interval,

		stop:     make(chan struct{}),
		scanning: new(sync.WaitGroup),
	}
}

func (radar *Radar) Scan(resource config.Resource) {
	radar.scanning.Add(1)

	go func() {
		defer radar.scanning.Done()

		ticker := time.NewTicker(radar.interval)

		for {
			select {
			case <-radar.stop:
				return

			case <-ticker.C:
				var from builds.Version

				if vr, err := radar.tracker.GetLatestVersionedResource(resource.Name); err == nil {
					from = vr.Version
				}

				log := radar.logger.Session("radar", lager.Data{
					"type":     resource.Type,
					"resource": resource.Name,
					"from":     from,
				})

				log.Debug("check")

				newVersions, err := radar.checker.CheckResource(resource, from)
				if err != nil {
					log.Error("failed-to-check", err)
					break
				}

				if len(newVersions) == 0 {
					break
				}

				log.Info("versions-found", lager.Data{
					"versions": newVersions,
					"total":    len(newVersions),
				})

				for _, version := range newVersions {
					err = radar.tracker.SaveVersionedResource(builds.VersionedResource{
						Name:    resource.Name,
						Type:    resource.Type,
						Source:  resource.Source,
						Version: version,
					})
					if err != nil {
						log.Error("failed-to-save-current-version", err, lager.Data{
							"version": version,
						})
					}
				}
			}
		}
	}()
}

func (radar *Radar) Stop() {
	close(radar.stop)
	radar.scanning.Wait()
}
<|endoftext|>


<commit_before>package random

import (
	"hash/crc64"
	"math/rand"
	"time"
)

// NewRand returns a seeded random number generator, using a seed derived
// from the provided string.
//
// If the seed string is empty, the current time is used as a seed.
func NewRand(seed string) *rand.Rand {
	var seedInt int64
	if seed != "" {
		crcTable := crc64.MakeTable(crc64.ISO)
		seedInt = int64(crc64.Checksum([]byte(seed), crcTable))
	} else {
		seedInt = time.Now().Unix()
	}

	randSource := rand.NewSource(seedInt)
	return rand.New(randSource)
}
<commit_msg>update seed.go with UnixNano instead of Unix<commit_after>package random

import (
	"hash/crc64"
	"math/rand"
	"time"
)

// NewRand returns a seeded random number generator, using a seed derived
// from the provided string.
//
// If the seed string is empty, the current time is used as a seed.
func NewRand(seed string) *rand.Rand {
	var seedInt int64
	if seed != "" {
		crcTable := crc64.MakeTable(crc64.ISO)
		seedInt = int64(crc64.Checksum([]byte(seed), crcTable))
	} else {
		seedInt = time.Now().UnixNano()
	}

	randSource := rand.NewSource(seedInt)
	return rand.New(randSource)
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// General environment variables.

package os

import (
	"internal/testlog"
	"syscall"
)

// Expand replaces ${var} or $var in the string based on the mapping function.
// For example, os.ExpandEnv(s) is equivalent to os.Expand(s, os.Getenv).
func Expand(s string, mapping func(string) string) string {
	buf := make([]byte, 0, 2*len(s))
	// ${} is all ASCII, so bytes are fine for this operation.
	i := 0
	for j := 0; j < len(s); j++ {
		if s[j] == '$' && j+1 < len(s) {
			buf = append(buf, s[i:j]...)
			name, w := getShellName(s[j+1:])
			// If the name is empty, keep the $.
			if name == "" {
				buf = append(buf, s[j])
			} else {
				buf = append(buf, mapping(name)...)
			}
			j += w
			i = j + 1
		}
	}
	return string(buf) + s[i:]
}

// ExpandEnv replaces ${var} or $var in the string according to the values
// of the current environment variables. References to undefined
// variables are replaced by the empty string.
func ExpandEnv(s string) string {
	return Expand(s, Getenv)
}

// isShellSpecialVar reports whether the character identifies a special
// shell variable such as $*.
func isShellSpecialVar(c uint8) bool {
	switch c {
	case '*', '#', '$', '@', '!', '?', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
		return true
	}
	return false
}

// isAlphaNum reports whether the byte is an ASCII letter, number, or underscore
func isAlphaNum(c uint8) bool {
	return c == '_' || '0' <= c && c <= '9' || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z'
}

// getShellName returns the name that begins the string and the number of bytes
// consumed to extract it. If the name is enclosed in {}, it's part of a ${}
// expansion and two more bytes are needed than the length of the name.
func getShellName(s string) (string, int) {
	switch {
	case s[0] == '{':
		if len(s) > 2 && isShellSpecialVar(s[1]) && s[2] == '}' {
			return s[1:2], 3
		}
		// Scan to closing brace
		for i := 1; i < len(s); i++ {
			if s[i] == '}' {
				return s[1:i], i + 1
			}
		}
		return "", 1 // Bad syntax; just eat the brace.
	case isShellSpecialVar(s[0]):
		return s[0:1], 1
	}
	// Scan alphanumerics.
	var i int
	for i = 0; i < len(s) && isAlphaNum(s[i]); i++ {
	}
	return s[:i], i
}

// Getenv retrieves the value of the environment variable named by the key.
// It returns the value, which will be empty if the variable is not present.
// To distinguish between an empty value and an unset value, use LookupEnv.
func Getenv(key string) string {
	testlog.Getenv(key)
	v, _ := syscall.Getenv(key)
	return v
}

// LookupEnv retrieves the value of the environment variable named
// by the key. If the variable is present in the environment the
// value (which may be empty) is returned and the boolean is true.
// Otherwise the returned value will be empty and the boolean will
// be false.
func LookupEnv(key string) (string, bool) {
	testlog.Getenv(key)
	return syscall.Getenv(key)
}

// Setenv sets the value of the environment variable named by the key.
// It returns an error, if any.
func Setenv(key, value string) error {
	err := syscall.Setenv(key, value)
	if err != nil {
		return NewSyscallError("setenv", err)
	}
	return nil
}

// Unsetenv unsets a single environment variable.
func Unsetenv(key string) error {
	return syscall.Unsetenv(key)
}

// Clearenv deletes all environment variables.
func Clearenv() {
	syscall.Clearenv()
}

// Environ returns a copy of strings representing the environment,
// in the form "key=value".
func Environ() []string {
	return syscall.Environ()
}
<commit_msg>os: allocate buffer lazily in Expand<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// General environment variables.

package os

import (
	"internal/testlog"
	"syscall"
)

// Expand replaces ${var} or $var in the string based on the mapping function.
// For example, os.ExpandEnv(s) is equivalent to os.Expand(s, os.Getenv).
func Expand(s string, mapping func(string) string) string {
	var buf []byte
	// ${} is all ASCII, so bytes are fine for this operation.
	i := 0
	for j := 0; j < len(s); j++ {
		if s[j] == '$' && j+1 < len(s) {
			if buf == nil {
				buf = make([]byte, 0, 2*len(s))
			}
			buf = append(buf, s[i:j]...)
			name, w := getShellName(s[j+1:])
			// If the name is empty, keep the $.
			if name == "" {
				buf = append(buf, s[j])
			} else {
				buf = append(buf, mapping(name)...)
			}
			j += w
			i = j + 1
		}
	}
	if buf == nil {
		return s
	}
	return string(buf) + s[i:]
}

// ExpandEnv replaces ${var} or $var in the string according to the values
// of the current environment variables. References to undefined
// variables are replaced by the empty string.
func ExpandEnv(s string) string {
	return Expand(s, Getenv)
}

// isShellSpecialVar reports whether the character identifies a special
// shell variable such as $*.
func isShellSpecialVar(c uint8) bool {
	switch c {
	case '*', '#', '$', '@', '!', '?', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
		return true
	}
	return false
}

// isAlphaNum reports whether the byte is an ASCII letter, number, or underscore
func isAlphaNum(c uint8) bool {
	return c == '_' || '0' <= c && c <= '9' || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z'
}

// getShellName returns the name that begins the string and the number of bytes
// consumed to extract it. If the name is enclosed in {}, it's part of a ${}
// expansion and two more bytes are needed than the length of the name.
func getShellName(s string) (string, int) {
	switch {
	case s[0] == '{':
		if len(s) > 2 && isShellSpecialVar(s[1]) && s[2] == '}' {
			return s[1:2], 3
		}
		// Scan to closing brace
		for i := 1; i < len(s); i++ {
			if s[i] == '}' {
				return s[1:i], i + 1
			}
		}
		return "", 1 // Bad syntax; just eat the brace.
	case isShellSpecialVar(s[0]):
		return s[0:1], 1
	}
	// Scan alphanumerics.
	var i int
	for i = 0; i < len(s) && isAlphaNum(s[i]); i++ {
	}
	return s[:i], i
}

// Getenv retrieves the value of the environment variable named by the key.
// It returns the value, which will be empty if the variable is not present.
// To distinguish between an empty value and an unset value, use LookupEnv.
func Getenv(key string) string {
	testlog.Getenv(key)
	v, _ := syscall.Getenv(key)
	return v
}

// LookupEnv retrieves the value of the environment variable named
// by the key. If the variable is present in the environment the
// value (which may be empty) is returned and the boolean is true.
// Otherwise the returned value will be empty and the boolean will
// be false.
func LookupEnv(key string) (string, bool) {
	testlog.Getenv(key)
	return syscall.Getenv(key)
}

// Setenv sets the value of the environment variable named by the key.
// It returns an error, if any.
func Setenv(key, value string) error {
	err := syscall.Setenv(key, value)
	if err != nil {
		return NewSyscallError("setenv", err)
	}
	return nil
}

// Unsetenv unsets a single environment variable.
func Unsetenv(key string) error {
	return syscall.Unsetenv(key)
}

// Clearenv deletes all environment variables.
func Clearenv() {
	syscall.Clearenv()
}

// Environ returns a copy of strings representing the environment,
// in the form "key=value".
func Environ() []string {
	return syscall.Environ()
}
<|endoftext|>


<commit_before>package srv

import (
	"testing"
)
import (
	"github.com/uniqush/uniqush-push/push"
)

func testToGCMPayload(t *testing.T, postData map[string]string, regIds []string, expectedPayload string) {
	notif := push.NewEmptyNotification()
	notif.Data = postData
	// Create a push service, just for the sake of realistically testing building payloads
	stubPushService := newGCMPushService()
	defer stubPushService.Finalize()
	payload, err := stubPushService.ToCMPayload(notif, regIds)
	if err != nil {
		t.Fatalf("Encountered error %v\n", err)
	}
	if string(payload) != expectedPayload {
		t.Errorf("Expected %s, got %s", expectedPayload, string(payload))
	}
}

func TestToGCMPayloadWithRawPayload(t *testing.T) {
	postData := map[string]string{
		"msggroup":            "somegroup",
		"uniqush.payload.gcm": `{"message":{"key": {},"x":"y"},"other":{}}`,
		"foo": "bar",
	}
	regIds := []string{"CAFE1-FF", "42-607"}
	expectedPayload := `{"registration_ids":["CAFE1-FF","42-607"],"collapse_key":"somegroup","data":{"message":{"key":{},"x":"y"},"other":{}},"time_to_live":3600}`
	testToGCMPayload(t, postData, regIds, expectedPayload)
}

func TestToGCMPayloadWithRawUnescapedPayload(t *testing.T) {
	postData := map[string]string{
		"msggroup":            "somegroup",
		"uniqush.payload.gcm": `{"message":{"key": {},"x":"<a☃?>\"'"},"other":{}}`,
		"foo": "bar",
	}
	regIds := []string{"CAFE1-FF", "42-607"}
	expectedPayload := `{"registration_ids":["CAFE1-FF","42-607"],"collapse_key":"somegroup","data":{"message":{"key":{},"x":"<a☃?>\"'"},"other":{}},"time_to_live":3600}`
	testToGCMPayload(t, postData, regIds, expectedPayload)
}

func TestToGCMPayloadWithCommonParameters(t *testing.T) {
	postData := map[string]string{
		"msggroup":  "somegroup",
		"other":     "value",
		"other.foo": "bar",
		"ttl":       "5",
		// GCM module should ignore anything it doesn't recognize begining with "uniqush.", those are reserved.
		"uniqush.payload.apns": "{}",
		"uniqush.foo":          "foo",
	}
	regIds := []string{"CAFE1-FF", "42-607"}
	expectedPayload := `{"registration_ids":["CAFE1-FF","42-607"],"collapse_key":"somegroup","data":{"other":"value","other.foo":"bar"},"time_to_live":5}`
	testToGCMPayload(t, postData, regIds, expectedPayload)
}

// Test that it will be encoded properly if uniqush.payload.gcm is provided instead of uniqush.payload
func TestToGCMPayloadNewWay(t *testing.T) {
	postData := map[string]string{
		"msggroup":            "somegroup",
		"uniqush.payload.gcm": `{"message":{"aPushType":{"foo":"bar","other":"value"},"gcm":{},"others":{"type":"aPushType"}}}`,
	}
	regIds := []string{"CAFE1-FF", "42-607"}
	expectedPayload := `{"registration_ids":["CAFE1-FF","42-607"],"collapse_key":"somegroup","data":{"message":{"aPushType":{"foo":"bar","other":"value"},"gcm":{},"others":{"type":"aPushType"}}},"time_to_live":3600}`
	testToGCMPayload(t, postData, regIds, expectedPayload)
}

// Test that the push type isn't used as a fallback collapse key or anything else.
func TestToGCMPayloadUsesMsggroupForCollapseKey(t *testing.T) {
	postData := map[string]string{
		"uniqush.payload.gcm": `{"message":{"aPushType":{"foo":"bar","other":"value"},"gcm":{},"others":{"type":"aPushType"}}}`,
		"msggroup":            "AMsgGroup",
	}
	regIds := []string{"CAFE1-FF", "42-607"}
	expectedPayload := `{"registration_ids":["CAFE1-FF","42-607"],"collapse_key":"AMsgGroup","data":{"message":{"aPushType":{"foo":"bar","other":"value"},"gcm":{},"others":{"type":"aPushType"}}},"time_to_live":3600}`
	testToGCMPayload(t, postData, regIds, expectedPayload)
}

// Test the return value of Name()
func TestGCMPushServiceName(t *testing.T) {
	stubPushService := newGCMPushService()
	defer stubPushService.Finalize()
	name := stubPushService.Name()
	if name != "gcm" {
		t.Errorf("Expected %s, got %s", "gcm", name)
	}
}
<commit_msg>Use unescaped JSON for GCM/FCM, otherwise, "<" is needlessly escaped<commit_after>package srv

import (
	"testing"
)
import (
	"github.com/uniqush/uniqush-push/push"
)

func testToGCMPayload(t *testing.T, postData map[string]string, regIds []string, expectedPayload string) {
	notif := push.NewEmptyNotification()
	notif.Data = postData
	// Create a push service, just for the sake of realistically testing building payloads
	stubPushService := newGCMPushService()
	defer stubPushService.Finalize()
	payload, err := stubPushService.ToCMPayload(notif, regIds)
	if err != nil {
		t.Fatalf("Encountered error %v\n", err)
	}
	if string(payload) != expectedPayload {
		t.Errorf("Expected %s, got %s", expectedPayload, string(payload))
	}
}

func TestToGCMPayloadWithRawPayload(t *testing.T) {
	postData := map[string]string{
		"msggroup":            "somegroup",
		"uniqush.payload.gcm": `{"message":{"key": {},"x":"y"},"other":{}}`,
		"foo": "bar",
	}
	regIds := []string{"CAFE1-FF", "42-607"}
	expectedPayload := `{"registration_ids":["CAFE1-FF","42-607"],"collapse_key":"somegroup","data":{"message":{"key":{},"x":"y"},"other":{}},"time_to_live":3600}`
	testToGCMPayload(t, postData, regIds, expectedPayload)
}

func TestToGCMPayloadWithRawUnescapedPayload(t *testing.T) {
	postData := map[string]string{
		"msggroup":            "somegroup",
		"uniqush.payload.gcm": `{"message":{"key": {},"x":"<a☃?>\"'"},"other":{}}`,
		"foo": "bar",
	}
	regIds := []string{"CAFE1-FF", "42-607"}
	expectedPayload := `{"registration_ids":["CAFE1-FF","42-607"],"collapse_key":"somegroup","data":{"message":{"key":{},"x":"<a☃?>\"'"},"other":{}},"time_to_live":3600}`
	testToGCMPayload(t, postData, regIds, expectedPayload)
}

func TestToGCMPayloadWithCommonParameters(t *testing.T) {
	postData := map[string]string{
		"msggroup":            "somegroup",
		"uniqush.payload.gcm": `{"message":{"key": {},"x":"<a☃?>\"'"},"other":{}}`,
		"foo": "bar",
	}
	regIds := []string{"CAFE1-FF", "42-607"}
	expectedPayload := `{"registration_ids":["CAFE1-FF","42-607"],"collapse_key":"somegroup","data":{"message":{"key":{},"x":"<a☃?>\"'"},"other":{}},"time_to_live":3600}`
	testToGCMPayload(t, postData, regIds, expectedPayload)
}

func TestToGCMPayloadWithCommonParametersV2(t *testing.T) {
	postData := map[string]string{
		"msggroup":  "somegroup",
		"other":     "value",
		"other.foo": "bar",
		"ttl":       "5",
		// GCM module should ignore anything it doesn't recognize begining with "uniqush.", those are reserved.
		"uniqush.payload.apns": "{}",
		"uniqush.foo":          "foo",
	}
	regIds := []string{"CAFE1-FF", "42-607"}
	expectedPayload := `{"registration_ids":["CAFE1-FF","42-607"],"collapse_key":"somegroup","data":{"other":"value","other.foo":"bar"},"time_to_live":5}`
	testToGCMPayload(t, postData, regIds, expectedPayload)
}

// Test that it will be encoded properly if uniqush.payload.gcm is provided instead of uniqush.payload
func TestToGCMPayloadNewWay(t *testing.T) {
	postData := map[string]string{
		"msggroup":            "somegroup",
		"uniqush.payload.gcm": `{"message":{"aPushType":{"foo":"bar","other":"value"},"gcm":{},"others":{"type":"aPushType"}}}`,
	}
	regIds := []string{"CAFE1-FF", "42-607"}
	expectedPayload := `{"registration_ids":["CAFE1-FF","42-607"],"collapse_key":"somegroup","data":{"message":{"aPushType":{"foo":"bar","other":"value"},"gcm":{},"others":{"type":"aPushType"}}},"time_to_live":3600}`
	testToGCMPayload(t, postData, regIds, expectedPayload)
}

// Test that the push type isn't used as a fallback collapse key or anything else.
func TestToGCMPayloadUsesMsggroupForCollapseKey(t *testing.T) {
	postData := map[string]string{
		"uniqush.payload.gcm": `{"message":{"aPushType":{"foo":"bar","other":"value"},"gcm":{},"others":{"type":"aPushType"}}}`,
		"msggroup":            "AMsgGroup",
	}
	regIds := []string{"CAFE1-FF", "42-607"}
	expectedPayload := `{"registration_ids":["CAFE1-FF","42-607"],"collapse_key":"AMsgGroup","data":{"message":{"aPushType":{"foo":"bar","other":"value"},"gcm":{},"others":{"type":"aPushType"}}},"time_to_live":3600}`
	testToGCMPayload(t, postData, regIds, expectedPayload)
}

// Test the return value of Name()
func TestGCMPushServiceName(t *testing.T) {
	stubPushService := newGCMPushService()
	defer stubPushService.Finalize()
	name := stubPushService.Name()
	if name != "gcm" {
		t.Errorf("Expected %s, got %s", "gcm", name)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package context

import (
	"io"
	"net/http"
	"path/filepath"
	"time"

	"github.com/issue9/upload"
	"github.com/issue9/utils"
)

// ServeFile 提供文件下载
//
// 文件可能提供连续的下载功能，其状态码是未定的，
// 所以提供了一个类似于 Render 的变体专门用于下载功能。
//
// path 指向本地文件的地址；
// name 下载时，显示的文件，若为空，则直接使用 path 中的文件名部分；
// headers 额外显示的报头内容。
func (ctx *Context) ServeFile(path, name string, headers map[string]string) {
	if !utils.FileExists(path) {
		ctx.Exit(http.StatusNotFound)
	}

	if name == "" {
		name = filepath.Base(path)
	}

	for k, v := range headers {
		ctx.Response.Header().Set(k, v)
	}
	// BUG(caixw): 无法自定义 ServeFile 中的错误处理
	http.ServeFile(ctx.Response, ctx.Request, path)
}

// ServeFileBuffer 将一块内存中的内容转换为文件提供下载
//
// 文件可能提供连续的下载功能，其状态码是未定的，
// 所以提供了一个类似于 Render 的变体专门用于下载功能。
//
// buf 实现 io.ReadSeeker 接口的内存块；
// name 下载时，显示的文件；
// headers 文件报头内容。
func (ctx *Context) ServeFileBuffer(buf io.ReadSeeker, name string, headers map[string]string) {
	for k, v := range headers {
		ctx.Response.Header().Set(k, v)
	}

	// BUG(caixw): 无法自定义 ServeContent 中的错误处理
	http.ServeContent(ctx.Response, ctx.Request, name, time.Now(), buf)
}

// Upload 执行上传文件的相关操作。
//
// 返回的是文件列表
func (ctx *Context) Upload(field string, u *upload.Upload) ([]string, error) {
	return u.Do(field, ctx.Request)
}
<commit_msg>修改 context/file 的实现<commit_after>// Copyright 2018 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package context

import (
	"io"
	"net/http"
	"path/filepath"
	"time"

	"github.com/issue9/upload"
	"github.com/issue9/utils"

	"github.com/issue9/web/internal/fileserver"
)

// ServeFile 提供文件下载
//
// 文件可能提供连续的下载功能，其状态码是未定的，
// 所以提供了一个类似于 Render 的变体专门用于下载功能。
//
// path 指向本地文件的地址；
// name 下载时，显示的文件，若为空，则直接使用 path 中的文件名部分；
// headers 额外显示的报头内容。
func (ctx *Context) ServeFile(path, name string, headers map[string]string) {
	if !utils.FileExists(path) {
		ctx.Exit(http.StatusNotFound)
	}

	if name == "" {
		name = filepath.Base(path)
	}

	for k, v := range headers {
		ctx.Response.Header().Set(k, v)
	}
	fileserver.ServeFile(ctx.Response, ctx.Request, path)
}

// ServeFileBuffer 将一块内存中的内容转换为文件提供下载
//
// 文件可能提供连续的下载功能，其状态码是未定的，
// 所以提供了一个类似于 Render 的变体专门用于下载功能。
//
// buf 实现 io.ReadSeeker 接口的内存块；
// name 下载时，显示的文件；
// headers 文件报头内容。
func (ctx *Context) ServeFileBuffer(buf io.ReadSeeker, name string, headers map[string]string) {
	for k, v := range headers {
		ctx.Response.Header().Set(k, v)
	}

	fileserver.ServeContent(ctx.Response, ctx.Request, name, time.Now(), buf)
}

// Upload 执行上传文件的相关操作。
//
// 返回的是文件列表
func (ctx *Context) Upload(field string, u *upload.Upload) ([]string, error) {
	return u.Do(field, ctx.Request)
}
<|endoftext|>


<commit_before>package dotweb

import (
	"testing"
	"github.com/devfeel/dotweb/test"
)

type Animal struct{
	Hair     string
	HasMouth bool
}

func TestWriteJsonpBlob(t *testing.T) {
	param := &InitContextParam{
		t,
		&Animal{},
		"",
		test.ToDefault,
	}

	//init param
	context := initResponseContext(param)

	excepted:=&Animal{
		"Black",
		true,
	}

	context.WriteJsonp("jsonCallBack",excepted)

	t.Log(string(context.response.body))
}<commit_msg>context.go测试用例.<commit_after>package dotweb

import (
	"testing"
	"github.com/devfeel/dotweb/test"
	"encoding/json"
	"fmt"
)

type Animal struct{
	Hair     string
	HasMouth bool
}

//normal jsonp
func TestWriteJsonp(t *testing.T) {
	param := &InitContextParam{
		t,
		&Animal{},
		"",
		test.ToDefault,
	}

	//init param
	context := initResponseContext(param)

	exceptedObject:=&Animal{
		"Black",
		true,
	}

	callback:="jsonCallBack"

	//call function
	context.WriteJsonp(callback,exceptedObject)

	//check result

	//header
	contentType:=context.response.header.Get(HeaderContentType)
	test.Equal(t,contentType,MIMEApplicationJavaScriptCharsetUTF8)

	//body
	body:=string(context.response.body)

	animalJson,err:=json.Marshal(exceptedObject)
	test.Nil(t,err)

	excepted:=fmt.Sprint(callback,"(",string(animalJson),");")

	test.Equal(t,body,excepted)
}<|endoftext|>


<commit_before>// Copyright 2016 Marcel Gotsch. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package goserv

import "testing"

func TestContext(t *testing.T) {
	c := newContext()

	if c.Exists("key") {
		t.Error("The key 'key' should not exist in context")
	}

	if c.Get("key") != nil {
		t.Error("Expected nil value for 'key'")
	}

	c.Set("key", "value")

	if !c.Exists("key") {
		t.Error("Expected 'key' to exist")
	}

	if c.Get("key").(string) != "value" {
		t.Error("Expected 'key' to be 'value'")
	}

	if c.Get("key2") != nil {
		t.Error("Expected nil value for 'key2'")
	}
}
<commit_msg>Remove context_test<commit_after><|endoftext|>


<commit_before>package liquid

import (
	"fmt"
	"testing"
	"time"
)

func TestVariables(t *testing.T) {
	ctx := newContext()
	ctx.scopes.push()

	ctx.Assign("string", "string")
	v, err := ctx.Get("string")
	if err != nil {
		t.Fatal(err.Error())
	}
	if v != "string" {
		t.Fatal(fmt.Sprintf(`Assigned "string" to Context but Get returns %+v`, v))
	}

	ctx.Assign("num", 5)
	v, err = ctx.Get("num")
	if err != nil {
		t.Fatal(err.Error())
	}
	if v != 5 {
		t.Fatal(fmt.Sprintf(`Assigned 5 to Context but Get returns %+v`, v))
	}

	now := time.Now()
	ctx.Assign("time", now)
	v, err = ctx.Get("time")
	if err != nil {
		t.Fatal(err.Error())
	}
	if v != now {
		t.Fatal(fmt.Sprintf(`Assigned a time.Time to Context but Get returns %+v`, v))
	}

	ctx.Assign("nil", nil)
	v, err = ctx.Get("nil")
	if err != nil {
		t.Fatal(err.Error())
	}
	if v != nil {
		t.Fatal(fmt.Sprintf(`Assigned nil to Context but Get returns %+v`, v))
	}
}

func TestVariablesNotExisting(t *testing.T) {
	ctx := newContext()
	ctx.scopes.push()

	v, err := ctx.Get("wat")
	if err != nil {
		t.Fatal(err.Error())
	}
	if v != nil {
		t.Fatal(`Non-existent variable fetched from Context was not nil`)
	}
}

func TestHyphenatedVariable(t *testing.T) {
	ctx := newContext()
	ctx.scopes.push()

	err := ctx.Assign("oh-my", "godz")
	if err != nil {
		t.Fatal(err.Error())
	}

	v, err := ctx.Get("oh-my")
	if err != nil {
		t.Fatal(err.Error())
	}

	if v != "godz" {
		t.Fatal(fmt.Sprintf(`expected "godz", got %+v`, v))
	}
}

// Copied the name from the ruby implementation but it's badly named.
// This actually tests if you can _access_ a variable set in a higher scope.
func TestAddItemInOuterScope(t *testing.T) {
	ctx := newContext()
	ctx.scopes.push()

	err := ctx.Assign("test", "test")
	if err != nil {
		t.Fatal(err.Error())
	}

	ctx.scopes.push()

	v, err := ctx.Get("test")
	if err != nil {
		t.Fatal(err.Error())
	}

	if v != "test" {
		t.Fatal(fmt.Sprintf(`Expected "test" but got %+v when looking up variable defined in higher scope"`, v))
	}

	ctx.scopes.pop()
	v, err = ctx.Get("test")
	if err != nil {
		t.Fatal(err.Error())
	}
	if v != "test" {
		t.Fatal(fmt.Sprintf(`Expected "test" but got %+v when looking up variable`, v))
	}

}

//scopeStack tests

func TestScopeStack(t *testing.T) {
	s := scopeStack{}

	vars, err := s.curr()
	if err == nil {
		t.Fatal(`empty scopeStack did not throw error on curr()`)
	}

	if vars != nil {
		t.Fatal(`empty scopeStack returned a Var`)
	}

	s.push()

	vars, err = s.curr()
	if err != nil {
		t.Fatal(err.Error())
	}
	if vars == nil {
		t.Fatal(`nil Var returned from scopeStack`)
	}

	if len(s) != 1 {
		t.Fatal(`unexpected scopeStack length`)
	}
}
<commit_msg>Add test to verify scoping only goes one way<commit_after>package liquid

import (
	"fmt"
	"testing"
	"time"
)

func TestVariables(t *testing.T) {
	ctx := newContext()
	ctx.scopes.push()

	ctx.Assign("string", "string")
	v, err := ctx.Get("string")
	if err != nil {
		t.Fatal(err.Error())
	}
	if v != "string" {
		t.Fatal(fmt.Sprintf(`Assigned "string" to Context but Get returns %+v`, v))
	}

	ctx.Assign("num", 5)
	v, err = ctx.Get("num")
	if err != nil {
		t.Fatal(err.Error())
	}
	if v != 5 {
		t.Fatal(fmt.Sprintf(`Assigned 5 to Context but Get returns %+v`, v))
	}

	now := time.Now()
	ctx.Assign("time", now)
	v, err = ctx.Get("time")
	if err != nil {
		t.Fatal(err.Error())
	}
	if v != now {
		t.Fatal(fmt.Sprintf(`Assigned a time.Time to Context but Get returns %+v`, v))
	}

	ctx.Assign("nil", nil)
	v, err = ctx.Get("nil")
	if err != nil {
		t.Fatal(err.Error())
	}
	if v != nil {
		t.Fatal(fmt.Sprintf(`Assigned nil to Context but Get returns %+v`, v))
	}
}

func TestVariablesNotExisting(t *testing.T) {
	ctx := newContext()
	ctx.scopes.push()

	v, err := ctx.Get("wat")
	if err != nil {
		t.Fatal(err.Error())
	}
	if v != nil {
		t.Fatal(`Non-existent variable fetched from Context was not nil`)
	}
}

func TestHyphenatedVariable(t *testing.T) {
	ctx := newContext()
	ctx.scopes.push()

	err := ctx.Assign("oh-my", "godz")
	if err != nil {
		t.Fatal(err.Error())
	}

	v, err := ctx.Get("oh-my")
	if err != nil {
		t.Fatal(err.Error())
	}

	if v != "godz" {
		t.Fatal(fmt.Sprintf(`expected "godz", got %+v`, v))
	}
}

// Copied the name from the ruby implementation but it's badly named.
// This actually tests if you can _access_ a variable set in a higher scope.
func TestAddItemInOuterScope(t *testing.T) {
	ctx := newContext()
	ctx.scopes.push()

	err := ctx.Assign("test", "test")
	if err != nil {
		t.Fatal(err.Error())
	}

	ctx.scopes.push()

	v, err := ctx.Get("test")
	if err != nil {
		t.Fatal(err.Error())
	}

	if v != "test" {
		t.Fatal(fmt.Sprintf(`Expected "test" but got %+v when looking up variable defined in higher scope"`, v))
	}

	ctx.scopes.pop()
	v, err = ctx.Get("test")
	if err != nil {
		t.Fatal(err.Error())
	}
	if v != "test" {
		t.Fatal(fmt.Sprintf(`Expected "test" but got %+v when looking up variable`, v))
	}
}

func TestAddItemInInnerScope(t *testing.T) {
	ctx := newContext()
	ctx.scopes.push() // Pristine
	ctx.scopes.push() // To be defiled with our variables

	err := ctx.Assign("test", "test")
	if err != nil {
		t.Fatal(err.Error())
	}

	v, err := ctx.Get("test")
	if err != nil {
		t.Fatal(err.Error())
	}
	if v != "test" {
		t.Fatal(fmt.Sprintf(`Expected "test" but got %+v when looking up variable`, v))
	}

	ctx.scopes.pop()

	v, err = ctx.Get("test")
	if err != nil {
		t.Fatal(err.Error())
	}
	if v != nil {
		t.Fatal(fmt.Sprintf(`Got %+v from scope higher scope than it was defined in!`, v))
	}
}

//scopeStack tests

func TestScopeStack(t *testing.T) {
	s := scopeStack{}

	vars, err := s.curr()
	if err == nil {
		t.Fatal(`empty scopeStack did not throw error on curr()`)
	}

	if vars != nil {
		t.Fatal(`empty scopeStack returned a Var`)
	}

	s.push()

	vars, err = s.curr()
	if err != nil {
		t.Fatal(err.Error())
	}
	if vars == nil {
		t.Fatal(`nil Var returned from scopeStack`)
	}

	if len(s) != 1 {
		t.Fatal(`unexpected scopeStack length`)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

const shutdownGraceTime = 3 * time.Second

var flagPort int
var flagConcurrency string
var flagRestart bool
var shutdownNow bool

var processes = map[string]*Process{}
var shutdown_mutex = new(sync.Mutex)
var wg sync.WaitGroup

var cmdStart = &Command{
	Run:   runStart,
	Usage: "start [process name] [-f procfile] [-e env] [-c concurrency] [-p port] [-r]",
	Short: "Start the application",
	Long: `
Start the application specified by a Procfile (defaults to ./Procfile)

Examples:

  forego start
  forego start web
  forego start -f Procfile.test -e .env.test
`,
}

func init() {
	cmdStart.Flag.StringVar(&flagProcfile, "f", "Procfile", "procfile")
	cmdStart.Flag.StringVar(&flagEnv, "e", "", "env")
	cmdStart.Flag.IntVar(&flagPort, "p", 5000, "port")
	cmdStart.Flag.StringVar(&flagConcurrency, "c", "", "concurrency")
	cmdStart.Flag.BoolVar(&flagRestart, "r", false, "restart")
}

func parseConcurrency(value string) (map[string]int, error) {
	concurrency := map[string]int{}
	if strings.TrimSpace(value) == "" {
		return concurrency, nil
	}

	parts := strings.Split(value, ",")
	for _, part := range parts {
		if !strings.Contains(part, "=") {
			return concurrency, errors.New("Parsing concurency")
		}

		nameValue := strings.Split(part, "=")
		n, v := strings.TrimSpace(nameValue[0]), strings.TrimSpace(nameValue[1])
		if n == "" || v == "" {
			return concurrency, errors.New("Parsing concurency")
		}

		numProcs, err := strconv.ParseInt(v, 10, 16)
		if err != nil {
			return concurrency, err
		}

		concurrency[n] = int(numProcs)
	}
	return concurrency, nil
}

func startProcess(idx, procNum int, proc ProcfileEntry, env Env, of *OutletFactory) {
	shutdown_mutex.Lock()
	wg.Add(1)
	port := flagPort + (idx * 100)
	ps := NewProcess(proc.Command, env)
	procName := strings.Join([]string{
		proc.Name,
		strconv.FormatInt(int64(procNum+1), 10)}, ".")
	processes[procName] = ps
	ps.Env["PORT"] = strconv.Itoa(port)
	ps.Root = filepath.Dir(flagProcfile)
	ps.Stdin = nil
	ps.Stdout = of.CreateOutlet(procName, idx, false)
	ps.Stderr = of.CreateOutlet(procName, idx, true)
	ps.Start()
	of.SystemOutput(fmt.Sprintf("starting %s on port %d", procName, port))
	go func(proc ProcfileEntry, ps *Process) {
		ps.Wait()

		if flagRestart && !shutdownNow {
			delete(processes, proc.Name)
			startProcess(idx, procNum, proc, env, of)
			wg.Done()
			return
		}

		wg.Done()
		delete(processes, proc.Name)
		ShutdownProcesses(of)

	}(proc, ps)
	shutdown_mutex.Unlock()
}

func runStart(cmd *Command, args []string) {
	root := filepath.Dir(flagProcfile)

	if flagEnv == "" {
		flagEnv = filepath.Join(root, ".env")
	}

	pf, err := ReadProcfile(flagProcfile)
	handleError(err)

	env, err := ReadEnv(flagEnv)
	handleError(err)

	concurrency, err := parseConcurrency(flagConcurrency)
	handleError(err)

	of := NewOutletFactory()
	of.Padding = pf.LongestProcessName()

	handler := make(chan os.Signal, 1)
	signal.Notify(handler, os.Interrupt)

	go func() {
		for sig := range handler {
			switch sig {
			case os.Interrupt:
				shutdownNow = true
				fmt.Println("      | ctrl-c detected")
				go func() { ShutdownProcesses(of) }()
			}
		}
	}()

	var singleton string = ""
	if len(args) > 0 {
		singleton = args[0]
		if !pf.HasProcess(singleton) {
			of.ErrorOutput(fmt.Sprintf("no such process: %s", singleton))
		}
	}

	for idx, proc := range pf.Entries {
		numProcs := 1
		if value, ok := concurrency[proc.Name]; ok {
			numProcs = value
		}
		for i := 0; i < numProcs; i++ {
			if (singleton == "") || (singleton == proc.Name) {
				startProcess(idx, i, proc, env, of)
			}
		}
	}

	wg.Wait()
}
<commit_msg>Simplify procName calculation<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

const shutdownGraceTime = 3 * time.Second

var flagPort int
var flagConcurrency string
var flagRestart bool
var shutdownNow bool

var processes = map[string]*Process{}
var shutdown_mutex = new(sync.Mutex)
var wg sync.WaitGroup

var cmdStart = &Command{
	Run:   runStart,
	Usage: "start [process name] [-f procfile] [-e env] [-c concurrency] [-p port] [-r]",
	Short: "Start the application",
	Long: `
Start the application specified by a Procfile (defaults to ./Procfile)

Examples:

  forego start
  forego start web
  forego start -f Procfile.test -e .env.test
`,
}

func init() {
	cmdStart.Flag.StringVar(&flagProcfile, "f", "Procfile", "procfile")
	cmdStart.Flag.StringVar(&flagEnv, "e", "", "env")
	cmdStart.Flag.IntVar(&flagPort, "p", 5000, "port")
	cmdStart.Flag.StringVar(&flagConcurrency, "c", "", "concurrency")
	cmdStart.Flag.BoolVar(&flagRestart, "r", false, "restart")
}

func parseConcurrency(value string) (map[string]int, error) {
	concurrency := map[string]int{}
	if strings.TrimSpace(value) == "" {
		return concurrency, nil
	}

	parts := strings.Split(value, ",")
	for _, part := range parts {
		if !strings.Contains(part, "=") {
			return concurrency, errors.New("Parsing concurency")
		}

		nameValue := strings.Split(part, "=")
		n, v := strings.TrimSpace(nameValue[0]), strings.TrimSpace(nameValue[1])
		if n == "" || v == "" {
			return concurrency, errors.New("Parsing concurency")
		}

		numProcs, err := strconv.ParseInt(v, 10, 16)
		if err != nil {
			return concurrency, err
		}

		concurrency[n] = int(numProcs)
	}
	return concurrency, nil
}

func startProcess(idx, procNum int, proc ProcfileEntry, env Env, of *OutletFactory) {
	shutdown_mutex.Lock()
	wg.Add(1)
	port := flagPort + (idx * 100)
	ps := NewProcess(proc.Command, env)
	procName := fmt.Sprint(proc.Name, ".", procNum+1)
	processes[procName] = ps
	ps.Env["PORT"] = strconv.Itoa(port)
	ps.Root = filepath.Dir(flagProcfile)
	ps.Stdin = nil
	ps.Stdout = of.CreateOutlet(procName, idx, false)
	ps.Stderr = of.CreateOutlet(procName, idx, true)
	ps.Start()

	of.SystemOutput(fmt.Sprintf("starting %s on port %d", procName, port))
	go func(proc ProcfileEntry, ps *Process) {
		ps.Wait()

		if flagRestart && !shutdownNow {
			delete(processes, proc.Name)
			startProcess(idx, procNum, proc, env, of)
			wg.Done()
			return
		}

		wg.Done()
		delete(processes, proc.Name)
		ShutdownProcesses(of)

	}(proc, ps)
	shutdown_mutex.Unlock()
}

func runStart(cmd *Command, args []string) {
	root := filepath.Dir(flagProcfile)

	if flagEnv == "" {
		flagEnv = filepath.Join(root, ".env")
	}

	pf, err := ReadProcfile(flagProcfile)
	handleError(err)

	env, err := ReadEnv(flagEnv)
	handleError(err)

	concurrency, err := parseConcurrency(flagConcurrency)
	handleError(err)

	of := NewOutletFactory()
	of.Padding = pf.LongestProcessName()

	handler := make(chan os.Signal, 1)
	signal.Notify(handler, os.Interrupt)

	go func() {
		for sig := range handler {
			switch sig {
			case os.Interrupt:
				shutdownNow = true
				fmt.Println("      | ctrl-c detected")
				go func() { ShutdownProcesses(of) }()
			}
		}
	}()

	var singleton string = ""
	if len(args) > 0 {
		singleton = args[0]
		if !pf.HasProcess(singleton) {
			of.ErrorOutput(fmt.Sprintf("no such process: %s", singleton))
		}
	}

	for idx, proc := range pf.Entries {
		numProcs := 1
		if value, ok := concurrency[proc.Name]; ok {
			numProcs = value
		}
		for i := 0; i < numProcs; i++ {
			if (singleton == "") || (singleton == proc.Name) {
				startProcess(idx, i, proc, env, of)
			}
		}
	}

	wg.Wait()
}
<|endoftext|>


<commit_before>package gapi

const (
	getTeamJSON = `
{
  "id": 1,
  "orgId": 1,
  "name": "MyTestTeam",
  "email": "",
  "created": "2017-12-15T10:40:45+01:00",
  "updated": "2017-12-15T10:40:45+01:00"
}
`
	addTeamsJSON = `
{
  "message":"Team created",
  "teamId":2
}
`
	updateTeamJSON     = `{ "message":"Team updated"}`
	deleteTeamJSON     = `{"message":"Team deleted"}`
	getTeamMembersJSON = `
[
  {
    "orgId": 1,
    "teamId": 1,
    "userId": 3,
    "email": "user1@email.com",
    "login": "user1",
    "avatarUrl": "\/avatar\/1b3c32f6386b0185c40d359cdc733a79"
  },
  {
    "orgId": 1,
    "teamId": 1,
    "userId": 2,
    "email": "user2@email.com",
    "login": "user2",
    "avatarUrl": "\/avatar\/cad3c68da76e45d10269e8ef02f8e73e"
  }
]
`
	addTeamMemberJSON = `
{
  "userId": 2
}
`
	removeMemberFromTeamJSON = `{"message":"Team Member removed"}`
	getTeamPreferencesJSON   = `
{
  "theme": "",
  "homeDashboardId": 0,
  "timezone": ""
}
`
	updateTeamPreferencesJSON = `
{
  "message":"Preferences updated"
}
`
)
<commit_msg>add test<commit_after>package gapi

import (
	"github.com/gobs/pretty"
	"testing"
)

const (
	getTeamJSON = `
{
  "id": 1,
  "orgId": 1,
  "name": "MyTestTeam",
  "email": "",
  "avatarUrl": "avatar/abcdef",
  "memberCount": 1,
  "permission": 0
}
`
	addTeamsJSON = `
{
  "message":"Team created",
  "teamId":2
}
`
	updateTeamJSON     = `{ "message":"Team updated"}`
	deleteTeamJSON     = `{"message":"Team deleted"}`
	getTeamMembersJSON = `
[
  {
    "orgId": 1,
    "teamId": 1,
    "userId": 3,
    "email": "user1@email.com",
    "login": "user1",
    "avatarUrl": "\/avatar\/1b3c32f6386b0185c40d359cdc733a79"
  },
  {
    "orgId": 1,
    "teamId": 1,
    "userId": 2,
    "email": "user2@email.com",
    "login": "user2",
    "avatarUrl": "\/avatar\/cad3c68da76e45d10269e8ef02f8e73e"
  }
]
`
	addTeamMemberJSON = `
{
  "userId": 2
}
`
	removeMemberFromTeamJSON = `{"message":"Team Member removed"}`
	getTeamPreferencesJSON   = `
{
  "theme": "",
  "homeDashboardId": 0,
  "timezone": ""
}
`
	updateTeamPreferencesJSON = `
{
  "message":"Preferences updated"
}
`
)

func TestTeam(t *testing.T) {
	server, client := gapiTestTools(200, getTeamJSON)
	defer server.Close()

	id := int64(1)
	resp, err := client.Team(id)
	if err != nil {
		t.Error(err)
	}

	t.Log(pretty.PrettyFormat(resp))

	expect := &Team{
		Id:          1,
		OrgId:       1,
		Name:        "MyTestTeam",
		Email:       "",
		AvatarUrl:   "avatar/abcdef",
		MemberCount: 1,
		Permission:  0,
	}
	t.Run("check data", func(t *testing.T) {
		if expect.Id != resp.Id || expect.Name != expect.Name {
			t.Error("Not correctly data")
		}
	})
}

func TestAddTeam(t *testing.T) {
	server, client := gapiTestTools(200, addTeamsJSON)
	defer server.Close()

	name := "TestTeam"
	email := ""

	err := client.AddTeam(name, email)
	if err != nil {
		t.Error(err)
	}
}

func TestUpdateTeam(t *testing.T) {
	server, client := gapiTestTools(200, addTeamsJSON)
	defer server.Close()

	id := int64(1)
	name := "TestTeam"
	email := ""

	err := client.UpdateTeam(id, name, email)
	if err != nil {
		t.Error(err)
	}
}

func TestDeleteTeam(t *testing.T) {
	server, client := gapiTestTools(200, addTeamsJSON)
	defer server.Close()

	id := int64(1)

	err := client.DeleteTeam(id)
	if err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"hash"
	"io"
	"sync/atomic"

	"github.com/montanaflynn/stats"
	"github.com/pkg/errors"
)

// SegmentStat holds stats for a single segment
type SegmentStat struct {
	SeqNum uint64 // First occurrence of this segment in the file
	Length int    // Length of segment
	Freq   int    // How many times this segment occurred in the file
}

// ParseStats holds stats about the parsed file
type ParseStats struct {
	Cutpoints   []uint64               // indices at which we have cutpoints
	SegLengths  []uint64               // lengths of segments (between cutpoints)
	SegHashes   map[string]SegmentStat // map[crypto hash of seg] -> SegmentStat
	BytesParsed uint64                 // number of bytes parsed

	// internal:
	segNum    uint64    // tracks the segment numbers we've issued
	segHasher hash.Hash // used to generate the crypto hash of segments
}

// NewParseStats returns an initialized ParseStats struct
func NewParseStats(hasher hash.Hash) *ParseStats {
	return &ParseStats{
		SegHashes: make(map[string]SegmentStat),
		segNum:    uint64(0),
		segHasher: hasher,
	}
}

// UpdateStats updates the ParseStats with stats for the specified chunk
func (s *ParseStats) UpdateStats(segment []byte) string {

	// Sprint'ing the hash sum causes an allocation that is unnecessary and
	// is completely avoidable.
	//segHash := fmt.Sprintf("%X", s.segHasher.Sum(segment))
	segHash := string(s.segHasher.Sum(segment))
	segStat, there := s.SegHashes[segHash]
	if there {
		segStat.Freq++
	} else {
		//segStat = SegmentStat{SeqNum: s.segNum, Length: len(segment), Freq: 1}
		segStat.Freq = 1
		segStat.Length = len(segment)
		segStat.SeqNum = atomic.AddUint64(&s.segNum, 1)
	}
	s.SegHashes[segHash] = segStat

	// Additional book keeping (TODO: allow this to be disabled)
	s.Cutpoints = append(s.Cutpoints, s.BytesParsed+uint64(len(segment)))
	s.SegLengths = append(s.SegLengths, uint64(len(segment)))
	s.BytesParsed += uint64(len(segment))

	return segHash
}

// Print prints the specified ParseStats on the given output (io.Writer)
//
func (s ParseStats) Print(out io.Writer) error {
	segLens := make([]float64, 0, len(s.SegLengths))
	for _, s := range s.SegLengths {
		segLens = append(segLens, float64(s))
	}

	med, err := stats.Median(segLens)
	if err != nil {
		return errors.Wrapf(err, "Failed compute median")
	}
	max, err := stats.Max(segLens)
	if err != nil {
		return errors.Wrapf(err, "Failed to compute max")
	}
	min, err := stats.Min(segLens)
	if err != nil {
		return errors.Wrapf(err, "Failed to compute min")
	}
	mea, err := stats.Mean(segLens)
	if err != nil {
		return errors.Wrapf(err, "Failed to compute mean")
	}

	dupCount := 0
	dupBytes := 0
	lenUnique := uint64(0)
	for _, segStat := range s.SegHashes {
		if segStat.Freq > 1 {
			dupCount += (segStat.Freq - 1)
			dupBytes += (segStat.Length * (segStat.Freq - 1))
		}
		lenUnique += uint64(segStat.Length)
	}

	output := struct {
		NumCutpoints    int
		MeanSegLength   float64
		MedianSegLength float64
		MaxSegLength    float64
		MinSegLength    float64
		DupSegCount     int
		DupBytes        int
		UniqueBytes     uint64
		TotalBytes      uint64
	}{
		NumCutpoints:    len(s.Cutpoints),
		MeanSegLength:   mea,
		MedianSegLength: med,
		MaxSegLength:    max,
		MinSegLength:    min,
		DupSegCount:     dupCount,
		DupBytes:        dupBytes,
		UniqueBytes:     lenUnique,
		TotalBytes:      s.BytesParsed,
	}

	marshalled, err := json.MarshalIndent(output, "", "  ")
	if err != nil {
		return errors.Wrapf(err, "Failed to marshal stats into JSON output")
	}
	fmt.Fprintln(out, string(marshalled))
	return nil
}
<commit_msg>Add PrintSegLengths to stats<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"hash"
	"io"
	"strconv"
	"strings"
	"sync/atomic"

	"github.com/montanaflynn/stats"
	"github.com/pkg/errors"
)

// SegmentStat holds stats for a single segment
type SegmentStat struct {
	SeqNum uint64 // First occurrence of this segment in the file
	Length int    // Length of segment
	Freq   int    // How many times this segment occurred in the file
}

// ParseStats holds stats about the parsed file
type ParseStats struct {
	Cutpoints   []uint64               // indices at which we have cutpoints
	SegLengths  []uint64               // lengths of segments (between cutpoints)
	SegHashes   map[string]SegmentStat // map[crypto hash of seg] -> SegmentStat
	BytesParsed uint64                 // number of bytes parsed

	// internal:
	segNum    uint64    // tracks the segment numbers we've issued
	segHasher hash.Hash // used to generate the crypto hash of segments
}

// NewParseStats returns an initialized ParseStats struct
func NewParseStats(hasher hash.Hash) *ParseStats {
	return &ParseStats{
		SegHashes: make(map[string]SegmentStat),
		segNum:    uint64(0),
		segHasher: hasher,
	}
}

// UpdateStats updates the ParseStats with stats for the specified chunk
func (s *ParseStats) UpdateStats(segment []byte) string {

	// Sprint'ing the hash sum causes an allocation that is unnecessary and
	// is completely avoidable.
	//segHash := fmt.Sprintf("%X", s.segHasher.Sum(segment))
	segHash := string(s.segHasher.Sum(segment))
	segStat, there := s.SegHashes[segHash]
	if there {
		segStat.Freq++
	} else {
		//segStat = SegmentStat{SeqNum: s.segNum, Length: len(segment), Freq: 1}
		segStat.Freq = 1
		segStat.Length = len(segment)
		segStat.SeqNum = atomic.AddUint64(&s.segNum, 1)
	}
	s.SegHashes[segHash] = segStat

	// Additional book keeping (TODO: allow this to be disabled)
	s.Cutpoints = append(s.Cutpoints, s.BytesParsed+uint64(len(segment)))
	s.SegLengths = append(s.SegLengths, uint64(len(segment)))
	s.BytesParsed += uint64(len(segment))

	return segHash
}

// Print prints the specified ParseStats on the given output (io.Writer)
//
func (s ParseStats) Print(out io.Writer) error {
	segLens := make([]float64, 0, len(s.SegLengths))
	for _, s := range s.SegLengths {
		segLens = append(segLens, float64(s))
	}

	med, err := stats.Median(segLens)
	if err != nil {
		return errors.Wrapf(err, "Failed compute median")
	}
	max, err := stats.Max(segLens)
	if err != nil {
		return errors.Wrapf(err, "Failed to compute max")
	}
	min, err := stats.Min(segLens)
	if err != nil {
		return errors.Wrapf(err, "Failed to compute min")
	}
	mea, err := stats.Mean(segLens)
	if err != nil {
		return errors.Wrapf(err, "Failed to compute mean")
	}

	dupCount := 0
	dupBytes := 0
	lenUnique := uint64(0)
	for _, segStat := range s.SegHashes {
		if segStat.Freq > 1 {
			dupCount += (segStat.Freq - 1)
			dupBytes += (segStat.Length * (segStat.Freq - 1))
		}
		lenUnique += uint64(segStat.Length)
	}

	output := struct {
		NumCutpoints    int
		MeanSegLength   float64
		MedianSegLength float64
		MaxSegLength    float64
		MinSegLength    float64
		DupSegCount     int
		DupBytes        int
		UniqueBytes     uint64
		TotalBytes      uint64
	}{
		NumCutpoints:    len(s.Cutpoints),
		MeanSegLength:   mea,
		MedianSegLength: med,
		MaxSegLength:    max,
		MinSegLength:    min,
		DupSegCount:     dupCount,
		DupBytes:        dupBytes,
		UniqueBytes:     lenUnique,
		TotalBytes:      s.BytesParsed,
	}

	marshalled, err := json.MarshalIndent(output, "", "  ")
	if err != nil {
		return errors.Wrapf(err, "Failed to marshal stats into JSON output")
	}
	fmt.Fprintln(out, string(marshalled))
	return nil
}

// PrintSegLengths prints segment lengths to the specified output separated by
// the specified separator
func (s ParseStats) PrintSegLengths(out io.Writer, sep string) error {
	lenStrings := []string{}
	for _, len := range s.SegLengths {
		lenStrings = append(lenStrings, strconv.Itoa(int(len)))
	}

	// Join our string slice.
	result := strings.Join(lenStrings, sep)
	_, err := fmt.Fprint(out, result)
	return err
}
<|endoftext|>


<commit_before>package oplog

import (
	"strconv"
	"sync/atomic"
)

type counter struct {
	uint64
}

type gauge struct {
	int64
}

// Stats stores all the statistics about the oplog
type Stats struct {
	Status string `json:"status"`
	// Total number of events recieved on the UDP interface
	EventsReceived *counter `json:"events_received"`
	// Total number of events ingested into MongoDB with success
	EventsIngested *counter `json:"events_ingested"`
	// Total number of events received on the UDP interface with an invalid format
	EventsError *counter `json:"events_error"`
	// Total number of events discarded because the queue was full
	EventsDiscarded *counter `json:"events_discarded"`
	// Current number of events in the ingestion queue
	QueueSize *gauge `json:"queue_size"`
	// Maximum number of events allowed in the ingestion queue before discarding events
	QueueMaxSize *gauge `json:"queue_max_size"`
	// Number of clients connected to the SSE API
	Clients *gauge `json:"clients"`
}

// NewStats create a new empty stats object
func NewStats() Stats {
	return Stats{
		Status:          "OK",
		EventsReceived:  &counter{0},
		EventsIngested:  &counter{0},
		EventsError:     &counter{0},
		EventsDiscarded: &counter{0},
		QueueSize:       &gauge{0},
		QueueMaxSize:    &gauge{0},
		Clients:         &gauge{0},
	}
}

func (c *counter) MarshalJSON() ([]byte, error) {
	return []byte(strconv.FormatUint(c.uint64, 10)), nil
}

// Incr atomically increment the counter by 1
func (c *counter) Incr() {
	atomic.AddUint64(&c.uint64, 1)
}

func (g *gauge) MarshalJSON() ([]byte, error) {
	return []byte(strconv.FormatInt(g.int64, 10)), nil
}

// Set atomically set the gauge value to the provided int
func (g *gauge) Set(val int) {
	atomic.StoreInt64(&g.int64, int64(val))
}

// Incr atomically increment the gauge by 1
func (g *gauge) Incr() {
	atomic.AddInt64(&g.int64, 1)
}

// Incr atomically decrement the gauge by 1
func (g *gauge) Decr() {
	atomic.AddInt64(&g.int64, -1)
}
<commit_msg>Refactor stats to in order to help integration of statsd support<commit_after>package oplog

import (
	"bytes"
	"fmt"
	"strconv"
	"sync/atomic"
)

type field interface {
	name() string
	value() string
}

type counter struct {
	uint64
	n string
}

type gauge struct {
	int64
	n string
}

// Stats stores all the statistics about the oplog
type Stats struct {
	Status string
	// Total number of events recieved on the UDP interface
	EventsReceived *counter
	// Total number of events ingested into MongoDB with success
	EventsIngested *counter
	// Total number of events received on the UDP interface with an invalid format
	EventsError *counter
	// Total number of events discarded because the queue was full
	EventsDiscarded *counter
	// Current number of events in the ingestion queue
	QueueSize *gauge
	// Maximum number of events allowed in the ingestion queue before discarding events
	QueueMaxSize *gauge
	// Number of clients connected to the SSE API
	Clients *gauge
}

// NewStats create a new empty stats object
func NewStats() Stats {
	return Stats{
		Status:          "OK",
		EventsReceived:  &counter{0, "events_received"},
		EventsIngested:  &counter{0, "events_ingested"},
		EventsError:     &counter{0, "events_error"},
		EventsDiscarded: &counter{0, "events_discarded"},
		QueueSize:       &gauge{0, "queue_size"},
		QueueMaxSize:    &gauge{0, "queue_max_size"},
		Clients:         &gauge{0, "clients"},
	}
}

func (s Stats) MarshalJSON() ([]byte, error) {
	b := &bytes.Buffer{}
	b.WriteByte('{')
	writeRawField("status", fmt.Sprintf("\"%s\"", s.Status), b, false)
	writeField(s.EventsReceived, b, false)
	writeField(s.EventsIngested, b, false)
	writeField(s.EventsError, b, false)
	writeField(s.EventsDiscarded, b, false)
	writeField(s.QueueSize, b, false)
	writeField(s.QueueMaxSize, b, false)
	writeField(s.Clients, b, true)
	b.WriteByte('}')
	return b.Bytes(), nil
}

func writeField(field field, b *bytes.Buffer, last bool) {
	writeRawField(field.name(), field.value(), b, last)
}

func writeRawField(name string, value string, b *bytes.Buffer, last bool) {
	b.WriteByte('"')
	b.WriteString(name)
	b.WriteString("\":")
	b.WriteString(value)
	if !last {
		b.WriteByte(',')
	}
}

func (c *counter) name() string {
	return c.n
}

func (c *counter) value() string {
	return strconv.FormatUint(atomic.LoadUint64(&c.uint64), 10)
}

// Incr atomically increment the counter by 1
func (c *counter) Incr() {
	atomic.AddUint64(&c.uint64, 1)
}

func (g *gauge) name() string {
	return g.n
}

func (g *gauge) value() string {
	return strconv.FormatInt(atomic.LoadInt64(&g.int64), 10)
}

// Set atomically set the gauge value to the provided int
func (g *gauge) Set(val int) {
	atomic.StoreInt64(&g.int64, int64(val))
}

// Incr atomically increment the gauge by 1
func (g *gauge) Incr() {
	atomic.AddInt64(&g.int64, 1)
}

// Incr atomically decrement the gauge by 1
func (g *gauge) Decr() {
	atomic.AddInt64(&g.int64, -1)
}
<|endoftext|>


<commit_before>// Copyright 2010 Whitham D. Reeve II. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package termioslib provides access to termios.h.
package termioslib

/*
#include <termios.h>
*/
import "C"

type Termios struct {
	i	C.struct_termios
    C_iflag, C_oflag, C_cflag, C_lflag C.tcflag_t
    C_cc [NCCS]C.cc_t
}

// Special Control Characters
// Index into c_cc[] character array.
const (
	VEOF 		= uint (C.VEOF);
	VEOL 		= uint (C.VEOL);
	VEOL2 		= uint (C.VEOL2);
	VERASE		= uint (C.VERASE);
	VWERASE 	= uint (C.VWERASE);
	VKILL 		= uint (C.VKILL);
	VREPRINT	= uint (C.VREPRINT);
	/* 7 is spare */
	VINTR		= uint (C.VINTR);
	VQUIT		= uint (C.VQUIT);
	VSUSP		= uint (C.VSUSP);
	//VDSUSP		= uint (C.VDSUSP);
	VSTART		= uint (C.VSTART);
	VSTOP		= uint (C.VSTOP);
	VLNEXT		= uint (C.VLNEXT);
	VDISCARD	= uint (C.VDISCARD);
	VMIN		= uint (C.VMIN);
	VTIME		= uint (C.VTIME);
	//VSTATUS		= uint (C.VSTATUS);
	/* 19 is spare */
	NCCS		= uint (C.NCCS)
)

// Input flags - software input processing
const (
	IGNBRK		= C.tcflag_t(C.IGNBRK)	/* ignore BREAK condition */
	BRKINT		= C.tcflag_t(C.BRKINT) 	/* map BREAK to SIGINTR */
	IGNPAR		= C.tcflag_t(C.IGNPAR) 	/* ignore (discard) parity errors */
	PARMRK		= C.tcflag_t(C.PARMRK) 	/* mark parity and framing errors */
	INPCK		= C.tcflag_t(C.INPCK)	/* enable checking of parity errors */
	ISTRIP		= C.tcflag_t(C.ISTRIP) 	/* strip 8th bit off chars */
	INLCR		= C.tcflag_t(C.INLCR) 	/* map NL into CR */
	IGNCR		= C.tcflag_t(C.IGNCR)	/* ignore CR */
	ICRNL		= C.tcflag_t(C.ICRNL)	/* map CR to NL (ala CRMOD) */
	IXON		= C.tcflag_t(C.IXON)	/* enable output flow control */
	IXOFF		= C.tcflag_t(C.IXOFF) 	/* enable input flow control */
	IXANY		= C.tcflag_t(C.IXANY) 	/* any char will restart after stop */
	IMAXBEL		= C.tcflag_t(C.IMAXBEL) /* ring bell on input queue full */
	IUTF8		= C.tcflag_t(C.IUTF8) 	/* maintain state for UTF-8 VERASE */
)

// Output flags - software output processing
const (
	OPOST		= C.tcflag_t(C.OPOST) 	/* enable following output processing */
	ONLCR		= C.tcflag_t(C.ONLCR) 	/* map NL to CR-NL (ala CRMOD) */
	//OXTABS		= tcflag_t (C.OXTABS); 	/* expand tabs to spaces */
	//ONOEOT		= tcflag_t (C.ONOEOT) 	/* discard EOT's (^D) on output) */
)

// Standard speeds
const (
	B0			= C.speed_t(C.B0)
	B50			= C.speed_t(C.B50)
	B75			= C.speed_t(C.B75)
	B110		= C.speed_t(C.B110)
	B134		= C.speed_t(C.B134)
	B150		= C.speed_t(C.B150)
	B200		= C.speed_t(C.B200)
	B300		= C.speed_t(C.B300)
	B600		= C.speed_t(C.B600)
	B1200		= C.speed_t(C.B1200)
	B1800		= C.speed_t(C.B1800)
	B2400		= C.speed_t(C.B2400)
	B4800		= C.speed_t(C.B4800)
	B9600		= C.speed_t(C.B9600)
	B19200		= C.speed_t(C.B19200)
	B38400		= C.speed_t(C.B38400 )
	// non posix but generally supported none the less
	//B7200		= C.speed_t(C.B7200)
	//B14400		= C.speed_t(C.B14400)
	//B28800		= C.speed_t(C.B28800)
	B57600		= C.speed_t(C.B57600)
	//B76800		= C.speed_t(C.B76800)
	B115200		= C.speed_t(C.B115200)
	B230400		= C.speed_t(C.B230400)
	EXTA		= C.speed_t(C.EXTA)
	EXTB		= C.speed_t(C.EXTB)
)

// Control flags - hardware control of terminal
const (
	//CIGNORE		= tcflag_t (C.CIGNORE);	/* ignore control flags */
	CSIZE		= C.tcflag_t(C.CSIZE)	/* character size mask */
		CS5		= C.tcflag_t(C.CS5)		/* 5 bits (pseudo) */
		CS6		= C.tcflag_t(C.CS6)		/* 6 bits */
		CS7		= C.tcflag_t(C.CS7)		/* 7 bits */
		CS8		= C.tcflag_t(C.CS8)		/* 8 bits */
	CSTOPB		= C.tcflag_t(C.CSTOPB)	/* send 2 stop bits */
	CREAD		= C.tcflag_t(C.CREAD)	/* enable receiver */
	PARENB		= C.tcflag_t(C.PARENB)	/* parity enable */
	PARODD		= C.tcflag_t(C.PARODD)	/* odd parity, else even */
	HUPCL		= C.tcflag_t(C.HUPCL)	/* hang up on last close */
	CLOCAL		= C.tcflag_t(C.CLOCAL)	/* ignore modem status lines */
	//CCTS_OFLOW	= tcflag_t (C.CCTS_OFLOW);	/* CTS flow control of output */
	//CRTS_IFLOW	= tcflag_t (C.CRTS_IFLOW);	/* RTS flow control of input */
	CRTSCTS		= C.tcflag_t(C.CRTSCTS)
	//CDTR_IFLOW	= tcflag_t (C.CDTR_IFLOW);	/* DTR flow control of input */
	//CDSR_OFLOW	= tcflag_t (C.CDSR_OFLOW);	/* DSR flow control of output */
	//CCAR_OFLOW	= tcflag_t (C.CCAR_OFLOW);	/* DCD flow control of output */
	//MDMBUF		= tcflag_t (C.MDMBUF)		/* old name for CCAR_OFLOW */
)

// "Local" flags - dumping ground for other state
const (
	ECHOKE		= C.tcflag_t(C.ECHOKE)	/* visual erase for line kill */
	ECHOE		= C.tcflag_t(C.ECHOE)	/* visually erase chars */
	ECHOK		= C.tcflag_t(C.ECHOK)	/* echo NL after line kill */
	ECHO		= C.tcflag_t(C.ECHO)	/* enable echoing */
	ECHONL		= C.tcflag_t(C.ECHONL)	/* echo NL even if ECHO is off */
	ECHOPRT		= C.tcflag_t(C.ECHOPRT)	/* visual erase mode for hardcopy */
	ECHOCTL		= C.tcflag_t(C.ECHOCTL)	/* echo control chars as ^(Char) */
	ISIG		= C.tcflag_t(C.ISIG)	/* enable signals INTR, QUIT, [D]SUSP */
	ICANON		= C.tcflag_t(C.ICANON)	/* canonicalize input lines */
	//ALTWERASE	= tcflag_t (C.ALTWERASE);	/* use alternate WERASE algorithm */
	IEXTEN		= C.tcflag_t(C.IEXTEN)	/* enable DISCARD and LNEXT */
	EXTPROC		= C.tcflag_t(C.EXTPROC)	/* external processing */
	TOSTOP		= C.tcflag_t(C.TOSTOP)	/* stop background jobs from output */
	FLUSHO		= C.tcflag_t(C.FLUSHO)	/* output being flushed (state) */
	//NOKERNINFO	= tcflag_t (C.NOKERNINFO);	/* no kernel output from VSTATUS */
	PENDIN		= C.tcflag_t(C.PENDIN)	/* XXX retype pending input (state) */
	NOFLSH		= C.tcflag_t(C.NOFLSH)	/* don't flush after interrupt */
)

// Commands passed to tcsetattr() for setting the termios structure.
const (
	TCSANOW		= int (C.TCSANOW)		/* make change immediate */
	TCSADRAIN	= int (C.TCSADRAIN)	/* drain output, then change */
	TCSAFLUSH	= int (C.TCSAFLUSH)	/* drain output, flush input */
)

const (
	TCIFLUSH	= int (C.TCIFLUSH)
	TCOFLUSH	= int (C.TCOFLUSH)
	TCIOFLUSH	= int (C.TCIOFLUSH)
	TCOOFF		= int (C.TCOOFF)
	TCOON		= int (C.TCOON)
	TCIOFF		= int (C.TCIOFF)
	TCION		= int (C.TCION)
)

//speed_t	cfgetispeed(const struct Termios *);
func Getispeed (src * Termios) (result C.speed_t) {
	result = C.cfgetispeed(&src.i)
	return
}

//speed_t	cfgetospeed(const struct Termios *);
func Getospeed (src * Termios) (result C.speed_t) {
	result = C.cfgetospeed(&src.i)
	return
}

//int	cfsetispeed(struct Termios *, speed_t);
func Setispeed(dst * Termios, baud C.speed_t) (error) {
	_,rv := C.cfsetispeed(&dst.i, baud)
	return rv
}

//int	cfsetospeed(struct Termios *, speed_t);
func Setospeed(dst * Termios, baud C.speed_t) (error) {
	_,rv := C.cfsetospeed(&dst.i, baud)
	return rv
}

func Getattr(fd uintptr, dst * Termios) (error) {
	_,rv := C.tcgetattr(C.int(fd),&dst.i)
	if rv == nil {
		dst.C_iflag = dst.i.c_iflag
		dst.C_oflag = dst.i.c_oflag
		dst.C_cflag = dst.i.c_cflag
		dst.C_lflag = dst.i.c_lflag
		dst.C_cc	= dst.i.c_cc
	}
	return rv
}

//int	tcsetattr(int, int, const struct termios *);
func Setattr(fd uintptr, optional_action int, src * Termios) (error) {
	src.i.c_iflag = src.C_iflag
	src.i.c_oflag = src.C_oflag
	src.i.c_cflag = src.C_cflag
	src.i.c_lflag = src.C_lflag
	src.i.c_cc = src.C_cc
	_,rv := C.tcsetattr(C.int(fd), C.int(optional_action), &src.i)
	return rv
}

//int	tcdrain(int) __DARWIN_ALIAS_C(tcdrain);
func Drain(fd uintptr) (error) {
	_, rv := C.tcdrain(C.int(fd))
	return rv
}

//int	tcflow(int, int);
func Flow(fd uintptr, action int) (error) {
	_,rv := C.tcflow(C.int(fd), C.int(action))
	return rv
}

//int	tcflush(int, int);
func Flush(fd uintptr, queue_selector int) (error) {
	_, rv := C.tcflush(C.int(fd), C.int(queue_selector))
	return rv
}
//int	tcsendbreak(int, int);
func Sendbreak(fd uintptr, duration int) (error) {
	_, rv := C.tcflush(C.int(fd), C.int(duration))
	return rv

}
//void	cfmakeraw(struct termios *);
func Makeraw(src * Termios) {
	C.cfmakeraw(&src.i)
}

// int	cfsetspeed(struct termios *, speed_t);
func Setspeed (dst * Termios, baud C.speed_t) (error) {
	_, rv := C.cfsetspeed(&dst.i, baud)
	return rv
}
<commit_msg>Fix speed not getting set. The speed was getting wiped out when c_cflags was getting set in the SetAttr function.<commit_after>// Copyright 2010 Whitham D. Reeve II. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package termioslib provides access to termios.h.
package termioslib

/*
#include <termios.h>
#include <unistd.h>
*/
import "C"

type Termios struct {
	i	C.struct_termios
    C_iflag, C_oflag, C_cflag, C_lflag C.tcflag_t
    C_cc [NCCS]C.cc_t
}

// Special Control Characters
// Index into c_cc[] character array.
const (
	VEOF 		= uint (C.VEOF);
	VEOL 		= uint (C.VEOL);
	VEOL2 		= uint (C.VEOL2);
	VERASE		= uint (C.VERASE);
	VWERASE 	= uint (C.VWERASE);
	VKILL 		= uint (C.VKILL);
	VREPRINT	= uint (C.VREPRINT);
	/* 7 is spare */
	VINTR		= uint (C.VINTR);
	VQUIT		= uint (C.VQUIT);
	VSUSP		= uint (C.VSUSP);
	//VDSUSP		= uint (C.VDSUSP);
	VSTART		= uint (C.VSTART);
	VSTOP		= uint (C.VSTOP);
	VLNEXT		= uint (C.VLNEXT);
	VDISCARD	= uint (C.VDISCARD);
	VMIN		= uint (C.VMIN);
	VTIME		= uint (C.VTIME);
	//VSTATUS		= uint (C.VSTATUS);
	/* 19 is spare */
	NCCS		= uint (C.NCCS)
)

// Input flags - software input processing
const (
	IGNBRK		= C.tcflag_t(C.IGNBRK)	/* ignore BREAK condition */
	BRKINT		= C.tcflag_t(C.BRKINT) 	/* map BREAK to SIGINTR */
	IGNPAR		= C.tcflag_t(C.IGNPAR) 	/* ignore (discard) parity errors */
	PARMRK		= C.tcflag_t(C.PARMRK) 	/* mark parity and framing errors */
	INPCK		= C.tcflag_t(C.INPCK)	/* enable checking of parity errors */
	ISTRIP		= C.tcflag_t(C.ISTRIP) 	/* strip 8th bit off chars */
	INLCR		= C.tcflag_t(C.INLCR) 	/* map NL into CR */
	IGNCR		= C.tcflag_t(C.IGNCR)	/* ignore CR */
	ICRNL		= C.tcflag_t(C.ICRNL)	/* map CR to NL (ala CRMOD) */
	IXON		= C.tcflag_t(C.IXON)	/* enable output flow control */
	IXOFF		= C.tcflag_t(C.IXOFF) 	/* enable input flow control */
	IXANY		= C.tcflag_t(C.IXANY) 	/* any char will restart after stop */
	IMAXBEL		= C.tcflag_t(C.IMAXBEL) /* ring bell on input queue full */
	IUTF8		= C.tcflag_t(C.IUTF8) 	/* maintain state for UTF-8 VERASE */
)

// Output flags - software output processing
const (
	OPOST		= C.tcflag_t(C.OPOST) 	/* enable following output processing */
	ONLCR		= C.tcflag_t(C.ONLCR) 	/* map NL to CR-NL (ala CRMOD) */
	//OXTABS		= tcflag_t (C.OXTABS); 	/* expand tabs to spaces */
	//ONOEOT		= tcflag_t (C.ONOEOT) 	/* discard EOT's (^D) on output) */
)

// Standard speeds
const (
	B0			= C.speed_t(C.B0)
	B50			= C.speed_t(C.B50)
	B75			= C.speed_t(C.B75)
	B110		= C.speed_t(C.B110)
	B134		= C.speed_t(C.B134)
	B150		= C.speed_t(C.B150)
	B200		= C.speed_t(C.B200)
	B300		= C.speed_t(C.B300)
	B600		= C.speed_t(C.B600)
	B1200		= C.speed_t(C.B1200)
	B1800		= C.speed_t(C.B1800)
	B2400		= C.speed_t(C.B2400)
	B4800		= C.speed_t(C.B4800)
	B9600		= C.speed_t(C.B9600)
	B19200		= C.speed_t(C.B19200)
	B38400		= C.speed_t(C.B38400)
	B57600		= C.speed_t(C.B57600)
	B115200		= C.speed_t(C.B115200)
	B230400		= C.speed_t(C.B230400)
	EXTA		= C.speed_t(C.EXTA)
	EXTB		= C.speed_t(C.EXTB)
)

// Control flags - hardware control of terminal
const (
	//CIGNORE		= tcflag_t (C.CIGNORE);	/* ignore control flags */
	CSIZE		= C.tcflag_t(C.CSIZE)	/* character size mask */
		CS5		= C.tcflag_t(C.CS5)		/* 5 bits (pseudo) */
		CS6		= C.tcflag_t(C.CS6)		/* 6 bits */
		CS7		= C.tcflag_t(C.CS7)		/* 7 bits */
		CS8		= C.tcflag_t(C.CS8)		/* 8 bits */
	CSTOPB		= C.tcflag_t(C.CSTOPB)	/* send 2 stop bits */
	CREAD		= C.tcflag_t(C.CREAD)	/* enable receiver */
	PARENB		= C.tcflag_t(C.PARENB)	/* parity enable */
	PARODD		= C.tcflag_t(C.PARODD)	/* odd parity, else even */
	HUPCL		= C.tcflag_t(C.HUPCL)	/* hang up on last close */
	CLOCAL		= C.tcflag_t(C.CLOCAL)	/* ignore modem status lines */
	//CCTS_OFLOW	= tcflag_t (C.CCTS_OFLOW);	/* CTS flow control of output */
	//CRTS_IFLOW	= tcflag_t (C.CRTS_IFLOW);	/* RTS flow control of input */
	CRTSCTS		= C.tcflag_t(C.CRTSCTS)
	//CDTR_IFLOW	= tcflag_t (C.CDTR_IFLOW);	/* DTR flow control of input */
	//CDSR_OFLOW	= tcflag_t (C.CDSR_OFLOW);	/* DSR flow control of output */
	//CCAR_OFLOW	= tcflag_t (C.CCAR_OFLOW);	/* DCD flow control of output */
	//MDMBUF		= tcflag_t (C.MDMBUF)		/* old name for CCAR_OFLOW */
)

// "Local" flags - dumping ground for other state
const (
	ECHOKE		= C.tcflag_t(C.ECHOKE)	/* visual erase for line kill */
	ECHOE		= C.tcflag_t(C.ECHOE)	/* visually erase chars */
	ECHOK		= C.tcflag_t(C.ECHOK)	/* echo NL after line kill */
	ECHO		= C.tcflag_t(C.ECHO)	/* enable echoing */
	ECHONL		= C.tcflag_t(C.ECHONL)	/* echo NL even if ECHO is off */
	ECHOPRT		= C.tcflag_t(C.ECHOPRT)	/* visual erase mode for hardcopy */
	ECHOCTL		= C.tcflag_t(C.ECHOCTL)	/* echo control chars as ^(Char) */
	ISIG		= C.tcflag_t(C.ISIG)	/* enable signals INTR, QUIT, [D]SUSP */
	ICANON		= C.tcflag_t(C.ICANON)	/* canonicalize input lines */
	//ALTWERASE	= tcflag_t (C.ALTWERASE);	/* use alternate WERASE algorithm */
	IEXTEN		= C.tcflag_t(C.IEXTEN)	/* enable DISCARD and LNEXT */
	EXTPROC		= C.tcflag_t(C.EXTPROC)	/* external processing */
	TOSTOP		= C.tcflag_t(C.TOSTOP)	/* stop background jobs from output */
	FLUSHO		= C.tcflag_t(C.FLUSHO)	/* output being flushed (state) */
	//NOKERNINFO	= tcflag_t (C.NOKERNINFO);	/* no kernel output from VSTATUS */
	PENDIN		= C.tcflag_t(C.PENDIN)	/* XXX retype pending input (state) */
	NOFLSH		= C.tcflag_t(C.NOFLSH)	/* don't flush after interrupt */
)

// Commands passed to tcsetattr() for setting the termios structure.
const (
	TCSANOW		= int (C.TCSANOW)		/* make change immediate */
	TCSADRAIN	= int (C.TCSADRAIN)	/* drain output, then change */
	TCSAFLUSH	= int (C.TCSAFLUSH)	/* drain output, flush input */
)

const (
	TCIFLUSH	= int (C.TCIFLUSH)
	TCOFLUSH	= int (C.TCOFLUSH)
	TCIOFLUSH	= int (C.TCIOFLUSH)
	TCOOFF		= int (C.TCOOFF)
	TCOON		= int (C.TCOON)
	TCIOFF		= int (C.TCIOFF)
	TCION		= int (C.TCION)
)

//speed_t	cfgetispeed(const struct Termios *);
func Getispeed (src * Termios) (result C.speed_t) {
	result = C.cfgetispeed(&(src.i))
	return
}

//speed_t	cfgetospeed(const struct Termios *);
func Getospeed (src * Termios) (result C.speed_t) {
	result = C.cfgetospeed(&src.i)
	return
}

//int	cfsetispeed(struct Termios *, speed_t);
func Setispeed(dst * Termios, baud C.speed_t) (error) {
	dst.i.c_iflag = dst.C_iflag
	dst.i.c_oflag = dst.C_oflag
	dst.i.c_cflag = dst.C_cflag
	dst.i.c_lflag = dst.C_lflag
	dst.i.c_cc = dst.C_cc
	_,rv := C.cfsetispeed(&dst.i, baud)
	if rv == nil {
		dst.C_iflag = dst.i.c_iflag
		dst.C_oflag = dst.i.c_oflag
		dst.C_cflag = dst.i.c_cflag
		dst.C_lflag = dst.i.c_lflag
		dst.C_cc	= dst.i.c_cc
	}
	return rv
}

//int	cfsetospeed(struct Termios *, speed_t);
func Setospeed(dst * Termios, baud C.speed_t) (error) {
	dst.i.c_iflag = dst.C_iflag
	dst.i.c_oflag = dst.C_oflag
	dst.i.c_cflag = dst.C_cflag
	dst.i.c_lflag = dst.C_lflag
	dst.i.c_cc = dst.C_cc
	_,rv := C.cfsetospeed(&dst.i, baud)
	if rv == nil {
		dst.C_iflag = dst.i.c_iflag
		dst.C_oflag = dst.i.c_oflag
		dst.C_cflag = dst.i.c_cflag
		dst.C_lflag = dst.i.c_lflag
		dst.C_cc	= dst.i.c_cc
	}
	return rv
}

func Getattr(fd uintptr, dst * Termios) (error) {
	_,rv := C.tcgetattr(C.int(fd),&dst.i)
	if rv == nil {
		dst.C_iflag = dst.i.c_iflag
		dst.C_oflag = dst.i.c_oflag
		dst.C_cflag = dst.i.c_cflag
		dst.C_lflag = dst.i.c_lflag
		dst.C_cc	= dst.i.c_cc
	}
	return rv
}

//int	tcsetattr(int, int, const struct termios *);
func Setattr(fd uintptr, optional_action int, src * Termios) (error) {
	src.i.c_iflag = src.C_iflag
	src.i.c_oflag = src.C_oflag
	src.i.c_cflag = src.C_cflag
	src.i.c_lflag = src.C_lflag
	src.i.c_cc = src.C_cc
	_,rv := C.tcsetattr(C.int(fd), C.TCSANOW, &src.i)
	return rv
}

//int	tcdrain(int) __DARWIN_ALIAS_C(tcdrain);
func Drain(fd uintptr) (error) {
	_, rv := C.tcdrain(C.int(fd))
	return rv
}

//int	tcflow(int, int);
func Flow(fd uintptr, action int) (error) {
	_,rv := C.tcflow(C.int(fd), C.int(action))
	return rv
}

//int	tcflush(int, int);
func Flush(fd uintptr, queue_selector int) (error) {
	_, rv := C.tcflush(C.int(fd), C.int(queue_selector))
	return rv
}
//int	tcsendbreak(int, int);
func Sendbreak(fd uintptr, duration int) (error) {
	_, rv := C.tcflush(C.int(fd), C.int(duration))
	return rv

}
//void	cfmakeraw(struct termios *);
func Makeraw(src * Termios) {
	C.cfmakeraw(&src.i)
}

// int	cfsetspeed(struct termios *, speed_t);
func Setspeed (dst * Termios, baud C.speed_t) (error) {
	dst.i.c_iflag = dst.C_iflag
	dst.i.c_oflag = dst.C_oflag
	dst.i.c_cflag = dst.C_cflag
	dst.i.c_lflag = dst.C_lflag
	dst.i.c_cc = dst.C_cc
	_, rv := C.cfsetspeed(&dst.i, baud)
	if rv == nil {
		dst.C_iflag = dst.i.c_iflag
		dst.C_oflag = dst.i.c_oflag
		dst.C_cflag = dst.i.c_cflag
		dst.C_lflag = dst.i.c_lflag
		dst.C_cc	= dst.i.c_cc
	}
	return rv
}
<|endoftext|>


<commit_before>package cookoo

import (
	"fmt"
	"io"
	"os"
)

// MultiWriter enables you to have a writer that passes on the writing to one
// of more Writers where the write is duplicated to each Writer. MultiWriter
// is similar to the multiWriter that is part of Go. The difference is
// this MultiWriter allows you to manager the Writers attached to it via CRUD
// operations. To do this you will need to mock the type. For example,
// mw := NewMultiWriter()
// mw.(*MultiWriter).AddWriter("foo", foo)
type MultiWriter struct {
	writers map[string]io.Writer
}

// Write sends the bytes to each of the attached writers to be written.
func (t *MultiWriter) Write(p []byte) (n int, err error) {
	for name, w := range t.writers {
		n, err = w.Write(p)
		if err != nil {
			// One broken logger should not stop the others.
			fmt.Fprintf(os.Stderr, "Error logging to '%s': %s", name, err)
			continue
		}
		if n < len(p) {
			// One broken logger should not stop the others.
			err = io.ErrShortWrite
			fmt.Fprintf(os.Stderr, "Short write logging to '%s': Expected to write %d (%V), wrote %d", name, len(p), w, n)
			continue
		}
	}
	return len(p), err
}

// Init initializes the MultiWriter.
func (t *MultiWriter) Init() *MultiWriter {
	t.writers = make(map[string]io.Writer)
	return t
}

// Writer retrieves a given io.Writer given its name.
func (t *MultiWriter) Writer(name string) (io.Writer, bool) {
	value, found := t.writers[name]
	return value, found
}

// Writers retrieves a map of all io.Writers keyed by name.
func (t *MultiWriter) Writers() map[string]io.Writer {
	return t.writers
}

// AddWriter adds an io.Writer with an associated name.
func (t *MultiWriter) AddWriter(name string, writer io.Writer) {
	t.writers[name] = writer
}

// RemoveWriter removes an io.Writer given a name.
func (t *MultiWriter) RemoveWriter(name string) {
	delete(t.writers, name)
}

// NewMultiWriter returns an initialized MultiWriter.
func NewMultiWriter() io.Writer {
	w := new(MultiWriter).Init()
	return w
}
<commit_msg>Remove unsed lib<commit_after><|endoftext|>


<commit_before><commit_msg>gc: Use better exponential buckets<commit_after><|endoftext|>


<commit_before><commit_msg>Properly work out whether packets belong to pingbeat by the ID field. Better logic around ICMP error packets. More comments.<commit_after><|endoftext|>


<commit_before><commit_msg>Support HEAD requests (#289)<commit_after><|endoftext|>


<commit_before>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

// TODO: Improved version parsers / checkers / handling (semver?).

var STORE_PREFIX = "data-" // File name prefix.
var STORE_SUFFIX = ".moss" // File name suffix.

var STORE_ENDIAN = binary.LittleEndian
var STORE_PAGE_SIZE = 4096

// STORE_VERSION must be bumped whenever the file format changes.
var STORE_VERSION = uint32(3)

var STORE_MAGIC_BEG []byte = []byte("0m1o2s")
var STORE_MAGIC_END []byte = []byte("3s4p5s")

var lenMagicBeg int = len(STORE_MAGIC_BEG)
var lenMagicEnd int = len(STORE_MAGIC_END)

// footerBegLen includes STORE_VERSION(uint32) & footerLen(uint32).
var footerBegLen int = lenMagicBeg + lenMagicBeg + 4 + 4

// footerEndLen includes footerOffset(int64) & footerLen(uint32) again.
var footerEndLen int = 8 + 4 + lenMagicEnd + lenMagicEnd

// --------------------------------------------------------

// Header represents the JSON stored at the head of a file, where the
// file header bytes should be less than STORE_PAGE_SIZE length.
type Header struct {
	Version       uint32 // The file format / STORE_VERSION.
	CreatedAt     string
	CreatedEndian string // The endian() of the file creator.
}

// Footer represents a footer record persisted in a file, and also
// implements the moss.Snapshot interface.
type Footer struct {
	m    sync.Mutex // Protects the fields that follow.
	refs int

	SegmentLocs SegmentLocs // Persisted; older SegmentLoc's come first.

	ss *segmentStack // Ephemeral.

	fileName string // Ephemeral; file name; "" when unpersisted.
	filePos  int64  // Ephemeral; byte offset of footer; <= 0 when unpersisted.
}

// --------------------------------------------------------

// Persist helps the store implement the lower-level-update func.  The
// higher snapshot may be nil.
func (s *Store) persist(higher Snapshot, persistOptions StorePersistOptions) (
	Snapshot, error) {
	wasCompacted, err := s.compactMaybe(higher, persistOptions)
	if err != nil {
		return nil, err
	}
	if wasCompacted {
		return s.Snapshot()
	}

	// If no dirty higher items, we're still clean, so just snapshot.
	// If in case of ReadOnly mode, just snapshot.
	if higher == nil || s.Options().CollectionOptions.ReadOnly {
		return s.Snapshot()
	}

	ss, ok := higher.(*segmentStack)
	if !ok {
		return nil, fmt.Errorf("store: can only persist segmentStack")
	}

	fref, file, err := s.startOrReuseFile()
	if err != nil {
		return nil, err
	}

	// TODO: Pre-allocate file space up front?

	ss.ensureSorted(0, len(ss.a)-1)

	numSegmentLocs := len(ss.a)

	s.m.Lock()
	if s.footer != nil {
		numSegmentLocs += len(s.footer.SegmentLocs)
	}
	segmentLocs := make([]SegmentLoc, 0, numSegmentLocs)
	if s.footer != nil {
		segmentLocs = append(segmentLocs, s.footer.SegmentLocs...)
	}
	s.m.Unlock()

	for _, segment := range ss.a {
		segmentLoc, err := s.persistSegment(file, segment)
		if err != nil {
			fref.DecRef()
			return nil, err
		}

		segmentLocs = append(segmentLocs, segmentLoc)
	}

	footer := &Footer{refs: 1, SegmentLocs: segmentLocs}

	err = footer.loadSegments(s.options, fref)
	if err != nil {
		fref.DecRef()
		return nil, err
	}

	err = s.persistFooter(file, footer, !persistOptions.NoSync)
	if err != nil {
		footer.DecRef()
		return nil, err
	}

	footer.AddRef() // One ref-count will be held by the store.

	s.m.Lock()
	prevFooter := s.footer
	s.footer = footer
	s.totPersists++
	s.m.Unlock()

	if prevFooter != nil {
		prevFooter.DecRef()
	}

	return footer, nil // The other ref-count returned to caller.
}

// --------------------------------------------------------

// startOrReuseFile either creates a new file or reuses the file from
// the last/current footer.
func (s *Store) startOrReuseFile() (fref *FileRef, file File, err error) {
	s.m.Lock()
	defer s.m.Unlock()

	if s.footer != nil {
		slocs, _ := s.footer.SegmentStack()
		defer s.footer.DecRef()

		if len(slocs) > 0 {
			fref := slocs[0].mref.fref
			file := fref.AddRef()

			return fref, file, nil
		}
	}

	return s.startFileLOCKED()
}

func (s *Store) startFileLOCKED() (*FileRef, File, error) {
	fname, file, err := s.createNextFileLOCKED()
	if err != nil {
		return nil, nil, err
	}

	if err = s.persistHeader(file); err != nil {
		file.Close()

		os.Remove(path.Join(s.dir, fname))

		return nil, nil, err
	}

	return &FileRef{file: file, refs: 1}, file, nil
}

func (s *Store) createNextFileLOCKED() (string, File, error) {
	fname := FormatFName(s.nextFNameSeq)
	s.nextFNameSeq++

	file, err := s.options.OpenFile(path.Join(s.dir, fname),
		os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return "", nil, err
	}

	return fname, file, nil
}

// --------------------------------------------------------

func HeaderLength() uint64 {
	return uint64(STORE_PAGE_SIZE)
}

func (s *Store) persistHeader(file File) error {
	buf, err := json.Marshal(Header{
		Version:       STORE_VERSION,
		CreatedAt:     time.Now().Format(time.RFC3339),
		CreatedEndian: endian(),
	})
	if err != nil {
		return err
	}

	str := "moss-data-store:\n" + string(buf) + "\n"
	if len(str) >= STORE_PAGE_SIZE {
		return fmt.Errorf("store: header size too big")
	}
	str = str + strings.Repeat("\n", STORE_PAGE_SIZE-len(str))

	n, err := file.WriteAt([]byte(str), 0)
	if err != nil {
		return err
	}
	if n != len(str) {
		return fmt.Errorf("store: could not write full header")
	}

	return nil
}

func checkHeader(file File) error {
	buf := make([]byte, STORE_PAGE_SIZE)

	n, err := file.ReadAt(buf, int64(0))
	if err != nil {
		return err
	}
	if n != len(buf) {
		return fmt.Errorf("store: readHeader too short")
	}

	lines := strings.Split(string(buf), "\n")
	if len(lines) < 2 {
		return fmt.Errorf("store: readHeader not enough lines")
	}
	if lines[0] != "moss-data-store:" {
		return fmt.Errorf("store: readHeader wrong file prefix")
	}

	hdr := Header{}
	err = json.Unmarshal([]byte(lines[1]), &hdr)
	if err != nil {
		return err
	}
	if hdr.Version != STORE_VERSION {
		return fmt.Errorf("store: readHeader wrong version")
	}
	if hdr.CreatedEndian != endian() {
		return fmt.Errorf("store: readHeader endian of file was: %s, need: %s",
			hdr.CreatedEndian, endian())
	}

	return nil
}

// --------------------------------------------------------

func (s *Store) persistSegment(file File, segIn Segment) (rv SegmentLoc, err error) {
	segPersister, ok := segIn.(SegmentPersister)
	if !ok {
		return rv, fmt.Errorf("store: can only persist SegmentPersister type")
	}
	return segPersister.Persist(file)
}

// --------------------------------------------------------

// ParseFNameSeq parses a file name like "data-000123.moss" into 123.
func ParseFNameSeq(fname string) (int64, error) {
	seqStr := fname[len(STORE_PREFIX) : len(fname)-len(STORE_SUFFIX)]
	return strconv.ParseInt(seqStr, 16, 64)
}

// FormatFName returns a file name like "data-000123.moss" given a seq of 123.
func FormatFName(seq int64) string {
	return fmt.Sprintf("%s%016x%s", STORE_PREFIX, seq, STORE_SUFFIX)
}

// --------------------------------------------------------

// pageAlign returns the pos if it's at the start of a page.  Else,
// pageAlign() returns pos bumped up to the next multiple of
// STORE_PAGE_SIZE.
func pageAlign(pos int64) int64 {
	rem := pos % int64(STORE_PAGE_SIZE)
	if rem != 0 {
		return pos + int64(STORE_PAGE_SIZE) - rem
	}
	return pos
}

// pageOffset returns the page offset for a given pos.
func pageOffset(pos, pageSize int64) int64 {
	rem := pos % pageSize
	if rem != 0 {
		return pos - rem
	}
	return pos
}

// --------------------------------------------------------

func openStore(dir string, options StoreOptions) (*Store, error) {
	fileInfos, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}

	var maxFNameSeq int64

	var fnames []string
	for _, fileInfo := range fileInfos { // Find candidate file names.
		fname := fileInfo.Name()
		if strings.HasPrefix(fname, STORE_PREFIX) &&
			strings.HasSuffix(fname, STORE_SUFFIX) {
			fnames = append(fnames, fname)
		}

		fnameSeq, err := ParseFNameSeq(fname)
		if err == nil && fnameSeq > maxFNameSeq {
			maxFNameSeq = fnameSeq
		}
	}

	if options.OpenFile == nil {
		options.OpenFile =
			func(name string, flag int, perm os.FileMode) (File, error) {
				return os.OpenFile(name, flag, perm)
			}
	}

	if len(fnames) <= 0 {
		emptyFooter := &Footer{
			refs: 1,
			ss: &segmentStack{
				options: &options.CollectionOptions,
			},
		}

		return &Store{
			dir:          dir,
			options:      &options,
			refs:         1,
			footer:       emptyFooter,
			nextFNameSeq: 1,
		}, nil
	}

	sort.Strings(fnames)

	for i := len(fnames) - 1; i >= 0; i-- {
		file, err := options.OpenFile(path.Join(dir, fnames[i]), os.O_RDWR, 0600)
		if err != nil {
			continue
		}

		err = checkHeader(file)
		if err != nil {
			file.Close()
			return nil, err
		}

		footer, err := ReadFooter(&options, file) // Footer owns file on success.
		if err != nil {
			file.Close()
			continue
		}

		if !options.KeepFiles {
			err := removeFiles(dir, append(fnames[0:i], fnames[i+1:]...))
			if err != nil {
				footer.Close()
				return nil, err
			}
		}

		return &Store{
			dir:          dir,
			options:      &options,
			refs:         1,
			footer:       footer,
			nextFNameSeq: maxFNameSeq + 1,
		}, nil
	}

	return nil, fmt.Errorf("store: could not successfully open/parse any file")
}

// --------------------------------------------------------

func (store *Store) openCollection(
	options StoreOptions,
	persistOptions StorePersistOptions) (Collection, error) {
	storeSnapshotInit, err := store.Snapshot()
	if err != nil {
		return nil, err
	}

	co := options.CollectionOptions
	co.LowerLevelInit = storeSnapshotInit
	co.LowerLevelUpdate = func(higher Snapshot) (Snapshot, error) {
		ss, err := store.Persist(higher, persistOptions)
		if err != nil {
			return nil, err
		}

		if storeSnapshotInit != nil {
			storeSnapshotInit.Close()
			storeSnapshotInit = nil
		}

		return ss, err
	}

	coll, err := NewCollection(co)
	if err != nil {
		storeSnapshotInit.Close()
		return nil, err
	}

	err = coll.Start()
	if err != nil {
		storeSnapshotInit.Close()
		return nil, err
	}

	return coll, nil
}

// --------------------------------------------------------

func removeFiles(dir string, fnames []string) error {
	for _, fname := range fnames {
		err := os.Remove(path.Join(dir, fname))
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>MB-22711: ReadOnly mode: file to be opened with O_RDONLY flag<commit_after>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

// TODO: Improved version parsers / checkers / handling (semver?).

var STORE_PREFIX = "data-" // File name prefix.
var STORE_SUFFIX = ".moss" // File name suffix.

var STORE_ENDIAN = binary.LittleEndian
var STORE_PAGE_SIZE = 4096

// STORE_VERSION must be bumped whenever the file format changes.
var STORE_VERSION = uint32(3)

var STORE_MAGIC_BEG []byte = []byte("0m1o2s")
var STORE_MAGIC_END []byte = []byte("3s4p5s")

var lenMagicBeg int = len(STORE_MAGIC_BEG)
var lenMagicEnd int = len(STORE_MAGIC_END)

// footerBegLen includes STORE_VERSION(uint32) & footerLen(uint32).
var footerBegLen int = lenMagicBeg + lenMagicBeg + 4 + 4

// footerEndLen includes footerOffset(int64) & footerLen(uint32) again.
var footerEndLen int = 8 + 4 + lenMagicEnd + lenMagicEnd

// --------------------------------------------------------

// Header represents the JSON stored at the head of a file, where the
// file header bytes should be less than STORE_PAGE_SIZE length.
type Header struct {
	Version       uint32 // The file format / STORE_VERSION.
	CreatedAt     string
	CreatedEndian string // The endian() of the file creator.
}

// Footer represents a footer record persisted in a file, and also
// implements the moss.Snapshot interface.
type Footer struct {
	m    sync.Mutex // Protects the fields that follow.
	refs int

	SegmentLocs SegmentLocs // Persisted; older SegmentLoc's come first.

	ss *segmentStack // Ephemeral.

	fileName string // Ephemeral; file name; "" when unpersisted.
	filePos  int64  // Ephemeral; byte offset of footer; <= 0 when unpersisted.
}

// --------------------------------------------------------

// Persist helps the store implement the lower-level-update func.  The
// higher snapshot may be nil.
func (s *Store) persist(higher Snapshot, persistOptions StorePersistOptions) (
	Snapshot, error) {
	wasCompacted, err := s.compactMaybe(higher, persistOptions)
	if err != nil {
		return nil, err
	}
	if wasCompacted {
		return s.Snapshot()
	}

	// If no dirty higher items, we're still clean, so just snapshot.
	// If in case of ReadOnly mode, just snapshot.
	if higher == nil || s.Options().CollectionOptions.ReadOnly {
		return s.Snapshot()
	}

	ss, ok := higher.(*segmentStack)
	if !ok {
		return nil, fmt.Errorf("store: can only persist segmentStack")
	}

	fref, file, err := s.startOrReuseFile()
	if err != nil {
		return nil, err
	}

	// TODO: Pre-allocate file space up front?

	ss.ensureSorted(0, len(ss.a)-1)

	numSegmentLocs := len(ss.a)

	s.m.Lock()
	if s.footer != nil {
		numSegmentLocs += len(s.footer.SegmentLocs)
	}
	segmentLocs := make([]SegmentLoc, 0, numSegmentLocs)
	if s.footer != nil {
		segmentLocs = append(segmentLocs, s.footer.SegmentLocs...)
	}
	s.m.Unlock()

	for _, segment := range ss.a {
		segmentLoc, err := s.persistSegment(file, segment)
		if err != nil {
			fref.DecRef()
			return nil, err
		}

		segmentLocs = append(segmentLocs, segmentLoc)
	}

	footer := &Footer{refs: 1, SegmentLocs: segmentLocs}

	err = footer.loadSegments(s.options, fref)
	if err != nil {
		fref.DecRef()
		return nil, err
	}

	err = s.persistFooter(file, footer, !persistOptions.NoSync)
	if err != nil {
		footer.DecRef()
		return nil, err
	}

	footer.AddRef() // One ref-count will be held by the store.

	s.m.Lock()
	prevFooter := s.footer
	s.footer = footer
	s.totPersists++
	s.m.Unlock()

	if prevFooter != nil {
		prevFooter.DecRef()
	}

	return footer, nil // The other ref-count returned to caller.
}

// --------------------------------------------------------

// startOrReuseFile either creates a new file or reuses the file from
// the last/current footer.
func (s *Store) startOrReuseFile() (fref *FileRef, file File, err error) {
	s.m.Lock()
	defer s.m.Unlock()

	if s.footer != nil {
		slocs, _ := s.footer.SegmentStack()
		defer s.footer.DecRef()

		if len(slocs) > 0 {
			fref := slocs[0].mref.fref
			file := fref.AddRef()

			return fref, file, nil
		}
	}

	return s.startFileLOCKED()
}

func (s *Store) startFileLOCKED() (*FileRef, File, error) {
	fname, file, err := s.createNextFileLOCKED()
	if err != nil {
		return nil, nil, err
	}

	if err = s.persistHeader(file); err != nil {
		file.Close()

		os.Remove(path.Join(s.dir, fname))

		return nil, nil, err
	}

	return &FileRef{file: file, refs: 1}, file, nil
}

func (s *Store) createNextFileLOCKED() (string, File, error) {
	// File to be opened in RDWR mode here because this is either
	// invoked by the persister or the compactor either of which
	// do not execute in the ReadOnly mode
	fname := FormatFName(s.nextFNameSeq)
	s.nextFNameSeq++

	file, err := s.options.OpenFile(path.Join(s.dir, fname),
		os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return "", nil, err
	}

	return fname, file, nil
}

// --------------------------------------------------------

func HeaderLength() uint64 {
	return uint64(STORE_PAGE_SIZE)
}

func (s *Store) persistHeader(file File) error {
	buf, err := json.Marshal(Header{
		Version:       STORE_VERSION,
		CreatedAt:     time.Now().Format(time.RFC3339),
		CreatedEndian: endian(),
	})
	if err != nil {
		return err
	}

	str := "moss-data-store:\n" + string(buf) + "\n"
	if len(str) >= STORE_PAGE_SIZE {
		return fmt.Errorf("store: header size too big")
	}
	str = str + strings.Repeat("\n", STORE_PAGE_SIZE-len(str))

	n, err := file.WriteAt([]byte(str), 0)
	if err != nil {
		return err
	}
	if n != len(str) {
		return fmt.Errorf("store: could not write full header")
	}

	return nil
}

func checkHeader(file File) error {
	buf := make([]byte, STORE_PAGE_SIZE)

	n, err := file.ReadAt(buf, int64(0))
	if err != nil {
		return err
	}
	if n != len(buf) {
		return fmt.Errorf("store: readHeader too short")
	}

	lines := strings.Split(string(buf), "\n")
	if len(lines) < 2 {
		return fmt.Errorf("store: readHeader not enough lines")
	}
	if lines[0] != "moss-data-store:" {
		return fmt.Errorf("store: readHeader wrong file prefix")
	}

	hdr := Header{}
	err = json.Unmarshal([]byte(lines[1]), &hdr)
	if err != nil {
		return err
	}
	if hdr.Version != STORE_VERSION {
		return fmt.Errorf("store: readHeader wrong version")
	}
	if hdr.CreatedEndian != endian() {
		return fmt.Errorf("store: readHeader endian of file was: %s, need: %s",
			hdr.CreatedEndian, endian())
	}

	return nil
}

// --------------------------------------------------------

func (s *Store) persistSegment(file File, segIn Segment) (rv SegmentLoc, err error) {
	segPersister, ok := segIn.(SegmentPersister)
	if !ok {
		return rv, fmt.Errorf("store: can only persist SegmentPersister type")
	}
	return segPersister.Persist(file)
}

// --------------------------------------------------------

// ParseFNameSeq parses a file name like "data-000123.moss" into 123.
func ParseFNameSeq(fname string) (int64, error) {
	seqStr := fname[len(STORE_PREFIX) : len(fname)-len(STORE_SUFFIX)]
	return strconv.ParseInt(seqStr, 16, 64)
}

// FormatFName returns a file name like "data-000123.moss" given a seq of 123.
func FormatFName(seq int64) string {
	return fmt.Sprintf("%s%016x%s", STORE_PREFIX, seq, STORE_SUFFIX)
}

// --------------------------------------------------------

// pageAlign returns the pos if it's at the start of a page.  Else,
// pageAlign() returns pos bumped up to the next multiple of
// STORE_PAGE_SIZE.
func pageAlign(pos int64) int64 {
	rem := pos % int64(STORE_PAGE_SIZE)
	if rem != 0 {
		return pos + int64(STORE_PAGE_SIZE) - rem
	}
	return pos
}

// pageOffset returns the page offset for a given pos.
func pageOffset(pos, pageSize int64) int64 {
	rem := pos % pageSize
	if rem != 0 {
		return pos - rem
	}
	return pos
}

// --------------------------------------------------------

func openStore(dir string, options StoreOptions) (*Store, error) {
	fileInfos, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}

	var maxFNameSeq int64

	var fnames []string
	for _, fileInfo := range fileInfos { // Find candidate file names.
		fname := fileInfo.Name()
		if strings.HasPrefix(fname, STORE_PREFIX) &&
			strings.HasSuffix(fname, STORE_SUFFIX) {
			fnames = append(fnames, fname)
		}

		fnameSeq, err := ParseFNameSeq(fname)
		if err == nil && fnameSeq > maxFNameSeq {
			maxFNameSeq = fnameSeq
		}
	}

	if options.OpenFile == nil {
		options.OpenFile =
			func(name string, flag int, perm os.FileMode) (File, error) {
				return os.OpenFile(name, flag, perm)
			}
	}

	if len(fnames) <= 0 {
		emptyFooter := &Footer{
			refs: 1,
			ss: &segmentStack{
				options: &options.CollectionOptions,
			},
		}

		return &Store{
			dir:          dir,
			options:      &options,
			refs:         1,
			footer:       emptyFooter,
			nextFNameSeq: 1,
		}, nil
	}

	sort.Strings(fnames)

	for i := len(fnames) - 1; i >= 0; i-- {
		var flag int
		var perm os.FileMode
		if options.CollectionOptions.ReadOnly {
			flag = os.O_RDONLY
			perm = 0400
		} else {
			flag = os.O_RDWR
			perm = 0600
		}
		file, err := options.OpenFile(path.Join(dir, fnames[i]), flag, perm)
		if err != nil {
			continue
		}

		err = checkHeader(file)
		if err != nil {
			file.Close()
			return nil, err
		}

		footer, err := ReadFooter(&options, file) // Footer owns file on success.
		if err != nil {
			file.Close()
			continue
		}

		if !options.KeepFiles {
			err := removeFiles(dir, append(fnames[0:i], fnames[i+1:]...))
			if err != nil {
				footer.Close()
				return nil, err
			}
		}

		return &Store{
			dir:          dir,
			options:      &options,
			refs:         1,
			footer:       footer,
			nextFNameSeq: maxFNameSeq + 1,
		}, nil
	}

	return nil, fmt.Errorf("store: could not successfully open/parse any file")
}

// --------------------------------------------------------

func (store *Store) openCollection(
	options StoreOptions,
	persistOptions StorePersistOptions) (Collection, error) {
	storeSnapshotInit, err := store.Snapshot()
	if err != nil {
		return nil, err
	}

	co := options.CollectionOptions
	co.LowerLevelInit = storeSnapshotInit
	co.LowerLevelUpdate = func(higher Snapshot) (Snapshot, error) {
		ss, err := store.Persist(higher, persistOptions)
		if err != nil {
			return nil, err
		}

		if storeSnapshotInit != nil {
			storeSnapshotInit.Close()
			storeSnapshotInit = nil
		}

		return ss, err
	}

	coll, err := NewCollection(co)
	if err != nil {
		storeSnapshotInit.Close()
		return nil, err
	}

	err = coll.Start()
	if err != nil {
		storeSnapshotInit.Close()
		return nil, err
	}

	return coll, nil
}

// --------------------------------------------------------

func removeFiles(dir string, fnames []string) error {
	for _, fname := range fnames {
		err := os.Remove(path.Join(dir, fname))
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before><commit_msg>fix logic bug with publish task<commit_after><|endoftext|>


<commit_before><commit_msg>Minor improvments to tests<commit_after><|endoftext|>