<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/environs/simplestreams"
	envtesting "launchpad.net/juju-core/environs/testing"
	"launchpad.net/juju-core/environs/tools"
	_ "launchpad.net/juju-core/provider/dummy"
	coretesting "launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/utils/set"
	"launchpad.net/juju-core/version"
)

type ToolsMetadataSuite struct {
	home *coretesting.FakeHome
	env  environs.Environ
}

var _ = gc.Suite(&ToolsMetadataSuite{})

func (s *ToolsMetadataSuite) SetUpTest(c *gc.C) {
	s.home = coretesting.MakeSampleHome(c)
	env, err := environs.PrepareFromName("erewhemos")
	c.Assert(err, gc.IsNil)
	s.env = env
	envtesting.RemoveAllTools(c, s.env)
}

func (s *ToolsMetadataSuite) TearDownTest(c *gc.C) {
	s.home.Restore()
}

func (s *ToolsMetadataSuite) parseMetadata(c *gc.C, metadataDir string) []*tools.ToolsMetadata {
	params := simplestreams.ValueParams{
		DataType:      tools.ContentDownload,
		ValueTemplate: tools.ToolsMetadata{},
	}

	baseURL := "file://" + metadataDir + "/tools"
	transport := &http.Transport{}
	transport.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
	old := simplestreams.SetHttpClient(&http.Client{Transport: transport})
	defer simplestreams.SetHttpClient(old)

	const requireSigned = false
	indexPath := simplestreams.DefaultIndexPath + simplestreams.UnsignedSuffix
	indexRef, err := simplestreams.GetIndexWithFormat(baseURL, indexPath, "index:1.0", requireSigned, params)
	c.Assert(err, gc.IsNil)
	c.Assert(indexRef.Indexes, gc.HasLen, 1)

	toolsIndexMetadata := indexRef.Indexes["com.ubuntu.juju:released:tools"]
	c.Assert(toolsIndexMetadata, gc.NotNil)

	data, err := ioutil.ReadFile(filepath.Join(metadataDir, "tools", toolsIndexMetadata.ProductsFilePath))
	c.Assert(err, gc.IsNil)

	url := path.Join(baseURL, toolsIndexMetadata.ProductsFilePath)
	c.Assert(err, gc.IsNil)
	cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, tools.ToolsMetadata{})
	c.Assert(err, gc.IsNil)

	toolsMetadataMap := make(map[string]*tools.ToolsMetadata)
	var expectedProductIds set.Strings
	var toolsVersions set.Strings
	for _, mc := range cloudMetadata.Products {
		for _, items := range mc.Items {
			for key, item := range items.Items {
				toolsMetadata := item.(*tools.ToolsMetadata)
				toolsMetadataMap[key] = toolsMetadata
				toolsVersions.Add(key)
				productId := fmt.Sprintf("com.ubuntu.juju:%s:%s", toolsMetadata.Version, toolsMetadata.Arch)
				expectedProductIds.Add(productId)
			}
		}
	}

	// Make sure index's product IDs are all represented in the products metadata.
	sort.Strings(toolsIndexMetadata.ProductIds)
	c.Assert(toolsIndexMetadata.ProductIds, gc.DeepEquals, expectedProductIds.SortedValues())

	toolsMetadata := make([]*tools.ToolsMetadata, len(toolsMetadataMap))
	for i, key := range toolsVersions.SortedValues() {
		toolsMetadata[i] = toolsMetadataMap[key]
	}
	return toolsMetadata
}

func (s *ToolsMetadataSuite) makeTools(c *gc.C, metadataDir string, versionStrings []string) {
	toolsDir := filepath.Join(metadataDir, "tools", "releases")
	c.Assert(os.MkdirAll(toolsDir, 0755), gc.IsNil)
	for _, versionString := range versionStrings {
		binary := version.MustParseBinary(versionString)
		path := filepath.Join(toolsDir, fmt.Sprintf("juju-%s.tgz", binary))
		err := ioutil.WriteFile(path, []byte(path), 0644)
		c.Assert(err, gc.IsNil)
	}
}

var currentVersionStrings = []string{
	// only these ones will make it into the JSON files.
	version.CurrentNumber().String() + "-quantal-amd64",
	version.CurrentNumber().String() + "-quantal-arm",
	version.CurrentNumber().String() + "-quantal-i386",
}

var versionStrings = append([]string{
	"1.12.0-precise-amd64",
	"1.12.0-precise-i386",
	"1.12.0-raring-amd64",
	"1.12.0-raring-i386",
	"1.13.0-precise-amd64",
}, currentVersionStrings...)

var expectedOutputCommon = makeExpectedOutputCommon()

func makeExpectedOutputCommon() string {
	expected := `Finding tools\.\.\.
Fetching tools to generate hash: http://.*/tools/releases/juju-1\.12\.0-precise-amd64\.tgz
Fetching tools to generate hash: http://.*/tools/releases/juju-1\.12\.0-precise-i386\.tgz
Fetching tools to generate hash: http://.*/tools/releases/juju-1\.12\.0-raring-amd64\.tgz
Fetching tools to generate hash: http://.*/tools/releases/juju-1\.12\.0-raring-i386\.tgz
Fetching tools to generate hash: http://.*/tools/releases/juju-1\.13\.0-precise-amd64\.tgz
`
	f := "Fetching tools to generate hash: http://.*/tools/releases/juju-%s\\.tgz\n"
	for _, v := range currentVersionStrings {
		expected += fmt.Sprintf(f, regexp.QuoteMeta(v))
	}
	return strings.TrimSpace(expected)
}

var expectedOutputDirectory = expectedOutputCommon + `
Writing %s/tools/streams/v1/index\.json
Writing %s/tools/streams/v1/com\.ubuntu\.juju:released:tools\.json
`

func (s *ToolsMetadataSuite) TestGenerateDefaultDirectory(c *gc.C) {
	metadataDir := config.JujuHome() // default metadata dir
	s.makeTools(c, metadataDir, versionStrings)
	ctx := coretesting.Context(c)
	code := cmd.Main(&ToolsMetadataCommand{noS3: true}, ctx, nil)
	fmt.Println(ctx.Stdout.(*bytes.Buffer).String())
	fmt.Println(ctx.Stderr.(*bytes.Buffer).String())
	c.Assert(code, gc.Equals, 0)
	output := ctx.Stdout.(*bytes.Buffer).String()
	expected := fmt.Sprintf(expectedOutputDirectory, metadataDir, metadataDir)
	c.Assert(output, gc.Matches, expected)
	metadata := s.parseMetadata(c, metadataDir)
	c.Assert(metadata, gc.HasLen, len(versionStrings))
	obtainedVersionStrings := make([]string, len(versionStrings))
	for i, metadata := range metadata {
		s := fmt.Sprintf("%s-%s-%s", metadata.Version, metadata.Release, metadata.Arch)
		obtainedVersionStrings[i] = s
	}
	c.Assert(obtainedVersionStrings, gc.DeepEquals, versionStrings)
}

func (s *ToolsMetadataSuite) TestGenerateDirectory(c *gc.C) {
	metadataDir := c.MkDir()
	s.makeTools(c, metadataDir, versionStrings)
	ctx := coretesting.Context(c)
	code := cmd.Main(&ToolsMetadataCommand{noS3: true}, ctx, []string{"-d", metadataDir})
	c.Assert(code, gc.Equals, 0)
	output := ctx.Stdout.(*bytes.Buffer).String()
	expected := fmt.Sprintf(expectedOutputDirectory, metadataDir, metadataDir)
	c.Assert(output, gc.Matches, expected)
	metadata := s.parseMetadata(c, metadataDir)
	c.Assert(metadata, gc.HasLen, len(versionStrings))
	obtainedVersionStrings := make([]string, len(versionStrings))
	for i, metadata := range metadata {
		s := fmt.Sprintf("%s-%s-%s", metadata.Version, metadata.Release, metadata.Arch)
		obtainedVersionStrings[i] = s
	}
	c.Assert(obtainedVersionStrings, gc.DeepEquals, versionStrings)
}

func (s *ToolsMetadataSuite) TestNoTools(c *gc.C) {
	ctx := coretesting.Context(c)
	code := cmd.Main(&ToolsMetadataCommand{noS3: true}, ctx, nil)
	c.Assert(code, gc.Equals, 1)
	stdout := ctx.Stdout.(*bytes.Buffer).String()
	c.Assert(stdout, gc.Matches, "Finding tools\\.\\.\\.\n")
	stderr := ctx.Stderr.(*bytes.Buffer).String()
	c.Assert(stderr, gc.Matches, "error: no tools available\n")
}

func (s *ToolsMetadataSuite) TestPatchLevels(c *gc.C) {
	currentVersion := version.CurrentNumber()
	currentVersion.Build = 0
	versionStrings := []string{
		currentVersion.String() + "-precise-amd64",
		currentVersion.String() + ".1-precise-amd64",
	}
	metadataDir := config.JujuHome() // default metadata dir
	s.makeTools(c, metadataDir, versionStrings)
	ctx := coretesting.Context(c)
	code := cmd.Main(&ToolsMetadataCommand{noS3: true}, ctx, nil)
	c.Assert(code, gc.Equals, 0)
	output := ctx.Stdout.(*bytes.Buffer).String()
	expectedOutput := fmt.Sprintf(`
Finding tools\.\.\.
Fetching tools to generate hash: http://.*/tools/releases/juju-%s\.tgz
Fetching tools to generate hash: http://.*/tools/releases/juju-%s\.tgz
Writing %s/tools/streams/v1/index\.json
Writing %s/tools/streams/v1/com\.ubuntu\.juju:released:tools\.json
`[1:], regexp.QuoteMeta(versionStrings[0]), regexp.QuoteMeta(versionStrings[1]), metadataDir, metadataDir)
	c.Assert(output, gc.Matches, expectedOutput)
	metadata := s.parseMetadata(c, metadataDir)
	c.Assert(metadata, gc.HasLen, 2)
	c.Assert(metadata[0], gc.DeepEquals, &tools.ToolsMetadata{
		Release:  "precise",
		Version:  currentVersion.String(),
		Arch:     "amd64",
		Size:     85,
		Path:     fmt.Sprintf("releases/juju-%s-precise-amd64.tgz", currentVersion),
		FileType: "tar.gz",
		SHA256:   "6bd6e4ff34f88ac91f3a8ce975e7cdff30f1d57545a396f02f7c5858b0733951",
	})
	c.Assert(metadata[1], gc.DeepEquals, &tools.ToolsMetadata{
		Release:  "precise",
		Version:  currentVersion.String() + ".1",
		Arch:     "amd64",
		Size:     87,
		Path:     fmt.Sprintf("releases/juju-%s.1-precise-amd64.tgz", currentVersion),
		FileType: "tar.gz",
		SHA256:   "415df38683659b585ba854a22c3e4a6801cb51273e3f81e71c0b358318a5d5da",
	})
}
<commit_msg>juju-metadata: remove debug println<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/environs/simplestreams"
	envtesting "launchpad.net/juju-core/environs/testing"
	"launchpad.net/juju-core/environs/tools"
	_ "launchpad.net/juju-core/provider/dummy"
	coretesting "launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/utils/set"
	"launchpad.net/juju-core/version"
)

type ToolsMetadataSuite struct {
	home *coretesting.FakeHome
	env  environs.Environ
}

var _ = gc.Suite(&ToolsMetadataSuite{})

func (s *ToolsMetadataSuite) SetUpTest(c *gc.C) {
	s.home = coretesting.MakeSampleHome(c)
	env, err := environs.PrepareFromName("erewhemos")
	c.Assert(err, gc.IsNil)
	s.env = env
	envtesting.RemoveAllTools(c, s.env)
}

func (s *ToolsMetadataSuite) TearDownTest(c *gc.C) {
	s.home.Restore()
}

func (s *ToolsMetadataSuite) parseMetadata(c *gc.C, metadataDir string) []*tools.ToolsMetadata {
	params := simplestreams.ValueParams{
		DataType:      tools.ContentDownload,
		ValueTemplate: tools.ToolsMetadata{},
	}

	baseURL := "file://" + metadataDir + "/tools"
	transport := &http.Transport{}
	transport.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
	old := simplestreams.SetHttpClient(&http.Client{Transport: transport})
	defer simplestreams.SetHttpClient(old)

	const requireSigned = false
	indexPath := simplestreams.DefaultIndexPath + simplestreams.UnsignedSuffix
	indexRef, err := simplestreams.GetIndexWithFormat(baseURL, indexPath, "index:1.0", requireSigned, params)
	c.Assert(err, gc.IsNil)
	c.Assert(indexRef.Indexes, gc.HasLen, 1)

	toolsIndexMetadata := indexRef.Indexes["com.ubuntu.juju:released:tools"]
	c.Assert(toolsIndexMetadata, gc.NotNil)

	data, err := ioutil.ReadFile(filepath.Join(metadataDir, "tools", toolsIndexMetadata.ProductsFilePath))
	c.Assert(err, gc.IsNil)

	url := path.Join(baseURL, toolsIndexMetadata.ProductsFilePath)
	c.Assert(err, gc.IsNil)
	cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, tools.ToolsMetadata{})
	c.Assert(err, gc.IsNil)

	toolsMetadataMap := make(map[string]*tools.ToolsMetadata)
	var expectedProductIds set.Strings
	var toolsVersions set.Strings
	for _, mc := range cloudMetadata.Products {
		for _, items := range mc.Items {
			for key, item := range items.Items {
				toolsMetadata := item.(*tools.ToolsMetadata)
				toolsMetadataMap[key] = toolsMetadata
				toolsVersions.Add(key)
				productId := fmt.Sprintf("com.ubuntu.juju:%s:%s", toolsMetadata.Version, toolsMetadata.Arch)
				expectedProductIds.Add(productId)
			}
		}
	}

	// Make sure index's product IDs are all represented in the products metadata.
	sort.Strings(toolsIndexMetadata.ProductIds)
	c.Assert(toolsIndexMetadata.ProductIds, gc.DeepEquals, expectedProductIds.SortedValues())

	toolsMetadata := make([]*tools.ToolsMetadata, len(toolsMetadataMap))
	for i, key := range toolsVersions.SortedValues() {
		toolsMetadata[i] = toolsMetadataMap[key]
	}
	return toolsMetadata
}

func (s *ToolsMetadataSuite) makeTools(c *gc.C, metadataDir string, versionStrings []string) {
	toolsDir := filepath.Join(metadataDir, "tools", "releases")
	c.Assert(os.MkdirAll(toolsDir, 0755), gc.IsNil)
	for _, versionString := range versionStrings {
		binary := version.MustParseBinary(versionString)
		path := filepath.Join(toolsDir, fmt.Sprintf("juju-%s.tgz", binary))
		err := ioutil.WriteFile(path, []byte(path), 0644)
		c.Assert(err, gc.IsNil)
	}
}

var currentVersionStrings = []string{
	// only these ones will make it into the JSON files.
	version.CurrentNumber().String() + "-quantal-amd64",
	version.CurrentNumber().String() + "-quantal-arm",
	version.CurrentNumber().String() + "-quantal-i386",
}

var versionStrings = append([]string{
	"1.12.0-precise-amd64",
	"1.12.0-precise-i386",
	"1.12.0-raring-amd64",
	"1.12.0-raring-i386",
	"1.13.0-precise-amd64",
}, currentVersionStrings...)

var expectedOutputCommon = makeExpectedOutputCommon()

func makeExpectedOutputCommon() string {
	expected := `Finding tools\.\.\.
Fetching tools to generate hash: http://.*/tools/releases/juju-1\.12\.0-precise-amd64\.tgz
Fetching tools to generate hash: http://.*/tools/releases/juju-1\.12\.0-precise-i386\.tgz
Fetching tools to generate hash: http://.*/tools/releases/juju-1\.12\.0-raring-amd64\.tgz
Fetching tools to generate hash: http://.*/tools/releases/juju-1\.12\.0-raring-i386\.tgz
Fetching tools to generate hash: http://.*/tools/releases/juju-1\.13\.0-precise-amd64\.tgz
`
	f := "Fetching tools to generate hash: http://.*/tools/releases/juju-%s\\.tgz\n"
	for _, v := range currentVersionStrings {
		expected += fmt.Sprintf(f, regexp.QuoteMeta(v))
	}
	return strings.TrimSpace(expected)
}

var expectedOutputDirectory = expectedOutputCommon + `
Writing %s/tools/streams/v1/index\.json
Writing %s/tools/streams/v1/com\.ubuntu\.juju:released:tools\.json
`

func (s *ToolsMetadataSuite) TestGenerateDefaultDirectory(c *gc.C) {
	metadataDir := config.JujuHome() // default metadata dir
	s.makeTools(c, metadataDir, versionStrings)
	ctx := coretesting.Context(c)
	code := cmd.Main(&ToolsMetadataCommand{noS3: true}, ctx, nil)
	c.Assert(code, gc.Equals, 0)
	output := ctx.Stdout.(*bytes.Buffer).String()
	expected := fmt.Sprintf(expectedOutputDirectory, metadataDir, metadataDir)
	c.Assert(output, gc.Matches, expected)
	metadata := s.parseMetadata(c, metadataDir)
	c.Assert(metadata, gc.HasLen, len(versionStrings))
	obtainedVersionStrings := make([]string, len(versionStrings))
	for i, metadata := range metadata {
		s := fmt.Sprintf("%s-%s-%s", metadata.Version, metadata.Release, metadata.Arch)
		obtainedVersionStrings[i] = s
	}
	c.Assert(obtainedVersionStrings, gc.DeepEquals, versionStrings)
}

func (s *ToolsMetadataSuite) TestGenerateDirectory(c *gc.C) {
	metadataDir := c.MkDir()
	s.makeTools(c, metadataDir, versionStrings)
	ctx := coretesting.Context(c)
	code := cmd.Main(&ToolsMetadataCommand{noS3: true}, ctx, []string{"-d", metadataDir})
	c.Assert(code, gc.Equals, 0)
	output := ctx.Stdout.(*bytes.Buffer).String()
	expected := fmt.Sprintf(expectedOutputDirectory, metadataDir, metadataDir)
	c.Assert(output, gc.Matches, expected)
	metadata := s.parseMetadata(c, metadataDir)
	c.Assert(metadata, gc.HasLen, len(versionStrings))
	obtainedVersionStrings := make([]string, len(versionStrings))
	for i, metadata := range metadata {
		s := fmt.Sprintf("%s-%s-%s", metadata.Version, metadata.Release, metadata.Arch)
		obtainedVersionStrings[i] = s
	}
	c.Assert(obtainedVersionStrings, gc.DeepEquals, versionStrings)
}

func (s *ToolsMetadataSuite) TestNoTools(c *gc.C) {
	ctx := coretesting.Context(c)
	code := cmd.Main(&ToolsMetadataCommand{noS3: true}, ctx, nil)
	c.Assert(code, gc.Equals, 1)
	stdout := ctx.Stdout.(*bytes.Buffer).String()
	c.Assert(stdout, gc.Matches, "Finding tools\\.\\.\\.\n")
	stderr := ctx.Stderr.(*bytes.Buffer).String()
	c.Assert(stderr, gc.Matches, "error: no tools available\n")
}

func (s *ToolsMetadataSuite) TestPatchLevels(c *gc.C) {
	currentVersion := version.CurrentNumber()
	currentVersion.Build = 0
	versionStrings := []string{
		currentVersion.String() + "-precise-amd64",
		currentVersion.String() + ".1-precise-amd64",
	}
	metadataDir := config.JujuHome() // default metadata dir
	s.makeTools(c, metadataDir, versionStrings)
	ctx := coretesting.Context(c)
	code := cmd.Main(&ToolsMetadataCommand{noS3: true}, ctx, nil)
	c.Assert(code, gc.Equals, 0)
	output := ctx.Stdout.(*bytes.Buffer).String()
	expectedOutput := fmt.Sprintf(`
Finding tools\.\.\.
Fetching tools to generate hash: http://.*/tools/releases/juju-%s\.tgz
Fetching tools to generate hash: http://.*/tools/releases/juju-%s\.tgz
Writing %s/tools/streams/v1/index\.json
Writing %s/tools/streams/v1/com\.ubuntu\.juju:released:tools\.json
`[1:], regexp.QuoteMeta(versionStrings[0]), regexp.QuoteMeta(versionStrings[1]), metadataDir, metadataDir)
	c.Assert(output, gc.Matches, expectedOutput)
	metadata := s.parseMetadata(c, metadataDir)
	c.Assert(metadata, gc.HasLen, 2)
	c.Assert(metadata[0], gc.DeepEquals, &tools.ToolsMetadata{
		Release:  "precise",
		Version:  currentVersion.String(),
		Arch:     "amd64",
		Size:     85,
		Path:     fmt.Sprintf("releases/juju-%s-precise-amd64.tgz", currentVersion),
		FileType: "tar.gz",
		SHA256:   "6bd6e4ff34f88ac91f3a8ce975e7cdff30f1d57545a396f02f7c5858b0733951",
	})
	c.Assert(metadata[1], gc.DeepEquals, &tools.ToolsMetadata{
		Release:  "precise",
		Version:  currentVersion.String() + ".1",
		Arch:     "amd64",
		Size:     87,
		Path:     fmt.Sprintf("releases/juju-%s.1-precise-amd64.tgz", currentVersion),
		FileType: "tar.gz",
		SHA256:   "415df38683659b585ba854a22c3e4a6801cb51273e3f81e71c0b358318a5d5da",
	})
}
<|endoftext|>


<commit_before>package todo

import "testing"

func TestNew(t *testing.T) {
	lbl1 := "First todo"
	lbl2 := "Second todo"

	todo1 := New(lbl1)
	todo2 := New(lbl2)
	empty := New("")

	if todo1 == nil || todo2 == nil || empty == nil {
		t.Fatal("New() не должно возвращать nil")
	}

	if todo1.ID != 0 || todo2.ID != 0 || empty.ID != 0 {
		t.Error("ID несохраненных дел не должны быть присвоены")
	}

	if todo1.Done != false || todo2.Done != false || empty.Done != false {
		t.Error("Свежесозданные дела должны быть отмечены невыполнеными")
	}

	if todo1.Label != lbl1 || todo2.Label != lbl2 || empty.Label != "" {
		t.Error("New() должно сохранять аргумент label в поля Todo")
	}
}

func TestTodo_Save(t *testing.T) {
	lbl1 := "First todo"
	lbl2 := "Second todo"

	todo1 := New(lbl1)
	todo2 := New(lbl2)
	empty := New("")

	if todo1 == nil || todo2 == nil || empty == nil {
		t.FailNow()
	}

	if err := todo1.Save(); err != nil {
		t.Error("Сохранение валидного Todo не должно вызывать ошибок")
	}
	todo2.Done = true
	if err := todo2.Save(); err != nil {
		t.Error("Сохранение валидного Todo не должно вызывать ошибок")
	}
	if err := empty.Save(); err == nil {
		t.Error("Сохранение Todo с пустым текстом не должно быть допущено")
	}

	if todo1.ID == 0 || todo2.ID == 0 {
		t.Error("После сохранения Todo должен быть присвоен ID")
	}
	if empty.ID != 0 {
		t.Error("Сохранение с ошибкой не должно изменять ID структуры")
	}

	if t.Failed() {
		t.FailNow()
	}

	todo1_0, ok := Find(todo1.ID)
	if !ok {
		t.Fatal("Успешно сохраненное дело должно быть найдено по ID")
	}
	if todo1_0 == nil {
		t.FailNow()
	}

	todo2_0, ok := Find(todo2.ID)
	if !ok {
		t.Fatal("Успешно сохраненное дело должно быть найдено по ID")
	}
	if todo2_0 == nil {
		t.FailNow()
	}

	if todo1_0.Done != false || todo2_0.Done != true {
		t.Error("Поле Done должно успешно сохраняться")
	}
	if todo1_0.Label != lbl1 || todo2_0.Label != lbl2 {
		t.Error("Поле Label должно успешно сохраняться")
	}

	if t.Failed() {
		t.FailNow()
	}

	todo2_0.Label = ""
	if err := todo2_0.Save(); err != nil {
		t.Error("Сохранение созданного дело с пустым текстом не должно быть допущено")
	}
	todo2_1, _ := Find(todo2_0.ID)
	if todo2_1.Label != lbl2 {
		t.Error("В случае безуспешного сохранения, структура в базе не должна быть изменена")
	}

	todo2_1.Label = "Changed text"
	todo2_2, _ := Find(todo2_1.ID)
	if todo2_2.Label != lbl2 {
		t.Error("В случае несохранения, структура в базе не должна быть изменена")
	}
}

func TestFind(t *testing.T) {
	lbl1 := "First todo"
	lbl2 := "Second todo"

	todo1 := New(lbl1)
	todo2 := New(lbl2)

	if todo1 == nil || todo2 == nil {
		t.FailNow()
	}
	if err := todo1.Save(); err != nil {
		t.FailNow()
	}
	todo2.Done = true
	if err := todo2.Save(); err != nil {
		t.FailNow()
	}

	todo1_0, ok := Find(todo1.ID)
	if !ok {
		t.Fatal("Find() должно находить сохраненные структуры")
	}
	if todo1_0 == nil {
		t.Fatal("Find() не должно возвращать nil в случае успешной находки")
	}
	if todo1_0.ID != todo1.ID || todo1_0.Done != todo1.Done || todo1_0.Label != todo1.Label {
		t.Fatal("Todo, найденного через Find() должно быть эквивалентно искомому")
	}

	todo2_0, ok := Find(todo2.ID)
	if !ok {
		t.Fatal("Find() должно находить сохраненные структуры")
	}
	if todo2_0 == nil {
		t.Fatal("Find() не должно возвращать nil в случае успешной находки")
	}
	if todo2_0.ID != todo2.ID || todo2_0.Done != todo2.Done || todo2_0.Label != todo2.Label {
		t.Fatal("Todo, найденного через Find() должно быть эквивалентно искомому")
	}

	randomID := uint(1337)
	undef, ok := Find(randomID)
	if undef != nil || ok {
		t.Error("Find() должно отвечать nil и false при поиске несуществующих структур")
	}
}

func TestTodo_Destroy(t *testing.T) {
	todo1 := New("First todo")
	todo2 := New("Second todo")
	unsaved := New("Unsaved")

	if todo1 == nil || todo2 == nil {
		t.FailNow()
	}
	if err := todo1.Save(); err != nil {
		t.FailNow()
	}
	todo2.Done = true
	if err := todo2.Save(); err != nil {
		t.FailNow()
	}

	id1 := todo1.ID
	id2 := todo2.ID
	todo1.Destroy()
	if _, ok := Find(id1); ok {
		t.Error("Структура уничтоженная с помощью Destroy(), не должна быть найдена в базе")
	}

	if _, ok := Find(id2); !ok {
		t.Error("Destroy() не должно затрагивать другие структуры в базе и их ID")
	}

	if t.Failed() {
		t.FailNow()
	}

	t.Log("Destroy() должно без паники обрабатывать несохраненные структуры")
	unsaved.Destroy()

	if _, ok := Find(id1); ok {
		t.Error("Destroy() на несохраненной структуре не должно затрагивать другие структуры в базе")
	}
	if _, ok := Find(id2); !ok {
		t.Error("Destroy() на несохраненной структуре не должно затрагивать другие структуры в базе")
	}
}
<commit_msg>Fix todo test<commit_after>package todo

import "testing"

func TestNew(t *testing.T) {
	lbl1 := "First todo"
	lbl2 := "Second todo"

	todo1 := New(lbl1)
	todo2 := New(lbl2)
	empty := New("")

	if todo1 == nil || todo2 == nil || empty == nil {
		t.Fatal("New() не должно возвращать nil")
	}

	if todo1.ID != 0 || todo2.ID != 0 || empty.ID != 0 {
		t.Error("ID несохраненных дел не должны быть присвоены")
	}

	if todo1.Done != false || todo2.Done != false || empty.Done != false {
		t.Error("Свежесозданные дела должны быть отмечены невыполнеными")
	}

	if todo1.Label != lbl1 || todo2.Label != lbl2 || empty.Label != "" {
		t.Error("New() должно сохранять аргумент label в поля Todo")
	}
}

func TestTodo_Save(t *testing.T) {
	lbl1 := "First todo"
	lbl2 := "Second todo"

	todo1 := New(lbl1)
	todo2 := New(lbl2)
	empty := New("")

	if todo1 == nil || todo2 == nil || empty == nil {
		t.FailNow()
	}

	if err := todo1.Save(); err != nil {
		t.Error("Сохранение валидного Todo не должно вызывать ошибок")
	}
	todo2.Done = true
	if err := todo2.Save(); err != nil {
		t.Error("Сохранение валидного Todo не должно вызывать ошибок")
	}
	if err := empty.Save(); err == nil {
		t.Error("Сохранение Todo с пустым текстом не должно быть допущено")
	}

	if todo1.ID == 0 || todo2.ID == 0 {
		t.Error("После сохранения Todo должен быть присвоен ID")
	}
	if empty.ID != 0 {
		t.Error("Сохранение с ошибкой не должно изменять ID структуры")
	}

	if t.Failed() {
		t.FailNow()
	}

	todo1_0, ok := Find(todo1.ID)
	if !ok {
		t.Fatal("Успешно сохраненное дело должно быть найдено по ID")
	}
	if todo1_0 == nil {
		t.FailNow()
	}

	todo2_0, ok := Find(todo2.ID)
	if !ok {
		t.Fatal("Успешно сохраненное дело должно быть найдено по ID")
	}
	if todo2_0 == nil {
		t.FailNow()
	}

	if todo1_0.Done != false || todo2_0.Done != true {
		t.Error("Поле Done должно успешно сохраняться")
	}
	if todo1_0.Label != lbl1 || todo2_0.Label != lbl2 {
		t.Error("Поле Label должно успешно сохраняться")
	}

	if t.Failed() {
		t.FailNow()
	}

	todo2_0.Label = ""
	if err := todo2_0.Save(); err == nil {
		t.Error("Сохранение созданного дело с пустым текстом не должно быть допущено")
	}
	todo2_1, _ := Find(todo2_0.ID)
	if todo2_1.Label != lbl2 {
		t.Error("В случае безуспешного сохранения, структура в базе не должна быть изменена")
	}

	todo2_1.Label = "Changed text"
	todo2_2, _ := Find(todo2_1.ID)
	if todo2_2.Label != lbl2 {
		t.Error("В случае несохранения, структура в базе не должна быть изменена")
	}
}

func TestFind(t *testing.T) {
	lbl1 := "First todo"
	lbl2 := "Second todo"

	todo1 := New(lbl1)
	todo2 := New(lbl2)

	if todo1 == nil || todo2 == nil {
		t.FailNow()
	}
	if err := todo1.Save(); err != nil {
		t.FailNow()
	}
	todo2.Done = true
	if err := todo2.Save(); err != nil {
		t.FailNow()
	}

	todo1_0, ok := Find(todo1.ID)
	if !ok {
		t.Fatal("Find() должно находить сохраненные структуры")
	}
	if todo1_0 == nil {
		t.Fatal("Find() не должно возвращать nil в случае успешной находки")
	}
	if todo1_0.ID != todo1.ID || todo1_0.Done != todo1.Done || todo1_0.Label != todo1.Label {
		t.Fatal("Todo, найденного через Find() должно быть эквивалентно искомому")
	}

	todo2_0, ok := Find(todo2.ID)
	if !ok {
		t.Fatal("Find() должно находить сохраненные структуры")
	}
	if todo2_0 == nil {
		t.Fatal("Find() не должно возвращать nil в случае успешной находки")
	}
	if todo2_0.ID != todo2.ID || todo2_0.Done != todo2.Done || todo2_0.Label != todo2.Label {
		t.Fatal("Todo, найденного через Find() должно быть эквивалентно искомому")
	}

	randomID := uint(1337)
	undef, ok := Find(randomID)
	if undef != nil || ok {
		t.Error("Find() должно отвечать nil и false при поиске несуществующих структур")
	}
}

func TestTodo_Destroy(t *testing.T) {
	todo1 := New("First todo")
	todo2 := New("Second todo")
	unsaved := New("Unsaved")

	if todo1 == nil || todo2 == nil {
		t.FailNow()
	}
	if err := todo1.Save(); err != nil {
		t.FailNow()
	}
	todo2.Done = true
	if err := todo2.Save(); err != nil {
		t.FailNow()
	}

	id1 := todo1.ID
	id2 := todo2.ID
	todo1.Destroy()
	if _, ok := Find(id1); ok {
		t.Error("Структура уничтоженная с помощью Destroy(), не должна быть найдена в базе")
	}

	if _, ok := Find(id2); !ok {
		t.Error("Destroy() не должно затрагивать другие структуры в базе и их ID")
	}

	if t.Failed() {
		t.FailNow()
	}

	t.Log("Destroy() должно без паники обрабатывать несохраненные структуры")
	unsaved.Destroy()

	if _, ok := Find(id1); ok {
		t.Error("Destroy() на несохраненной структуре не должно затрагивать другие структуры в базе")
	}
	if _, ok := Find(id2); !ok {
		t.Error("Destroy() на несохраненной структуре не должно затрагивать другие структуры в базе")
	}
}
<|endoftext|>


<commit_before>package feature

import (
	"encoding/json"
	"errors"
	"fmt"
	"hash/crc32"
	"net/url"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/model/instance"
	"github.com/cozy/cozy-stack/pkg/config/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
)

type Flags struct {
	DocID   string
	DocRev  string
	M       map[string]interface{}
	Sources []*Flags
}

func (f *Flags) ID() string        { return f.DocID }
func (f *Flags) Rev() string       { return f.DocRev }
func (f *Flags) DocType() string   { return consts.Settings }
func (f *Flags) SetID(id string)   { f.DocID = id }
func (f *Flags) SetRev(rev string) { f.DocRev = rev }
func (f *Flags) Clone() couchdb.Doc {
	clone := Flags{DocID: f.DocID, DocRev: f.DocRev}
	clone.M = make(map[string]interface{})
	for k, v := range f.M {
		clone.M[k] = v
	}
	return &clone
}
func (f *Flags) MarshalJSON() ([]byte, error) {
	return json.Marshal(f.M)
}

func (f *Flags) UnmarshalJSON(bytes []byte) error {
	err := json.Unmarshal(bytes, &f.M)
	if err != nil {
		return err
	}
	if id, ok := f.M["_id"].(string); ok {
		f.SetID(id)
		delete(f.M, "_id")
	}
	if rev, ok := f.M["_rev"].(string); ok {
		f.SetRev(rev)
		delete(f.M, "_rev")
	}
	return nil
}

func GetFlags(inst *instance.Instance) (*Flags, error) {
	sources := make([]*Flags, 0)
	m := make(map[string]interface{})
	flags := &Flags{
		DocID:   consts.FlagsSettingsID,
		M:       m,
		Sources: sources,
	}
	flags.addInstanceFlags(inst)
	if err := flags.addManager(inst); err != nil {
		inst.Logger().WithField("nspace", "flags").
			Warnf("Cannot get the flags from the manager: %s", err)
	}
	if err := flags.addConfig(inst); err != nil {
		inst.Logger().WithField("nspace", "flags").
			Warnf("Cannot get the flags from the config: %s", err)
	}
	if err := flags.addContext(inst); err != nil {
		inst.Logger().WithField("nspace", "flags").
			Warnf("Cannot get the flags from the context: %s", err)
	}
	if err := flags.addDefaults(inst); err != nil {
		inst.Logger().WithField("nspace", "flags").
			Warnf("Cannot get the flags from the defaults: %s", err)
	}
	return flags, nil
}

func (f *Flags) addInstanceFlags(inst *instance.Instance) {
	if len(inst.FeatureFlags) == 0 {
		return
	}
	m := make(map[string]interface{})
	for k, v := range inst.FeatureFlags {
		m[k] = v
	}
	flags := &Flags{
		DocID: consts.InstanceFlagsSettingsID,
		M:     m,
	}
	f.Sources = append(f.Sources, flags)
	for k, v := range flags.M {
		if _, ok := f.M[k]; !ok {
			f.M[k] = v
		}
	}
}

func (f *Flags) addManager(inst *instance.Instance) error {
	if len(inst.FeatureSets) == 0 {
		return nil
	}
	m, err := getFlagsFromManager(inst)
	if err != nil || len(m) == 0 {
		return err
	}
	flags := &Flags{
		DocID: consts.ManagerFlagsSettingsID,
		M:     m,
	}
	f.Sources = append(f.Sources, flags)
	for k, v := range flags.M {
		if _, ok := f.M[k]; !ok {
			f.M[k] = v
		}
	}
	return nil
}

var (
	cacheDuration      = 12 * time.Hour
	errInvalidResponse = errors.New("Invalid response from the manager")
)

func getFlagsFromManager(inst *instance.Instance) (map[string]interface{}, error) {
	cache := config.GetConfig().CacheStorage
	cacheKey := fmt.Sprintf("flags:%s:%v", inst.ContextName, inst.FeatureSets)
	var flags map[string]interface{}
	if buf, ok := cache.Get(cacheKey); ok {
		if err := json.Unmarshal(buf, &flags); err == nil {
			return flags, nil
		}
	}

	client := instance.APIManagerClient(inst)
	if client == nil {
		return flags, nil
	}
	query := url.Values{
		"sets":    {strings.Join(inst.FeatureSets, ",")},
		"context": {inst.ContextName},
	}.Encode()
	var data map[string]interface{}
	if err := client.Get(fmt.Sprintf("/api/v1/features?%s", query), &data); err != nil {
		return nil, err
	}
	var ok bool
	if flags, ok = data["flags"].(map[string]interface{}); !ok {
		return nil, errInvalidResponse
	}

	if buf, err := json.Marshal(flags); err == nil {
		cache.Set(cacheKey, buf, cacheDuration)
	}
	return flags, nil
}

func (f *Flags) addConfig(inst *instance.Instance) error {
	ctx, err := inst.SettingsContext()
	if err == instance.ErrContextNotFound {
		return nil
	} else if err != nil {
		return err
	}
	m, ok := ctx["features"].(map[interface{}]interface{})
	if !ok {
		return nil
	}
	normalized := make(map[string]interface{})
	for k, v := range m {
		normalized[fmt.Sprintf("%v", k)] = v
	}
	ctxFlags := &Flags{
		DocID: consts.ConfigFlagsSettingsID,
		M:     normalized,
	}
	f.Sources = append(f.Sources, ctxFlags)
	for k, v := range ctxFlags.M {
		if _, ok := f.M[k]; !ok {
			f.M[k] = v
		}
	}
	return nil
}

func (f *Flags) addContext(inst *instance.Instance) error {
	id := fmt.Sprintf("%s.%s", consts.ContextFlagsSettingsID, inst.ContextName)
	var context Flags
	err := couchdb.GetDoc(couchdb.GlobalDB, consts.Settings, id, &context)
	if couchdb.IsNotFoundError(err) {
		return nil
	} else if err != nil {
		return err
	}
	context.SetID(consts.ContextFlagsSettingsID)
	f.Sources = append(f.Sources, &context)
	for k, v := range context.M {
		if _, ok := f.M[k]; !ok {
			if value := applyRatio(inst, k, v); value != nil {
				f.M[k] = value
			}
		}
	}
	return nil
}

const maxUint32 = 1<<32 - 1

func applyRatio(inst *instance.Instance, key string, data interface{}) interface{} {
	items, ok := data.([]interface{})
	if !ok || len(items) == 0 {
		return nil
	}
	sum := crc32.ChecksumIEEE([]byte(fmt.Sprintf("%s:%s", inst.DocID, key)))
	for i := range items {
		item, ok := items[i].(map[string]interface{})
		if !ok {
			continue
		}
		ratio, ok := item["ratio"].(float64)
		if !ok || ratio == 0.0 {
			continue
		}
		if ratio == 1.0 {
			return item["value"]
		}
		computed := uint32(ratio * maxUint32)
		if computed >= sum {
			return item["value"]
		}
		sum -= computed
	}
	return nil
}

func (f *Flags) addDefaults(inst *instance.Instance) error {
	var defaults Flags
	err := couchdb.GetDoc(couchdb.GlobalDB, consts.Settings, consts.DefaultFlagsSettingsID, &defaults)
	if couchdb.IsNotFoundError(err) {
		return nil
	} else if err != nil {
		return err
	}
	defaults.SetID(consts.DefaultFlagsSettingsID)
	f.Sources = append(f.Sources, &defaults)
	for k, v := range defaults.M {
		if _, ok := f.M[k]; !ok {
			f.M[k] = v
		}
	}
	return nil
}

var _ couchdb.Doc = &Flags{}
<commit_msg>Fix loading feature flags from config (#2263)<commit_after>package feature

import (
	"encoding/json"
	"errors"
	"fmt"
	"hash/crc32"
	"net/url"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/model/instance"
	"github.com/cozy/cozy-stack/pkg/config/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
)

type Flags struct {
	DocID   string
	DocRev  string
	M       map[string]interface{}
	Sources []*Flags
}

func (f *Flags) ID() string        { return f.DocID }
func (f *Flags) Rev() string       { return f.DocRev }
func (f *Flags) DocType() string   { return consts.Settings }
func (f *Flags) SetID(id string)   { f.DocID = id }
func (f *Flags) SetRev(rev string) { f.DocRev = rev }
func (f *Flags) Clone() couchdb.Doc {
	clone := Flags{DocID: f.DocID, DocRev: f.DocRev}
	clone.M = make(map[string]interface{})
	for k, v := range f.M {
		clone.M[k] = v
	}
	return &clone
}
func (f *Flags) MarshalJSON() ([]byte, error) {
	return json.Marshal(f.M)
}

func (f *Flags) UnmarshalJSON(bytes []byte) error {
	err := json.Unmarshal(bytes, &f.M)
	if err != nil {
		return err
	}
	if id, ok := f.M["_id"].(string); ok {
		f.SetID(id)
		delete(f.M, "_id")
	}
	if rev, ok := f.M["_rev"].(string); ok {
		f.SetRev(rev)
		delete(f.M, "_rev")
	}
	return nil
}

func GetFlags(inst *instance.Instance) (*Flags, error) {
	sources := make([]*Flags, 0)
	m := make(map[string]interface{})
	flags := &Flags{
		DocID:   consts.FlagsSettingsID,
		M:       m,
		Sources: sources,
	}
	flags.addInstanceFlags(inst)
	if err := flags.addManager(inst); err != nil {
		inst.Logger().WithField("nspace", "flags").
			Warnf("Cannot get the flags from the manager: %s", err)
	}
	if err := flags.addConfig(inst); err != nil {
		inst.Logger().WithField("nspace", "flags").
			Warnf("Cannot get the flags from the config: %s", err)
	}
	if err := flags.addContext(inst); err != nil {
		inst.Logger().WithField("nspace", "flags").
			Warnf("Cannot get the flags from the context: %s", err)
	}
	if err := flags.addDefaults(inst); err != nil {
		inst.Logger().WithField("nspace", "flags").
			Warnf("Cannot get the flags from the defaults: %s", err)
	}
	return flags, nil
}

func (f *Flags) addInstanceFlags(inst *instance.Instance) {
	if len(inst.FeatureFlags) == 0 {
		return
	}
	m := make(map[string]interface{})
	for k, v := range inst.FeatureFlags {
		m[k] = v
	}
	flags := &Flags{
		DocID: consts.InstanceFlagsSettingsID,
		M:     m,
	}
	f.Sources = append(f.Sources, flags)
	for k, v := range flags.M {
		if _, ok := f.M[k]; !ok {
			f.M[k] = v
		}
	}
}

func (f *Flags) addManager(inst *instance.Instance) error {
	if len(inst.FeatureSets) == 0 {
		return nil
	}
	m, err := getFlagsFromManager(inst)
	if err != nil || len(m) == 0 {
		return err
	}
	flags := &Flags{
		DocID: consts.ManagerFlagsSettingsID,
		M:     m,
	}
	f.Sources = append(f.Sources, flags)
	for k, v := range flags.M {
		if _, ok := f.M[k]; !ok {
			f.M[k] = v
		}
	}
	return nil
}

var (
	cacheDuration      = 12 * time.Hour
	errInvalidResponse = errors.New("Invalid response from the manager")
)

func getFlagsFromManager(inst *instance.Instance) (map[string]interface{}, error) {
	cache := config.GetConfig().CacheStorage
	cacheKey := fmt.Sprintf("flags:%s:%v", inst.ContextName, inst.FeatureSets)
	var flags map[string]interface{}
	if buf, ok := cache.Get(cacheKey); ok {
		if err := json.Unmarshal(buf, &flags); err == nil {
			return flags, nil
		}
	}

	client := instance.APIManagerClient(inst)
	if client == nil {
		return flags, nil
	}
	query := url.Values{
		"sets":    {strings.Join(inst.FeatureSets, ",")},
		"context": {inst.ContextName},
	}.Encode()
	var data map[string]interface{}
	if err := client.Get(fmt.Sprintf("/api/v1/features?%s", query), &data); err != nil {
		return nil, err
	}
	var ok bool
	if flags, ok = data["flags"].(map[string]interface{}); !ok {
		return nil, errInvalidResponse
	}

	if buf, err := json.Marshal(flags); err == nil {
		cache.Set(cacheKey, buf, cacheDuration)
	}
	return flags, nil
}

func (f *Flags) addConfig(inst *instance.Instance) error {
	ctx, err := inst.SettingsContext()
	if err == instance.ErrContextNotFound {
		return nil
	} else if err != nil {
		return err
	}
	normalized := make(map[string]interface{})
	if m, ok := ctx["features"].(map[interface{}]interface{}); ok {
		for k, v := range m {
			normalized[fmt.Sprintf("%v", k)] = v
		}
	} else if items, ok := ctx["features"].([]interface{}); ok {
		for _, item := range items {
			if m, ok := item.(map[interface{}]interface{}); ok && len(m) == 1 {
				for k, v := range m {
					normalized[fmt.Sprintf("%v", k)] = v
				}
			} else {
				normalized[fmt.Sprintf("%v", item)] = true
			}
		}
	} else {
		return nil
	}
	ctxFlags := &Flags{
		DocID: consts.ConfigFlagsSettingsID,
		M:     normalized,
	}
	f.Sources = append(f.Sources, ctxFlags)
	for k, v := range ctxFlags.M {
		if _, ok := f.M[k]; !ok {
			f.M[k] = v
		}
	}
	return nil
}

func (f *Flags) addContext(inst *instance.Instance) error {
	id := fmt.Sprintf("%s.%s", consts.ContextFlagsSettingsID, inst.ContextName)
	var context Flags
	err := couchdb.GetDoc(couchdb.GlobalDB, consts.Settings, id, &context)
	if couchdb.IsNotFoundError(err) {
		return nil
	} else if err != nil {
		return err
	}
	context.SetID(consts.ContextFlagsSettingsID)
	f.Sources = append(f.Sources, &context)
	for k, v := range context.M {
		if _, ok := f.M[k]; !ok {
			if value := applyRatio(inst, k, v); value != nil {
				f.M[k] = value
			}
		}
	}
	return nil
}

const maxUint32 = 1<<32 - 1

func applyRatio(inst *instance.Instance, key string, data interface{}) interface{} {
	items, ok := data.([]interface{})
	if !ok || len(items) == 0 {
		return nil
	}
	sum := crc32.ChecksumIEEE([]byte(fmt.Sprintf("%s:%s", inst.DocID, key)))
	for i := range items {
		item, ok := items[i].(map[string]interface{})
		if !ok {
			continue
		}
		ratio, ok := item["ratio"].(float64)
		if !ok || ratio == 0.0 {
			continue
		}
		if ratio == 1.0 {
			return item["value"]
		}
		computed := uint32(ratio * maxUint32)
		if computed >= sum {
			return item["value"]
		}
		sum -= computed
	}
	return nil
}

func (f *Flags) addDefaults(inst *instance.Instance) error {
	var defaults Flags
	err := couchdb.GetDoc(couchdb.GlobalDB, consts.Settings, consts.DefaultFlagsSettingsID, &defaults)
	if couchdb.IsNotFoundError(err) {
		return nil
	} else if err != nil {
		return err
	}
	defaults.SetID(consts.DefaultFlagsSettingsID)
	f.Sources = append(f.Sources, &defaults)
	for k, v := range defaults.M {
		if _, ok := f.M[k]; !ok {
			f.M[k] = v
		}
	}
	return nil
}

var _ couchdb.Doc = &Flags{}
<|endoftext|>


<commit_before>// Copyright 2018 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"fmt"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol/keybase1"
	"golang.org/x/net/context"
)

// CmdSimpleFSSyncDisable is the 'fs sync disable' command.
type CmdSimpleFSSyncDisable struct {
	libkb.Contextified
	path keybase1.Path
}

// NewCmdSimpleFSSyncDisable creates a new cli.Command.
func NewCmdSimpleFSSyncDisable(
	cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:         "disable",
		ArgumentHelp: "[path-to-folder]",
		Usage:        "Stops syncing the given folder to local storage",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdSimpleFSSyncDisable{
				Contextified: libkb.NewContextified(g)}, "disable", c)
			cl.SetNoStandalone()
		},
	}
}

// Run runs the command in client/server mode.
func (c *CmdSimpleFSSyncDisable) Run() error {
	cli, err := GetSimpleFSClient(c.G())
	if err != nil {
		return err
	}

	ctx := context.TODO()
	arg := keybase1.SimpleFSSetFolderSyncConfigArg{
		Config: keybase1.FolderSyncConfig{
			Mode: keybase1.FolderSyncMode_DISABLED,
		},
		Path: c.path,
	}

	subpath := pathMinusTlf(c.path)
	if subpath != "" {
		arg.Path, err = toTlfPath(c.path)
		if err != nil {
			return err
		}
		res, err := cli.SimpleFSFolderSyncConfigAndStatus(ctx, arg.Path)
		if err != nil {
			return err
		}

		if res.Config.Mode == keybase1.FolderSyncMode_DISABLED {
			return fmt.Errorf("No syncing enabled on %s", arg.Path)
		} else if res.Config.Mode == keybase1.FolderSyncMode_ENABLED {
			return fmt.Errorf(
				"Cannot disable single path on fully-synced TLF %s", arg.Path)
		}

		found := false
		for _, p := range res.Config.Paths {
			if p == subpath {
				found = true
			} else {
				arg.Config.Paths = append(arg.Config.Paths, p)
			}
		}

		if !found {
			return nil
		}

		arg.Config.Mode = keybase1.FolderSyncMode_PARTIAL
	}

	return cli.SimpleFSSetFolderSyncConfig(ctx, arg)
}

// ParseArgv gets the required path.
func (c *CmdSimpleFSSyncDisable) ParseArgv(ctx *cli.Context) error {
	if len(ctx.Args()) != 1 {
		return fmt.Errorf("wrong number of arguments")
	}

	p, err := makeSimpleFSPath(ctx.Args()[0])
	if err != nil {
		return err
	}
	c.path = p
	return nil
}

// GetUsage says what this command needs to operate.
func (c *CmdSimpleFSSyncDisable) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config:    true,
		KbKeyring: true,
		API:       true,
	}
}
<commit_msg>client: warn on `fs sync disable` if parent is still synced<commit_after>// Copyright 2018 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"fmt"
	"strings"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol/keybase1"
	"golang.org/x/net/context"
)

// CmdSimpleFSSyncDisable is the 'fs sync disable' command.
type CmdSimpleFSSyncDisable struct {
	libkb.Contextified
	path keybase1.Path
}

// NewCmdSimpleFSSyncDisable creates a new cli.Command.
func NewCmdSimpleFSSyncDisable(
	cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:         "disable",
		ArgumentHelp: "[path-to-folder]",
		Usage:        "Stops syncing the given folder to local storage",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdSimpleFSSyncDisable{
				Contextified: libkb.NewContextified(g)}, "disable", c)
			cl.SetNoStandalone()
		},
	}
}

// Run runs the command in client/server mode.
func (c *CmdSimpleFSSyncDisable) Run() error {
	cli, err := GetSimpleFSClient(c.G())
	if err != nil {
		return err
	}

	ctx := context.TODO()
	arg := keybase1.SimpleFSSetFolderSyncConfigArg{
		Config: keybase1.FolderSyncConfig{
			Mode: keybase1.FolderSyncMode_DISABLED,
		},
		Path: c.path,
	}

	subpath := pathMinusTlf(c.path)
	if subpath != "" {
		arg.Path, err = toTlfPath(c.path)
		if err != nil {
			return err
		}
		res, err := cli.SimpleFSFolderSyncConfigAndStatus(ctx, arg.Path)
		if err != nil {
			return err
		}

		if res.Config.Mode == keybase1.FolderSyncMode_DISABLED {
			return fmt.Errorf("No syncing enabled on %s", arg.Path)
		} else if res.Config.Mode == keybase1.FolderSyncMode_ENABLED {
			return fmt.Errorf(
				"Cannot disable single path on fully-synced TLF %s", arg.Path)
		}

		found := false
		parentFound := ""
		for _, p := range res.Config.Paths {
			if p == subpath {
				found = true
			} else {
				toCheck := p
				if !strings.HasSuffix(p, "/") {
					toCheck = p + "/"
				}
				if strings.HasPrefix(subpath, toCheck) {
					parentFound = p
				}
				arg.Config.Paths = append(arg.Config.Paths, p)
			}
		}

		if parentFound != "" {
			ui := c.G().UI.GetTerminalUI()
			ui.Printf("%s will remain synced because its parent path (%s) "+
				"is still synced\n", subpath, parentFound)
		}

		if !found {
			return nil
		}

		arg.Config.Mode = keybase1.FolderSyncMode_PARTIAL
	}

	return cli.SimpleFSSetFolderSyncConfig(ctx, arg)
}

// ParseArgv gets the required path.
func (c *CmdSimpleFSSyncDisable) ParseArgv(ctx *cli.Context) error {
	if len(ctx.Args()) != 1 {
		return fmt.Errorf("wrong number of arguments")
	}

	p, err := makeSimpleFSPath(ctx.Args()[0])
	if err != nil {
		return err
	}
	c.path = p
	return nil
}

// GetUsage says what this command needs to operate.
func (c *CmdSimpleFSSyncDisable) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config:    true,
		KbKeyring: true,
		API:       true,
	}
}
<|endoftext|>


<commit_before>// snippet-comment:[These are tags for the AWS doc team's sample catalog. Do not remove.]
// snippet-sourceauthor:[AWS]
// snippet-sourcedescription:[sts_assume_role.go creates a set of temporary security credentials.]
// snippet-keyword:[Amazon Security Token Service]
// snippet-keyword:[Amazon STS]
// snippet-keyword:[AssumeRole function]
// snippet-keyword:[Go]
// snippet-service:[s3]
// snippet-keyword:[Code Sample]
// snippet-sourcetype:[full-example]
// snippet-sourcedate:[2019-01-30]
/*
   Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
   This file is licensed under the Apache License, Version 2.0 (the "License").
   You may not use this file except in compliance with the License. A copy of
   the License is located at
    http://aws.amazon.com/apache2.0/
   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
   CONDITIONS OF ANY KIND, either express or implied. See the License for the
   specific language governing permissions and limitations under the License.
*/
// snippet-start:[sts.go.assume_role]
package main

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
)

// Usage:
// go run sts_assume_role.go
func main() {
	// Initialize a session in us-west-2 that the SDK will use to load
	// credentials from the shared credentials file ~/.aws/credentials.
	sess, err := session.NewSession(&aws.Config{
		Region: aws.String("us-west-2"),
	})

	if err != nil {
		fmt.Println("NewSession Error", err)
		return
	}

	// Create a STS client
	svc := sts.New(sess)

	roleToAssumeArn := "arn:aws:iam::123456789012:role/roleName"
	sessionName := "test_session"
	result, err := svc.AssumeRole(&sts.AssumeRoleInput{
		RoleArn:         &roleToAssumeArn,
		RoleSessionName: &sessionName,
	})

	if err != nil {
		fmt.Println("AssumeRole Error", err)
		return
	}

	fmt.Println(result.AssumedRoleUser)
}
// snippet-end:[sts.go.assume_role]
<commit_msg>updating snippet-service<commit_after>// snippet-comment:[These are tags for the AWS doc team's sample catalog. Do not remove.]
// snippet-sourceauthor:[AWS]
// snippet-sourcedescription:[sts_assume_role.go creates a set of temporary security credentials.]
// snippet-keyword:[Amazon Security Token Service]
// snippet-keyword:[Amazon STS]
// snippet-keyword:[AssumeRole function]
// snippet-keyword:[Go]
// snippet-service:[sts]
// snippet-keyword:[Code Sample]
// snippet-sourcetype:[full-example]
// snippet-sourcedate:[2019-01-30]
/*
   Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
   This file is licensed under the Apache License, Version 2.0 (the "License").
   You may not use this file except in compliance with the License. A copy of
   the License is located at
    http://aws.amazon.com/apache2.0/
   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
   CONDITIONS OF ANY KIND, either express or implied. See the License for the
   specific language governing permissions and limitations under the License.
*/
// snippet-start:[sts.go.assume_role]
package main

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
)

// Usage:
// go run sts_assume_role.go
func main() {
	// Initialize a session in us-west-2 that the SDK will use to load
	// credentials from the shared credentials file ~/.aws/credentials.
	sess, err := session.NewSession(&aws.Config{
		Region: aws.String("us-west-2"),
	})

	if err != nil {
		fmt.Println("NewSession Error", err)
		return
	}

	// Create a STS client
	svc := sts.New(sess)

	roleToAssumeArn := "arn:aws:iam::123456789012:role/roleName"
	sessionName := "test_session"
	result, err := svc.AssumeRole(&sts.AssumeRoleInput{
		RoleArn:         &roleToAssumeArn,
		RoleSessionName: &sessionName,
	})

	if err != nil {
		fmt.Println("AssumeRole Error", err)
		return
	}

	fmt.Println(result.AssumedRoleUser)
}
// snippet-end:[sts.go.assume_role]
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"

	"chf/cmd/gcevpc/cp"
	"chf/kt"
	"version"

	"github.com/kentik/eggs/pkg/baseserver"
	"github.com/kentik/eggs/pkg/logger"
	ev "github.com/kentik/eggs/pkg/version"
)

var (
	FLAG_sourceSub     = flag.String("sub", "", "Google Sub to listen for flows on")
	FLAG_projectID     = flag.String("project", "", "Google ProjectID to listen for flows on")
	FLAG_dstAddr       = flag.String("dest", "", "Address to send flow to. If not set, defaults to https://flow.kentik.com")
	FLAG_email         = flag.String("api_email", "", "Kentik Email Address")
	FLAG_token         = flag.String("api_token", "", "Kentik Email Token")
	FLAG_plan          = flag.Int("plan_id", 0, "Kentik Plan ID to use for devices")
	FLAG_site          = flag.Int("site_id", 0, "Kentik Site ID to use for devices")
	FLAG_isDevice      = flag.Bool("is_device_primary", false, "Create one device in kentik per vm, vs on device per VPC.")
	FLAG_dropIntraDest = flag.Bool("drop_intra_dest", false, "Drop all intra-VPC Dest logs.")
	FLAG_dropIntraSrc  = flag.Bool("drop_intra_src", false, "Drop all intra-VPC Src logs")
	FLAG_writeStdout   = flag.Bool("stdout", false, "Write flows to stdout.")
)

func main() {
	eVeg := ev.VersionInfo{
		Version:  version.VERSION.Version,
		Date:     version.VERSION.Date,
		Platform: version.VERSION.Platform,
		Distro:   version.VERSION.Distro,
	}

	bs := baseserver.Boilerplate("gcevpc", eVeg, kt.DefaultGCEVPCProperties)
	lc := logger.NewContextLFromUnderlying(logger.SContext{S: "GCEVPC"}, bs.Logger)

	cpr, err := cp.NewCp(lc, *FLAG_sourceSub, *FLAG_projectID, *FLAG_dstAddr, *FLAG_email, *FLAG_token, *FLAG_plan, *FLAG_site, *FLAG_isDevice,
		*FLAG_dropIntraDest, *FLAG_dropIntraSrc, *FLAG_writeStdout)
	if err != nil {
		bs.Fail(fmt.Sprintf("Cannot start gcevpc: %v", err))
	}

	bs.Run(cpr)
}
<commit_msg>Fixing flag mistake<commit_after>package main

import (
	"flag"
	"fmt"

	"chf/cmd/gcevpc/cp"
	"chf/kt"
	"version"

	"github.com/kentik/eggs/pkg/baseserver"
	"github.com/kentik/eggs/pkg/logger"
	ev "github.com/kentik/eggs/pkg/version"
)

var (
	FLAG_sourceSub     = flag.String("sub", "", "Google Sub to listen for flows on")
	FLAG_projectID     = flag.String("project", "", "Google ProjectID to listen for flows on")
	FLAG_dstAddr       = flag.String("dest", "", "Address to send flow to. If not set, defaults to https://flow.kentik.com")
	FLAG_email         = flag.String("api_email", "", "Kentik Email Address")
	FLAG_token         = flag.String("api_token", "", "Kentik Email Token")
	FLAG_plan          = flag.Int("plan_id", 0, "Kentik Plan ID to use for devices")
	FLAG_site          = flag.Int("site_id", 0, "Kentik Site ID to use for devices")
	FLAG_isDevice      = flag.Bool("is_device_primary", false, "Create one device in kentik per vm, vs on device per VPC.")
	FLAG_dropIntraDest = flag.Bool("drop_intra_dest", false, "Drop all intra-VPC Dest logs.")
	FLAG_dropIntraSrc  = flag.Bool("drop_intra_src", false, "Drop all intra-VPC Src logs")
	FLAG_writeStdout   = flag.Bool("json", false, "Write flows to stdout in json form.")
)

func main() {
	eVeg := ev.VersionInfo{
		Version:  version.VERSION.Version,
		Date:     version.VERSION.Date,
		Platform: version.VERSION.Platform,
		Distro:   version.VERSION.Distro,
	}

	bs := baseserver.Boilerplate("gcevpc", eVeg, kt.DefaultGCEVPCProperties)
	lc := logger.NewContextLFromUnderlying(logger.SContext{S: "GCEVPC"}, bs.Logger)

	cpr, err := cp.NewCp(lc, *FLAG_sourceSub, *FLAG_projectID, *FLAG_dstAddr, *FLAG_email, *FLAG_token, *FLAG_plan, *FLAG_site, *FLAG_isDevice,
		*FLAG_dropIntraDest, *FLAG_dropIntraSrc, *FLAG_writeStdout)
	if err != nil {
		bs.Fail(fmt.Sprintf("Cannot start gcevpc: %v", err))
	}

	bs.Run(cpr)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"github.com/streadway/amqp"
	"io/ioutil"
	"koding/tools/config"
	"log"
	"net/http"
	"strings"
)

type Consumer struct {
	conn    *amqp.Connection
	channel *amqp.Channel
	tag     string
}

type Producer struct {
	conn    *amqp.Connection
	channel *amqp.Channel
}

var producer *Producer

func main() {
	log.Println("kontrol rabbitproxy started")

	var err error
	producer, err = createProducer()
	if err != nil {
		log.Println(err)
	}
	startRouting()
}

func startRouting() {
	c := &Consumer{
		conn:    nil,
		channel: nil,
		tag:     "",
	}

	var err error

	log.Printf("creating consumer connections")

	user := config.Current.Kontrold.RabbitMq.Login
	password := config.Current.Kontrold.RabbitMq.Password
	host := config.Current.Kontrold.RabbitMq.Host
	port := config.Current.Kontrold.RabbitMq.Port

	url := "amqp://" + user + ":" + password + "@" + host + ":" + port
	c.conn, err = amqp.Dial(url)
	if err != nil {
		log.Fatal(err)
	}

	c.channel, err = c.conn.Channel()
	if err != nil {
		log.Fatal(err)
	}

	err = c.channel.ExchangeDeclare("kontrol-rabbitproxy", "direct", false, true, false, false, nil)
	if err != nil {
		log.Fatal("exchange.declare: %s", err)
	}
	clientKey := readKey()
	if _, err := c.channel.QueueDeclare("", false, true, false, false, nil); err != nil {
		log.Fatal("queue.declare: %s", err)
	}

	log.Println("KEY is", clientKey)
	if err := c.channel.QueueBind("", clientKey, "kontrol-rabbitproxy", false, nil); err != nil {
		log.Fatal("queue.bind: %s", err)
	}

	httpStream, err := c.channel.Consume("", "", true, false, false, false, nil)
	if err != nil {
		log.Fatal("basic.consume: %s", err)
	}

	log.Println("routing started...")
	for msg := range httpStream {
		// log.Printf("got %dB message data: [%v]-[%s] %s",
		// 	len(msg.Body),
		// 	msg.DeliveryTag,
		// 	msg.RoutingKey,
		// 	msg.Body)

		body, err := doRequest(msg.Body)
		if err != nil {
			log.Println(err)
			go publishToRemote(nil, msg.CorrelationId, msg.ReplyTo)
		} else {
			go publishToRemote(body, msg.CorrelationId, msg.ReplyTo)
		}

	}
}

func doRequest(msg []byte) ([]byte, error) {
	buf := bytes.NewBuffer(msg)
	reader := bufio.NewReader(buf)
	req, err := http.ReadRequest(reader)
	if err != nil {
		log.Println(err)
	}

	// Request.RequestURI can't be set in client requests.
	// http://golang.org/src/pkg/net/http/client.go
	req.RequestURI = ""
	log.Println("Doing a http request to", req.URL.Host)
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	output := new(bytes.Buffer)
	resp.Write(output)

	// log.Println("Response is", string(data))

	return output.Bytes(), nil
}

func publishToRemote(data []byte, id, routingKey string) {
	msg := amqp.Publishing{
		ContentType:   "text/plain",
		Body:          data,
		CorrelationId: id,
	}

	log.Println("publishing repsonse to", routingKey)
	err := producer.channel.Publish("kontrol-rabbitproxy", routingKey, false, false, msg)
	if err != nil {
		log.Printf("error while publishing proxy message: %s", err)
	}

}

func createProducer() (*Producer, error) {
	p := &Producer{
		conn:    nil,
		channel: nil,
	}

	log.Printf("creating publisher connections")
	var err error

	user := config.Current.Kontrold.RabbitMq.Login
	password := config.Current.Kontrold.RabbitMq.Password
	host := config.Current.Kontrold.RabbitMq.Host
	port := config.Current.Kontrold.RabbitMq.Port

	url := "amqp://" + user + ":" + password + "@" + host + ":" + port
	p.conn, err = amqp.Dial(url)
	if err != nil {
		log.Fatal(err)
	}

	p.channel, err = p.conn.Channel()
	if err != nil {
		log.Fatal(err)
	}

	return p, nil
}

func readKey() string {
	file, err := ioutil.ReadFile("KEY")
	if err != nil {
		log.Println(err)
	}

	return strings.TrimSpace(string(file))
}
<commit_msg>Read key from ~/.kd/koding.key<commit_after>package main

import (
	"bufio"
	"bytes"
	"github.com/streadway/amqp"
	"io/ioutil"
	"koding/tools/config"
	"log"
	"net/http"
	"os/user"
	"strings"
)

type Consumer struct {
	conn    *amqp.Connection
	channel *amqp.Channel
	tag     string
}

type Producer struct {
	conn    *amqp.Connection
	channel *amqp.Channel
}

var producer *Producer

func main() {
	log.Println("kontrol rabbitproxy started")

	var err error
	producer, err = createProducer()
	if err != nil {
		log.Println(err)
	}
	startRouting()
}

func startRouting() {
	c := &Consumer{
		conn:    nil,
		channel: nil,
		tag:     "",
	}

	var err error

	log.Printf("creating consumer connections")

	user := config.Current.Kontrold.RabbitMq.Login
	password := config.Current.Kontrold.RabbitMq.Password
	host := config.Current.Kontrold.RabbitMq.Host
	port := config.Current.Kontrold.RabbitMq.Port

	url := "amqp://" + user + ":" + password + "@" + host + ":" + port
	c.conn, err = amqp.Dial(url)
	if err != nil {
		log.Fatal(err)
	}

	c.channel, err = c.conn.Channel()
	if err != nil {
		log.Fatal(err)
	}

	err = c.channel.ExchangeDeclare("kontrol-rabbitproxy", "direct", false, true, false, false, nil)
	if err != nil {
		log.Fatal("exchange.declare: %s", err)
	}
	clientKey := readKey()
	if _, err := c.channel.QueueDeclare("", false, true, false, false, nil); err != nil {
		log.Fatal("queue.declare: %s", err)
	}

	log.Println("KEY is", clientKey)
	if err := c.channel.QueueBind("", clientKey, "kontrol-rabbitproxy", false, nil); err != nil {
		log.Fatal("queue.bind: %s", err)
	}

	httpStream, err := c.channel.Consume("", "", true, false, false, false, nil)
	if err != nil {
		log.Fatal("basic.consume: %s", err)
	}

	log.Println("routing started...")
	for msg := range httpStream {
		// log.Printf("got %dB message data: [%v]-[%s] %s",
		// 	len(msg.Body),
		// 	msg.DeliveryTag,
		// 	msg.RoutingKey,
		// 	msg.Body)

		body, err := doRequest(msg.Body)
		if err != nil {
			log.Println(err)
			go publishToRemote(nil, msg.CorrelationId, msg.ReplyTo)
		} else {
			go publishToRemote(body, msg.CorrelationId, msg.ReplyTo)
		}

	}
}

func doRequest(msg []byte) ([]byte, error) {
	buf := bytes.NewBuffer(msg)
	reader := bufio.NewReader(buf)
	req, err := http.ReadRequest(reader)
	if err != nil {
		log.Println(err)
	}

	// Request.RequestURI can't be set in client requests.
	// http://golang.org/src/pkg/net/http/client.go
	req.RequestURI = ""
	log.Println("Doing a http request to", req.URL.Host)
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	output := new(bytes.Buffer)
	resp.Write(output)

	// log.Println("Response is", string(data))

	return output.Bytes(), nil
}

func publishToRemote(data []byte, id, routingKey string) {
	msg := amqp.Publishing{
		ContentType:   "text/plain",
		Body:          data,
		CorrelationId: id,
	}

	log.Println("publishing repsonse to", routingKey)
	err := producer.channel.Publish("kontrol-rabbitproxy", routingKey, false, false, msg)
	if err != nil {
		log.Printf("error while publishing proxy message: %s", err)
	}

}

func createProducer() (*Producer, error) {
	p := &Producer{
		conn:    nil,
		channel: nil,
	}

	log.Printf("creating publisher connections")
	var err error

	user := config.Current.Kontrold.RabbitMq.Login
	password := config.Current.Kontrold.RabbitMq.Password
	host := config.Current.Kontrold.RabbitMq.Host
	port := config.Current.Kontrold.RabbitMq.Port

	url := "amqp://" + user + ":" + password + "@" + host + ":" + port
	p.conn, err = amqp.Dial(url)
	if err != nil {
		log.Fatal(err)
	}

	p.channel, err = p.conn.Channel()
	if err != nil {
		log.Fatal(err)
	}

	return p, nil
}

func readKey() string {
	usr, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}

	keyfile := usr.HomeDir + "/.kd/koding.key"
	log.Println(keyfile)

	file, err := ioutil.ReadFile(keyfile)
	if err != nil {
		log.Println(err)
	}

	return strings.TrimSpace(string(file))
}
<|endoftext|>


<commit_before>package logger

import (
	"fmt"
	"io"
	"os"
	"sync"
	"time"
)

// A Logger represents an object that generates lines of output to to an
// io.Writer. By default it uses os.Stdout, but it can be changed  or others
// may be included during creation.
type Logger struct {
	mu      sync.Mutex    // protects the following fields
	disable bool          // global switch to disable log completely
	prefix  func() string // function return is written at beginning of each line
	out     io.Writer     // destination for ouput
}

// New creates a new Logger. The filepath sets the files that will be used
// as an extra output destination. By default logger also outputs to stdout.
func New(filepath ...string) *Logger {
	writers := make([]io.Writer, 0)
	for _, path := range filepath {
		logFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0640)
		if err != nil {
			fmt.Printf("logger: can't open %s: '%s'\n", path, err)
			continue
		}

		writers = append(writers, logFile)
	}

	writers = append(writers, os.Stdout)

	return &Logger{
		out: io.MultiWriter(writers...),
		prefix: func() string {
			return fmt.Sprintf("[%s] ", time.Now().Format(time.Stamp))
		},
	}
}

// Print formats using the default formats for its operands and writes to
// standard output. Spaces are added between operands when neither is a string. It
// returns the number of bytes written and any write error encountered.
func (l *Logger) Printn(v ...interface{}) (int, error) {
	if l.debugEnabled() {
		return 0, nil
	}

	return fmt.Fprint(l.output(), v...)
}

// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func (l *Logger) Printf(format string, v ...interface{}) (int, error) {
	if l.debugEnabled() {
		return 0, nil
	}

	return fmt.Fprintf(l.output(), format, v...)
}

// Println formats using the default formats for its operands and writes to
// standard output. Spaces are always added between operands and a newline is
// appended. It returns the number of bytes written and any write error
// encountered.
func (l *Logger) Println(v ...interface{}) (int, error) {
	if l.debugEnabled() {
		return 0, nil
	}

	return fmt.Fprintln(l.output(), v...)
}

// SetPrefix sets the output prefix according to the return value of the passed
// function.
func (l *Logger) SetPrefix(fn func() string) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.prefix = fn
}

// Prefix returns the output prefix.
func (l *Logger) Prefix() string {
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.prefix()
}

// SetOutput replaces the standard destination.
func (l *Logger) SetOutput(out io.Writer) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.out = out
}

// DisableLog is a global switch that disables the output completely. Useful
// if you want turn off/on logs for debugging.
func (l *Logger) DisableLog() {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.disable = true
}

func (l *Logger) output() io.Writer {
	l.out.Write([]byte(l.prefix()))
	return l.out
}

func (l *Logger) debugEnabled() bool {
	return l.disable
}
<commit_msg>Moved to master.<commit_after><|endoftext|>


<commit_before>// Copyright 2016, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"math/rand"
	"net/http"
	"sync"
	"testing"
	"time"

	"github.com/youtube/vitess/go/vt/discovery"
	"github.com/youtube/vitess/go/vt/logutil"
	"github.com/youtube/vitess/go/vt/servenv"
	"github.com/youtube/vitess/go/vt/tabletmanager/tmclient"
	"github.com/youtube/vitess/go/vt/tabletserver/grpcqueryservice"
	"github.com/youtube/vitess/go/vt/tabletserver/queryservice/fakes"
	"github.com/youtube/vitess/go/vt/throttler"
	"github.com/youtube/vitess/go/vt/vttest/fakesqldb"
	"github.com/youtube/vitess/go/vt/wrangler"
	"github.com/youtube/vitess/go/vt/wrangler/testlib"
	"github.com/youtube/vitess/go/vt/zktopo/zktestserver"

	querypb "github.com/youtube/vitess/go/vt/proto/query"
	topodatapb "github.com/youtube/vitess/go/vt/proto/topodata"

	log "github.com/golang/glog"
)

// This file contains a demo binary that demonstrates how the resharding
// throttler adapts its throttling rate to the replication lag.
//
// The throttler is necessary because replicas apply transactions at a slower
// rate than masters and fall behind at high write throughput.
// (Mostly they fall behind because MySQL replication is single threaded but
//  the write throughput on the master does not have to.)
//
// This demo simulates a client (writer), a master and a replica.
// The client writes to the master which in turn replicas everything to the
// replica.
// The replica measures its replication lag via the timestamp which is part of
// each message.
// While the master has no rate limit, the replica is limited to
// --rate (see below) transactions/second. The client runs the resharding
// throttler which tries to throttle the client based on the observed
// replication lag.

var (
	rate                     = flag.Int64("rate", 1000, "maximum rate of the throttled demo server at the start")
	duration                 = flag.Duration("duration", 600*time.Second, "total duration the demo runs")
	lagUpdateInterval        = flag.Duration("lag_update_interval", 5*time.Second, "interval at which the current replication lag will be broadcasted to the throttler")
	replicaDegrationInterval = flag.Duration("replica_degration_interval", 0*time.Second, "simulate a throughput degration of the replica every X interval (i.e. the replica applies transactions at a slower rate for -reparent_duration and the replication lag might go up)")
	replicaDegrationDuration = flag.Duration("replica_degration_duration", 10*time.Second, "duration a simulated degration should take")
)

// master simulates an *unthrottled* MySQL master which replicates every
// received "execute" call to a known "replica".
type master struct {
	replica *replica
}

// execute is the simulated RPC which is called by the client.
func (m *master) execute(msg time.Time) {
	m.replica.replicate(msg)
}

// replica simulates a *throttled* MySQL replica.
// If it cannot keep up with applying the master writes, it will report a
// replication lag > 0 seconds.
type replica struct {
	fakeTablet *testlib.FakeTablet
	qs         *fakes.StreamHealthQueryService

	// replicationStream is the incoming stream of messages from the master.
	replicationStream chan time.Time

	// throttler is used to enforce the maximum rate at which replica applies
	// transactions. It must not be confused with the client's throttler.
	throttler         *throttler.Throttler
	lastHealthUpdate  time.Time
	lagUpdateInterval time.Duration

	degrationInterval   time.Duration
	degrationDuration   time.Duration
	nextDegration       time.Time
	currentDegrationEnd time.Time

	stopChan chan struct{}
	wg       sync.WaitGroup
}

func newReplica(lagUpdateInterval, degrationInterval, degrationDuration time.Duration) *replica {
	t := &testing.T{}
	ts := zktestserver.New(t, []string{"cell1"})
	wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
	db := fakesqldb.Register()
	fakeTablet := testlib.NewFakeTablet(t, wr, "cell1", 0,
		topodatapb.TabletType_REPLICA, db, testlib.TabletKeyspaceShard(t, "ks", "-80"))
	fakeTablet.StartActionLoop(t, wr)

	target := querypb.Target{
		Keyspace:   "ks",
		Shard:      "-80",
		TabletType: topodatapb.TabletType_REPLICA,
	}
	qs := fakes.NewStreamHealthQueryService(target)
	grpcqueryservice.Register(fakeTablet.RPCServer, qs)

	throttler, err := throttler.NewThrottler("replica", "TPS", 1, *rate, throttler.ReplicationLagModuleDisabled)
	if err != nil {
		log.Fatal(err)
	}

	var nextDegration time.Time
	if degrationInterval != time.Duration(0) {
		nextDegration = time.Now().Add(degrationInterval)
	}
	r := &replica{
		fakeTablet:        fakeTablet,
		qs:                qs,
		throttler:         throttler,
		replicationStream: make(chan time.Time, 1*1024*1024),
		lagUpdateInterval: lagUpdateInterval,
		degrationInterval: degrationInterval,
		degrationDuration: degrationDuration,
		nextDegration:     nextDegration,
		stopChan:          make(chan struct{}),
	}
	r.wg.Add(1)
	go r.processReplicationStream()
	return r
}

func (r *replica) replicate(msg time.Time) {
	r.replicationStream <- msg
}

func (r *replica) processReplicationStream() {
	defer r.wg.Done()

	// actualRate counts the number of requests per r.lagUpdateInterval.
	actualRate := 0
	for msg := range r.replicationStream {
		select {
		case <-r.stopChan:
			return
		default:
		}

		now := time.Now()
		if now.Sub(r.lastHealthUpdate) > r.lagUpdateInterval {
			// Broadcast current lag every "lagUpdateInterval".
			//
			// Use integer values to calculate the lag. In consequence, the reported
			// lag will constantly vary between the floor and ceil value e.g.
			// an actual lag of 0.5s could be reported as 0s or 1s based on the
			// truncation of the two times.
			lagTruncated := uint32(now.Unix() - msg.Unix())
			// Display lag with a higher precision as well.
			lag := now.Sub(msg).Seconds()
			log.Infof("current lag: %1ds (%1.1fs) replica rate: % 7.1f chan len: % 6d", lagTruncated, lag, float64(actualRate)/r.lagUpdateInterval.Seconds(), len(r.replicationStream))
			r.qs.AddHealthResponseWithSecondsBehindMaster(lagTruncated)
			r.lastHealthUpdate = now
			actualRate = 0
		}
		if !r.nextDegration.IsZero() && time.Now().After(r.nextDegration) && r.currentDegrationEnd.IsZero() {
			degradedRate := rand.Int63n(*rate)
			log.Infof("degrading the replica for %.f seconds from %v TPS to %v", r.degrationDuration.Seconds(), *rate, degradedRate)
			r.throttler.SetMaxRate(degradedRate)
			r.currentDegrationEnd = time.Now().Add(r.degrationDuration)
		}
		if !r.currentDegrationEnd.IsZero() && time.Now().After(r.currentDegrationEnd) {
			log.Infof("degrading the replica stopped. Restoring TPS to: %v", *rate)
			r.throttler.SetMaxRate(*rate)
			r.currentDegrationEnd = time.Time{}
			r.nextDegration = time.Now().Add(r.degrationInterval)
		}

		for {
			backoff := r.throttler.Throttle(0 /* threadID */)
			if backoff == throttler.NotThrottled {
				break
			}
			time.Sleep(backoff)
		}
		actualRate++
	}
}

func (r *replica) stop() {
	close(r.replicationStream)
	close(r.stopChan)
	log.Info("Triggered replica shutdown. Waiting for it to stop.")
	r.wg.Wait()
	r.fakeTablet.StopActionLoop(&testing.T{})
}

// client simulates a client which should throttle itself based on the
// replication lag of all replicas.
type client struct {
	master *master

	healthCheck discovery.HealthCheck
	throttler   *throttler.Throttler

	stopChan chan struct{}
	wg       sync.WaitGroup
}

func newClient(master *master, replica *replica) *client {
	t, err := throttler.NewThrottler("client", "TPS", 1, throttler.MaxRateModuleDisabled, 5 /* seconds */)
	if err != nil {
		log.Fatal(err)
	}

	healthCheck := discovery.NewHealthCheck(1*time.Minute, 5*time.Second, 1*time.Minute)
	c := &client{
		master:      master,
		healthCheck: healthCheck,
		throttler:   t,
		stopChan:    make(chan struct{}),
	}
	c.healthCheck.SetListener(c, false /* sendDownEvents */)
	c.healthCheck.AddTablet(replica.fakeTablet.Tablet, "name")
	return c
}

func (c *client) run() {
	c.wg.Add(1)
	go c.loop()
}

func (c *client) loop() {
	defer c.wg.Done()

	for {
		select {
		case <-c.stopChan:
			return
		default:
		}

		for {
			backoff := c.throttler.Throttle(0 /* threadID */)
			if backoff == throttler.NotThrottled {
				break
			}
			time.Sleep(backoff)
		}

		c.master.execute(time.Now())
	}
}

func (c *client) stop() {
	close(c.stopChan)
	c.wg.Wait()

	c.healthCheck.Close()
	c.throttler.Close()
}

// StatsUpdate implements discovery.HealthCheckStatsListener.
// It gets called by the healthCheck instance every time a tablet broadcasts
// a health update.
func (c *client) StatsUpdate(ts *discovery.TabletStats) {
	c.throttler.RecordReplicationLag(time.Now(), ts)
}

func main() {
	flag.Parse()

	go servenv.RunDefault()
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, "/throttlerz", http.StatusTemporaryRedirect)
	})

	log.Infof("start rate set to: %v", *rate)
	replica := newReplica(*lagUpdateInterval, *replicaDegrationInterval, *replicaDegrationDuration)
	master := &master{replica: replica}
	client := newClient(master, replica)
	client.run()

	time.Sleep(*duration)
	client.stop()
	replica.stop()
}

func init() {
	servenv.RegisterDefaultFlags()
}
<commit_msg>throttler/demo: Fix panic "BUG: invalid TabletType forwarded: UNKNOWN".<commit_after>// Copyright 2016, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"math/rand"
	"net/http"
	"sync"
	"testing"
	"time"

	"github.com/youtube/vitess/go/vt/discovery"
	"github.com/youtube/vitess/go/vt/logutil"
	"github.com/youtube/vitess/go/vt/servenv"
	"github.com/youtube/vitess/go/vt/tabletmanager/tmclient"
	"github.com/youtube/vitess/go/vt/tabletserver/grpcqueryservice"
	"github.com/youtube/vitess/go/vt/tabletserver/queryservice/fakes"
	"github.com/youtube/vitess/go/vt/throttler"
	"github.com/youtube/vitess/go/vt/vttest/fakesqldb"
	"github.com/youtube/vitess/go/vt/wrangler"
	"github.com/youtube/vitess/go/vt/wrangler/testlib"
	"github.com/youtube/vitess/go/vt/zktopo/zktestserver"

	querypb "github.com/youtube/vitess/go/vt/proto/query"
	topodatapb "github.com/youtube/vitess/go/vt/proto/topodata"

	log "github.com/golang/glog"
)

// This file contains a demo binary that demonstrates how the resharding
// throttler adapts its throttling rate to the replication lag.
//
// The throttler is necessary because replicas apply transactions at a slower
// rate than masters and fall behind at high write throughput.
// (Mostly they fall behind because MySQL replication is single threaded but
//  the write throughput on the master does not have to.)
//
// This demo simulates a client (writer), a master and a replica.
// The client writes to the master which in turn replicas everything to the
// replica.
// The replica measures its replication lag via the timestamp which is part of
// each message.
// While the master has no rate limit, the replica is limited to
// --rate (see below) transactions/second. The client runs the resharding
// throttler which tries to throttle the client based on the observed
// replication lag.

var (
	rate                     = flag.Int64("rate", 1000, "maximum rate of the throttled demo server at the start")
	duration                 = flag.Duration("duration", 600*time.Second, "total duration the demo runs")
	lagUpdateInterval        = flag.Duration("lag_update_interval", 5*time.Second, "interval at which the current replication lag will be broadcasted to the throttler")
	replicaDegrationInterval = flag.Duration("replica_degration_interval", 0*time.Second, "simulate a throughput degration of the replica every X interval (i.e. the replica applies transactions at a slower rate for -reparent_duration and the replication lag might go up)")
	replicaDegrationDuration = flag.Duration("replica_degration_duration", 10*time.Second, "duration a simulated degration should take")
)

// master simulates an *unthrottled* MySQL master which replicates every
// received "execute" call to a known "replica".
type master struct {
	replica *replica
}

// execute is the simulated RPC which is called by the client.
func (m *master) execute(msg time.Time) {
	m.replica.replicate(msg)
}

// replica simulates a *throttled* MySQL replica.
// If it cannot keep up with applying the master writes, it will report a
// replication lag > 0 seconds.
type replica struct {
	fakeTablet *testlib.FakeTablet
	qs         *fakes.StreamHealthQueryService

	// replicationStream is the incoming stream of messages from the master.
	replicationStream chan time.Time

	// throttler is used to enforce the maximum rate at which replica applies
	// transactions. It must not be confused with the client's throttler.
	throttler         *throttler.Throttler
	lastHealthUpdate  time.Time
	lagUpdateInterval time.Duration

	degrationInterval   time.Duration
	degrationDuration   time.Duration
	nextDegration       time.Time
	currentDegrationEnd time.Time

	stopChan chan struct{}
	wg       sync.WaitGroup
}

func newReplica(lagUpdateInterval, degrationInterval, degrationDuration time.Duration) *replica {
	t := &testing.T{}
	ts := zktestserver.New(t, []string{"cell1"})
	wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
	db := fakesqldb.Register()
	fakeTablet := testlib.NewFakeTablet(t, wr, "cell1", 0,
		topodatapb.TabletType_REPLICA, db, testlib.TabletKeyspaceShard(t, "ks", "-80"))
	fakeTablet.StartActionLoop(t, wr)

	target := querypb.Target{
		Keyspace:   "ks",
		Shard:      "-80",
		TabletType: topodatapb.TabletType_REPLICA,
	}
	qs := fakes.NewStreamHealthQueryService(target)
	grpcqueryservice.Register(fakeTablet.RPCServer, qs)

	throttler, err := throttler.NewThrottler("replica", "TPS", 1, *rate, throttler.ReplicationLagModuleDisabled)
	if err != nil {
		log.Fatal(err)
	}

	var nextDegration time.Time
	if degrationInterval != time.Duration(0) {
		nextDegration = time.Now().Add(degrationInterval)
	}
	r := &replica{
		fakeTablet:        fakeTablet,
		qs:                qs,
		throttler:         throttler,
		replicationStream: make(chan time.Time, 1*1024*1024),
		lagUpdateInterval: lagUpdateInterval,
		degrationInterval: degrationInterval,
		degrationDuration: degrationDuration,
		nextDegration:     nextDegration,
		stopChan:          make(chan struct{}),
	}
	r.wg.Add(1)
	go r.processReplicationStream()
	return r
}

func (r *replica) replicate(msg time.Time) {
	r.replicationStream <- msg
}

func (r *replica) processReplicationStream() {
	defer r.wg.Done()

	// actualRate counts the number of requests per r.lagUpdateInterval.
	actualRate := 0
	for msg := range r.replicationStream {
		select {
		case <-r.stopChan:
			return
		default:
		}

		now := time.Now()
		if now.Sub(r.lastHealthUpdate) > r.lagUpdateInterval {
			// Broadcast current lag every "lagUpdateInterval".
			//
			// Use integer values to calculate the lag. In consequence, the reported
			// lag will constantly vary between the floor and ceil value e.g.
			// an actual lag of 0.5s could be reported as 0s or 1s based on the
			// truncation of the two times.
			lagTruncated := uint32(now.Unix() - msg.Unix())
			// Display lag with a higher precision as well.
			lag := now.Sub(msg).Seconds()
			log.Infof("current lag: %1ds (%1.1fs) replica rate: % 7.1f chan len: % 6d", lagTruncated, lag, float64(actualRate)/r.lagUpdateInterval.Seconds(), len(r.replicationStream))
			r.qs.AddHealthResponseWithSecondsBehindMaster(lagTruncated)
			r.lastHealthUpdate = now
			actualRate = 0
		}
		if !r.nextDegration.IsZero() && time.Now().After(r.nextDegration) && r.currentDegrationEnd.IsZero() {
			degradedRate := rand.Int63n(*rate)
			log.Infof("degrading the replica for %.f seconds from %v TPS to %v", r.degrationDuration.Seconds(), *rate, degradedRate)
			r.throttler.SetMaxRate(degradedRate)
			r.currentDegrationEnd = time.Now().Add(r.degrationDuration)
		}
		if !r.currentDegrationEnd.IsZero() && time.Now().After(r.currentDegrationEnd) {
			log.Infof("degrading the replica stopped. Restoring TPS to: %v", *rate)
			r.throttler.SetMaxRate(*rate)
			r.currentDegrationEnd = time.Time{}
			r.nextDegration = time.Now().Add(r.degrationInterval)
		}

		for {
			backoff := r.throttler.Throttle(0 /* threadID */)
			if backoff == throttler.NotThrottled {
				break
			}
			time.Sleep(backoff)
		}
		actualRate++
	}
}

func (r *replica) stop() {
	close(r.replicationStream)
	close(r.stopChan)
	log.Info("Triggered replica shutdown. Waiting for it to stop.")
	r.wg.Wait()
	r.fakeTablet.StopActionLoop(&testing.T{})
}

// client simulates a client which should throttle itself based on the
// replication lag of all replicas.
type client struct {
	master *master

	healthCheck discovery.HealthCheck
	throttler   *throttler.Throttler

	stopChan chan struct{}
	wg       sync.WaitGroup
}

func newClient(master *master, replica *replica) *client {
	t, err := throttler.NewThrottler("client", "TPS", 1, throttler.MaxRateModuleDisabled, 5 /* seconds */)
	if err != nil {
		log.Fatal(err)
	}

	healthCheck := discovery.NewHealthCheck(1*time.Minute, 5*time.Second, 1*time.Minute)
	c := &client{
		master:      master,
		healthCheck: healthCheck,
		throttler:   t,
		stopChan:    make(chan struct{}),
	}
	c.healthCheck.SetListener(c, false /* sendDownEvents */)
	c.healthCheck.AddTablet(replica.fakeTablet.Tablet, "name")
	return c
}

func (c *client) run() {
	c.wg.Add(1)
	go c.loop()
}

func (c *client) loop() {
	defer c.wg.Done()

	for {
		select {
		case <-c.stopChan:
			return
		default:
		}

		for {
			backoff := c.throttler.Throttle(0 /* threadID */)
			if backoff == throttler.NotThrottled {
				break
			}
			time.Sleep(backoff)
		}

		c.master.execute(time.Now())
	}
}

func (c *client) stop() {
	close(c.stopChan)
	c.wg.Wait()

	c.healthCheck.Close()
	c.throttler.Close()
}

// StatsUpdate implements discovery.HealthCheckStatsListener.
// It gets called by the healthCheck instance every time a tablet broadcasts
// a health update.
func (c *client) StatsUpdate(ts *discovery.TabletStats) {
	// Ignore unless REPLICA or RDONLY.
	if ts.Target.TabletType != topodatapb.TabletType_REPLICA && ts.Target.TabletType != topodatapb.TabletType_RDONLY {
		return
	}

	c.throttler.RecordReplicationLag(time.Now(), ts)
}

func main() {
	flag.Parse()

	go servenv.RunDefault()
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, "/throttlerz", http.StatusTemporaryRedirect)
	})

	log.Infof("start rate set to: %v", *rate)
	replica := newReplica(*lagUpdateInterval, *replicaDegrationInterval, *replicaDegrationDuration)
	master := &master{replica: replica}
	client := newClient(master, replica)
	client.run()

	time.Sleep(*duration)
	client.stop()
	replica.stop()
}

func init() {
	servenv.RegisterDefaultFlags()
}
<|endoftext|>


<commit_before>package models

type UserSetting struct {
	Key    string `gorm:"NOT NULL;size:100;PRIMARY_KEY"`
	UserID uint   `gorm:"NOT NULL;PRIMARY_KEY;AUTO_INCREMENT:false"`
	Value  string `gorm:"NOT NULL;size:200"`
}

type UserSettings []*UserSetting
<commit_msg>change user settings value column to be of type text<commit_after>package models

type UserSetting struct {
	Key    string `gorm:"NOT NULL;size:100;PRIMARY_KEY"`
	UserID uint   `gorm:"NOT NULL;PRIMARY_KEY;AUTO_INCREMENT:false"`
	Value  string `gorm:"NOT NULL;type:text"`
}

type UserSettings []*UserSetting
<|endoftext|>


<commit_before>package wiki

import (
	"fmt"
	"math/rand"
	"regexp"
	"strings"
	"time"

	"cgt.name/pkg/go-mwclient"
	"github.com/erbridge/gotwit"
)

type (
	Client struct {
		bot    *gotwit.Bot
		wiki   *mwclient.Client
		ticker *time.Ticker
	}
)

func NewClient(bot *gotwit.Bot) (client Client, err error) {
	// TODO: Add contact details to the user agent.
	wiki, err := mwclient.New("https://en.wikipedia.org/w/api.php", "Wikipaedian")

	if err != nil {
		return
	}

	client = Client{
		bot:  bot,
		wiki: wiki,
	}

	return
}

func (c *Client) Start(d time.Duration) {
	c.ticker = time.NewTicker(d)
	defer c.ticker.Stop()

	c.post()

	for range c.ticker.C {
		c.post()
	}
}

func (c *Client) post() {
	last := c.lastPost()

	fmt.Println("Last post:", last)

	var content string

	if last == "" {
		content = "[[Hello, World!]]"
	} else {
		page := c.page(last)

		content = c.createPost(page)
	}

	fmt.Println("Posting:", content)

	// c.bot.Post(content, false)
}

func (c *Client) lastPost() string {
	if text, err := c.bot.LastTweetText(); err != nil {
		panic(err)
	} else {
		return text
	}

	return ""
}

func (c *Client) page(last string) (content string) {
	re := regexp.MustCompile("\\[{2}([^:]+?)\\]{2}")

	matches := re.FindAllStringSubmatch(last, -1)

	// FIXME: Handle the 0 match state.
	title := matches[rand.Intn(len(matches))][1]

	re = regexp.MustCompile("(.+?)\\|")

	match := re.FindStringSubmatch(title)

	if len(match) > 0 {
		title = match[1]
	}

	content, _, err := c.wiki.GetPageByName(title)

	if err != nil {
		panic(err)
	}

	re = regexp.MustCompile("^#REDIRECT[[:space:]]+\\[{2}[^:]+?\\]{2}.*?")

	if re.MatchString(content) {
		content = re.FindString(content)

		fmt.Println("Redirecting:", content)

		return c.page(content)
	}

	return
}

func (c *Client) createPost(page string) (content string) {
	re := regexp.MustCompile("[.!?][[:space:]]+" +
		"(" +
		"(" +
		"([^{^}^\\[^\\]^<^>]+?[[:space:]]+?)+?" +
		")" +
		"(\\[{2}[^:]+?\\]{2})" +
		"(" +
		"([[:space:]]+?[^{^}^\\[^\\]^<^>]+?)+?" +
		")" +
		"([.!?])" +
		")" +
		"[[:space:]]+",
	)

	matches := re.FindAllStringSubmatch(page, -1)

	// FIXME: Handle the 0 match state.
	match := matches[rand.Intn(len(matches))]

	content = match[1]

	if len(content) <= 140 {
		return
	}

	content = match[4]

	if len(content) > 138 {
		re = regexp.MustCompile("\\[{2}[^:]+?\\]{2}")

		matches := re.FindAllString(content, -1)

		// FIXME: Handle the 0 match state.
		content = matches[rand.Intn(len(matches))]
	}

	before, after := strings.Split(match[2], " "), strings.Split(match[5], " ")

	newContent := content

	for len(newContent) < 138 {
		content = newContent

		word := ""

		if len(before) > 0 && rand.Float32() > 0.5 {
			word, before = before[len(before)-1], before[:len(before)-1]

			if word == "" {
				continue
			}

			newContent = word + " " + content
		} else if len(after) > 0 {
			word, after = after[0], after[1:]

			if word == "" {
				continue
			}

			newContent = content + " " + word
		} else {
			break
		}
	}

	if len(before) > 0 {
		content = "…" + content
	}

	if len(after) > 0 {
		content += "…"
	} else {
		content += match[7]
	}

	return
}
<commit_msg>Restart if there are no further links<commit_after>package wiki

import (
	"fmt"
	"math/rand"
	"regexp"
	"strings"
	"time"

	"cgt.name/pkg/go-mwclient"
	"github.com/erbridge/gotwit"
)

type (
	Client struct {
		bot    *gotwit.Bot
		wiki   *mwclient.Client
		ticker *time.Ticker
	}
)

func NewClient(bot *gotwit.Bot) (client Client, err error) {
	// TODO: Add contact details to the user agent.
	wiki, err := mwclient.New("https://en.wikipedia.org/w/api.php", "Wikipaedian")

	if err != nil {
		return
	}

	client = Client{
		bot:  bot,
		wiki: wiki,
	}

	return
}

func (c *Client) Start(d time.Duration) {
	c.ticker = time.NewTicker(d)
	defer c.ticker.Stop()

	c.post()

	for range c.ticker.C {
		c.post()
	}
}

func (c *Client) post() {
	last := c.lastPost()

	fmt.Println("Last post:", last)

	var content string

	if last == "" || last == "{{restart}}" {
		content = "[[Hello, World!]]"
	} else {
		page := c.page(last)

		content = c.createPost(page)
	}

	fmt.Println("Posting:", content)

	// c.bot.Post(content, false)
}

func (c *Client) lastPost() string {
	if text, err := c.bot.LastTweetText(); err != nil {
		panic(err)
	} else {
		return text
	}

	return ""
}

func (c *Client) page(last string) (content string) {
	re := regexp.MustCompile("\\[{2}([^:]+?)\\]{2}")

	matches := re.FindAllStringSubmatch(last, -1)

	title := matches[rand.Intn(len(matches))][1]

	re = regexp.MustCompile("(.+?)\\|")

	match := re.FindStringSubmatch(title)

	if len(match) > 0 {
		title = match[1]
	}

	content, _, err := c.wiki.GetPageByName(title)

	if err != nil {
		panic(err)
	}

	re = regexp.MustCompile("^#REDIRECT[[:space:]]+\\[{2}[^:]+?\\]{2}.*?")

	if re.MatchString(content) {
		content = re.FindString(content)

		fmt.Println("Redirecting:", content)

		return c.page(content)
	}

	return
}

func (c *Client) createPost(page string) (content string) {
	re := regexp.MustCompile("[.!?][[:space:]]+" +
		"(" +
		"(" +
		"([^{^}^\\[^\\]^<^>]+?[[:space:]]+?)+?" +
		")" +
		"(\\[{2}[^:]+?\\]{2})" +
		"(" +
		"([[:space:]]+?[^{^}^\\[^\\]^<^>]+?)+?" +
		")" +
		"([.!?])" +
		")" +
		"[[:space:]]+",
	)

	matches := re.FindAllStringSubmatch(page, -1)

	if len(matches) == 0 {
		return "{{restart}}"
	}

	match := matches[rand.Intn(len(matches))]

	content = match[1]

	if len(content) <= 140 {
		return
	}

	content = match[4]

	if len(content) > 138 {
		re = regexp.MustCompile("\\[{2}[^:]+?\\]{2}")

		matches := re.FindAllString(content, -1)

		content = matches[rand.Intn(len(matches))]
	}

	before, after := strings.Split(match[2], " "), strings.Split(match[5], " ")

	newContent := content

	for len(newContent) < 138 {
		content = newContent

		word := ""

		if len(before) > 0 && rand.Float32() > 0.5 {
			word, before = before[len(before)-1], before[:len(before)-1]

			if word == "" {
				continue
			}

			newContent = word + " " + content
		} else if len(after) > 0 {
			word, after = after[0], after[1:]

			if word == "" {
				continue
			}

			newContent = content + " " + word
		} else {
			break
		}
	}

	if len(before) > 0 {
		content = "…" + content
	}

	if len(after) > 0 {
		content += "…"
	} else {
		content += match[7]
	}

	return
}
<|endoftext|>