<commit_before>package main

import (
	"fmt"
	"io"
	"math/rand"
	"sync"
	"time"

	"github.com/vbauerster/mpb/v4"
	"github.com/vbauerster/mpb/v4/decor"
)

func main() {
	p := mpb.New(mpb.PopCompletedMode())

	total, numBars := 100, 2
	for i := 0; i < numBars; i++ {
		name := fmt.Sprintf("Bar#%d:", i)
		bar := p.AddBar(int64(total),
			mpb.BarNoPop(),
			mpb.PrependDecorators(
				decor.Name(name),
				decor.Percentage(decor.WCSyncSpace),
			),
			mpb.AppendDecorators(
				decor.OnComplete(
					decor.EwmaETA(decor.ET_STYLE_GO, 60), "done!",
				),
			),
		)
		// simulating some work
		go func() {
			rng := rand.New(rand.NewSource(time.Now().UnixNano()))
			max := 100 * time.Millisecond
			for i := 0; i < total; i++ {
				start := time.Now()
				time.Sleep(time.Duration(rng.Intn(10)+1) * max / 10)
				bar.Increment(time.Since(start))
			}
		}()
	}

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		rng := rand.New(rand.NewSource(time.Now().UnixNano()))
		max := 3000 * time.Millisecond
		for i := 0; i < 10; i++ {
			filler := makeLogBar(fmt.Sprintf("some log: %d", i))
			p.Add(0, filler, mpb.BarPriority(-1)).SetTotal(0, true)
			time.Sleep(time.Duration(rng.Intn(10)+1) * max / 10)
		}
	}()

	wg.Wait()
	p.Wait()
}

func makeLogBar(msg string) mpb.Filler {
	limit := "%%.%ds"
	return mpb.FillerFunc(func(w io.Writer, width int, st *decor.Statistics) {
		fmt.Fprintf(w, fmt.Sprintf(limit, width), msg)
	})
}
<commit_msg>set BarPriority isn't necessary<commit_after>package main

import (
	"fmt"
	"io"
	"math/rand"
	"sync"
	"time"

	"github.com/vbauerster/mpb/v4"
	"github.com/vbauerster/mpb/v4/decor"
)

func main() {
	p := mpb.New(mpb.PopCompletedMode())

	total, numBars := 100, 2
	for i := 0; i < numBars; i++ {
		name := fmt.Sprintf("Bar#%d:", i)
		bar := p.AddBar(int64(total),
			mpb.BarNoPop(),
			mpb.PrependDecorators(
				decor.Name(name),
				decor.Percentage(decor.WCSyncSpace),
			),
			mpb.AppendDecorators(
				decor.OnComplete(
					decor.EwmaETA(decor.ET_STYLE_GO, 60), "done!",
				),
			),
		)
		// simulating some work
		go func() {
			rng := rand.New(rand.NewSource(time.Now().UnixNano()))
			max := 100 * time.Millisecond
			for i := 0; i < total; i++ {
				start := time.Now()
				time.Sleep(time.Duration(rng.Intn(10)+1) * max / 10)
				bar.Increment(time.Since(start))
			}
		}()
	}

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		rng := rand.New(rand.NewSource(time.Now().UnixNano()))
		max := 3000 * time.Millisecond
		for i := 0; i < 10; i++ {
			filler := makeLogBar(fmt.Sprintf("some log: %d", i))
			p.Add(0, filler).SetTotal(0, true)
			time.Sleep(time.Duration(rng.Intn(10)+1) * max / 10)
		}
	}()

	wg.Wait()
	p.Wait()
}

func makeLogBar(msg string) mpb.Filler {
	limit := "%%.%ds"
	return mpb.FillerFunc(func(w io.Writer, width int, st *decor.Statistics) {
		fmt.Fprintf(w, fmt.Sprintf(limit, width), msg)
	})
}
<|endoftext|>


<commit_before>package flags

import (
	"path"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"
	"testing"
)

type TestComplete struct {
}

func (t *TestComplete) Complete(match string) []Completion {
	options := []string{
		"hello world",
		"hello universe",
		"hello multiverse",
	}

	ret := make([]Completion, 0, len(options))

	for _, o := range options {
		if strings.HasPrefix(o, match) {
			ret = append(ret, Completion{
				Item: o,
			})
		}
	}

	return ret
}

var completionTestOptions struct {
	Verbose bool `short:"v" long:"verbose"`
	Debug   bool `short:"d" long:"debug"`
	Version bool `long:"version"`

	AddCommand struct {
		Positional struct {
			Filename Filename
		} `positional-args:"yes"`
	} `command:"add"`

	AddMultiCommand struct {
		Positional struct {
			Filename []Filename
		} `positional-args:"yes"`
	} `command:"add-multi"`

	RemoveCommand struct {
		Other bool     `short:"o"`
		File  Filename `short:"f" long:"filename"`
	} `command:"rm"`

	RenameCommand struct {
		Completed TestComplete `short:"c" long:"completed"`
	} `command:"rename"`
}

func TestCompletion(t *testing.T) {
	_, sourcefile, _, _ := runtime.Caller(0)
	sourcedir := filepath.Join(filepath.SplitList(path.Dir(sourcefile))...)

	excompl := []string{filepath.Join(sourcedir, "completion.go"), filepath.Join(sourcedir, "completion_test.go")}

	tests := []struct {
		Args      []string
		Completed []string
	}{
		{
			// Short names
			[]string{"-"},
			[]string{"-d", "-v"},
		},

		{
			// Short names concatenated
			[]string{"-dv"},
			[]string{"-dv"},
		},

		{
			// Long names
			[]string{"--"},
			[]string{"--debug", "--verbose", "--version"},
		},

		{
			// Long names partial
			[]string{"--ver"},
			[]string{"--verbose", "--version"},
		},

		{
			// Commands
			[]string{""},
			[]string{"add", "add-multi", "rename", "rm"},
		},

		{
			// Commands partial
			[]string{"r"},
			[]string{"rename", "rm"},
		},

		{
			// Positional filename
			[]string{"add", filepath.Join(sourcedir, "completion")},
			excompl,
		},

		{
			// Multiple positional filename (1 arg)
			[]string{"add-multi", filepath.Join(sourcedir, "completion")},
			excompl,
		},
		{
			// Multiple positional filename (2 args)
			[]string{"add-multi", filepath.Join(sourcedir, "completion.go"), filepath.Join(sourcedir, "completion")},
			excompl,
		},
		{
			// Multiple positional filename (3 args)
			[]string{"add-multi", filepath.Join(sourcedir, "completion.go"), filepath.Join(sourcedir, "completion.go"), filepath.Join(sourcedir, "completion")},
			excompl,
		},

		{
			// Flag filename
			[]string{"rm", "-f", path.Join(sourcedir, "completion")},
			excompl,
		},

		{
			// Flag short concat last filename
			[]string{"rm", "-of", path.Join(sourcedir, "completion")},
			excompl,
		},

		{
			// Flag concat filename
			[]string{"rm", "-f" + path.Join(sourcedir, "completion")},
			[]string{"-f" + excompl[0], "-f" + excompl[1]},
		},

		{
			// Flag equal concat filename
			[]string{"rm", "-f=" + path.Join(sourcedir, "completion")},
			[]string{"-f=" + excompl[0], "-f=" + excompl[1]},
		},

		{
			// Flag concat long filename
			[]string{"rm", "--filename=" + path.Join(sourcedir, "completion")},
			[]string{"--filename=" + excompl[0], "--filename=" + excompl[1]},
		},

		{
			// Flag long filename
			[]string{"rm", "--filename", path.Join(sourcedir, "completion")},
			excompl,
		},

		{
			// Custom completed
			[]string{"rename", "-c", "hello un"},
			[]string{"hello universe"},
		},
	}

	p := NewParser(&completionTestOptions, Default)
	c := &completion{parser: p}

	for _, test := range tests {
		ret := c.complete(test.Args)
		items := make([]string, len(ret))

		for i, v := range ret {
			items[i] = v.Item
		}

		if !reflect.DeepEqual(items, test.Completed) {
			t.Errorf("Args: %#v\n  Expected: %#v\n  Got:     %#v", test.Args, test.Completed, items)
		}
	}
}
<commit_msg>test required options<commit_after>package flags

import (
	"path"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"
	"testing"
)

type TestComplete struct {
}

func (t *TestComplete) Complete(match string) []Completion {
	options := []string{
		"hello world",
		"hello universe",
		"hello multiverse",
	}

	ret := make([]Completion, 0, len(options))

	for _, o := range options {
		if strings.HasPrefix(o, match) {
			ret = append(ret, Completion{
				Item: o,
			})
		}
	}

	return ret
}

var completionTestOptions struct {
	Verbose  bool `short:"v" long:"verbose"`
	Debug    bool `short:"d" long:"debug"`
	Version  bool `long:"version"`
	Required bool `long:"required" required:"true"`

	AddCommand struct {
		Positional struct {
			Filename Filename
		} `positional-args:"yes"`
	} `command:"add"`

	AddMultiCommand struct {
		Positional struct {
			Filename []Filename
		} `positional-args:"yes"`
	} `command:"add-multi"`

	RemoveCommand struct {
		Other bool     `short:"o"`
		File  Filename `short:"f" long:"filename"`
	} `command:"rm"`

	RenameCommand struct {
		Completed TestComplete `short:"c" long:"completed"`
	} `command:"rename"`
}

func TestCompletion(t *testing.T) {
	_, sourcefile, _, _ := runtime.Caller(0)
	sourcedir := filepath.Join(filepath.SplitList(path.Dir(sourcefile))...)

	excompl := []string{filepath.Join(sourcedir, "completion.go"), filepath.Join(sourcedir, "completion_test.go")}

	tests := []struct {
		Args      []string
		Completed []string
	}{
		{
			// Short names
			[]string{"-"},
			[]string{"-d", "-v"},
		},

		{
			// Short names concatenated
			[]string{"-dv"},
			[]string{"-dv"},
		},

		{
			// Long names
			[]string{"--"},
			[]string{"--debug", "--required", "--verbose", "--version"},
		},

		{
			// Long names partial
			[]string{"--ver"},
			[]string{"--verbose", "--version"},
		},

		{
			// Commands
			[]string{""},
			[]string{"add", "add-multi", "rename", "rm"},
		},

		{
			// Commands partial
			[]string{"r"},
			[]string{"rename", "rm"},
		},

		{
			// Positional filename
			[]string{"add", filepath.Join(sourcedir, "completion")},
			excompl,
		},

		{
			// Multiple positional filename (1 arg)
			[]string{"add-multi", filepath.Join(sourcedir, "completion")},
			excompl,
		},
		{
			// Multiple positional filename (2 args)
			[]string{"add-multi", filepath.Join(sourcedir, "completion.go"), filepath.Join(sourcedir, "completion")},
			excompl,
		},
		{
			// Multiple positional filename (3 args)
			[]string{"add-multi", filepath.Join(sourcedir, "completion.go"), filepath.Join(sourcedir, "completion.go"), filepath.Join(sourcedir, "completion")},
			excompl,
		},

		{
			// Flag filename
			[]string{"rm", "-f", path.Join(sourcedir, "completion")},
			excompl,
		},

		{
			// Flag short concat last filename
			[]string{"rm", "-of", path.Join(sourcedir, "completion")},
			excompl,
		},

		{
			// Flag concat filename
			[]string{"rm", "-f" + path.Join(sourcedir, "completion")},
			[]string{"-f" + excompl[0], "-f" + excompl[1]},
		},

		{
			// Flag equal concat filename
			[]string{"rm", "-f=" + path.Join(sourcedir, "completion")},
			[]string{"-f=" + excompl[0], "-f=" + excompl[1]},
		},

		{
			// Flag concat long filename
			[]string{"rm", "--filename=" + path.Join(sourcedir, "completion")},
			[]string{"--filename=" + excompl[0], "--filename=" + excompl[1]},
		},

		{
			// Flag long filename
			[]string{"rm", "--filename", path.Join(sourcedir, "completion")},
			excompl,
		},

		{
			// Custom completed
			[]string{"rename", "-c", "hello un"},
			[]string{"hello universe"},
		},
	}

	p := NewParser(&completionTestOptions, Default)
	c := &completion{parser: p}

	for _, test := range tests {
		ret := c.complete(test.Args)
		items := make([]string, len(ret))

		for i, v := range ret {
			items[i] = v.Item
		}

		if !reflect.DeepEqual(items, test.Completed) {
			t.Errorf("Args: %#v\n  Expected: %#v\n  Got:     %#v", test.Args, test.Completed, items)
		}
	}
}
<|endoftext|>


<commit_before>// Take well-formed json from a sensu check result and create an elasticsearch document to be used to
// generate user specific dashboards or highly contextual alerts.
//
// LICENSE:
//   Copyright 2015 Yieldbot. <devops@yieldbot.com>
//   Released under the MIT License; see LICENSE
//   for details.

package main

import (
	"flag"
	"fmt"
	"github.com/olivere/elastic"
	"github.com/yieldbot/sensues/lib"
	"github.com/yieldbot/sensuplugin/sensuhandler"
	"github.com/yieldbot/sensuplugin/sensuutil"
	"time"
)

func main() {

	// set commandline flags
	esIndexPtr := flag.String("index", lib.StatusEsIndex, "the elasticsearch index to use")
	esHostPtr := flag.String("host", lib.DefaultEsHost, "the elasticsearch host")
	esPortPtr := flag.String("port", lib.DefaultEsPort, "the elasticsearch port")

	flag.Parse()
	esIndex := *esIndexPtr
	esType := lib.DefaultEsType
	esHost := *esHostPtr
	esPort := *esPortPtr

	sensuEvent := new(sensuhandler.SensuEvent)

	sensuEnv := sensuhandler.SetSensuEnv()
	sensuEvent = sensuEvent.AcquireSensuEvent()

	// Create a client
	client, err := elastic.NewClient(
		elastic.SetURL("http://" + esHost + ":" + esPort),
	)
	if err != nil {
		sensuutil.EHndlr(err)
	}

	// Check to see if the index exists and if not create it
	if client.IndexExists(esIndex) == nil { // need to test to make sure this does what I want
		_, err = client.CreateIndex(esIndex).Do()
		if err != nil {
			sensuutil.EHndlr(err)
		}
	}

	// Create an Elasticsearch document. The document type will define the mapping used for the document.
	doc := make(map[string]interface{})
	var docID string
	docID = sensuhandler.EventName(sensuEvent.Client.Name, sensuEvent.Check.Name)
	doc["monitored_instance"] = sensuEvent.AcquireMonitoredInstance()
	doc["sensu_client"] = sensuEvent.Client.Name
	doc["incident_timestamp"] = time.Unix(sensuEvent.Check.Issued, 0).Format(time.RFC3339)
	doc["check_name"] = sensuhandler.CreateCheckName(sensuEvent.Check.Name)
	doc["check_state"] = sensuhandler.DefineStatus(sensuEvent.Check.Status)
	doc["sensuEnv"] = sensuhandler.DefineSensuEnv(sensuEnv.Sensu.Environment)
	doc["tags"] = sensuEvent.Check.Tags
	doc["instance_address"] = sensuEvent.Client.Address
	doc["check_state_duration"] = sensuhandler.DefineCheckStateDuration()

	// Add a document to the Elasticsearch index
	_, err = client.Index().
		Index(esIndex).
		Type(esType).
		Id(docID).
		BodyJson(doc).
		Do()
	if err != nil {
		sensuutil.EHndlr(err)
	}

	// Log a successful document push to stdout. I don't add the id here as some id's are fixed but
	// the user has the ability to autogenerate an id if they don't want to provide one.
	fmt.Printf("Record added to ES\n")
}
<commit_msg>add debugging statement to print out host info<commit_after>// Take well-formed json from a sensu check result and create an elasticsearch document to be used to
// generate user specific dashboards or highly contextual alerts.
//
// LICENSE:
//   Copyright 2015 Yieldbot. <devops@yieldbot.com>
//   Released under the MIT License; see LICENSE
//   for details.

package main

import (
	"flag"
	"fmt"
	"github.com/olivere/elastic"
	"github.com/yieldbot/sensues/lib"
	"github.com/yieldbot/sensuplugin/sensuhandler"
	"github.com/yieldbot/sensuplugin/sensuutil"
	"time"
)

func main() {

	// set commandline flags
	esIndexPtr := flag.String("index", lib.StatusEsIndex, "the elasticsearch index to use")
	esHostPtr := flag.String("host", lib.DefaultEsHost, "the elasticsearch host")
	esPortPtr := flag.String("port", lib.DefaultEsPort, "the elasticsearch port")

	flag.Parse()
	esIndex := *esIndexPtr
	esType := lib.DefaultEsType
	esHost := *esHostPtr
	esPort := *esPortPtr

	sensuEvent := new(sensuhandler.SensuEvent)

	sensuEnv := sensuhandler.SetSensuEnv()
	sensuEvent = sensuEvent.AcquireSensuEvent()

	// Create a client
	client, err := elastic.NewClient(
		elastic.SetURL("http://" + esHost + ":" + esPort),
	)
	fmt.Printf("http://" = esHost + ":" + esPort)
	if err != nil {
		sensuutil.EHndlr(err)
	}

	// Check to see if the index exists and if not create it
	if client.IndexExists(esIndex) == nil { // need to test to make sure this does what I want
		_, err = client.CreateIndex(esIndex).Do()
		if err != nil {
			sensuutil.EHndlr(err)
		}
	}

	// Create an Elasticsearch document. The document type will define the mapping used for the document.
	doc := make(map[string]interface{})
	var docID string
	docID = sensuhandler.EventName(sensuEvent.Client.Name, sensuEvent.Check.Name)
	doc["monitored_instance"] = sensuEvent.AcquireMonitoredInstance()
	doc["sensu_client"] = sensuEvent.Client.Name
	doc["incident_timestamp"] = time.Unix(sensuEvent.Check.Issued, 0).Format(time.RFC3339)
	doc["check_name"] = sensuhandler.CreateCheckName(sensuEvent.Check.Name)
	doc["check_state"] = sensuhandler.DefineStatus(sensuEvent.Check.Status)
	doc["sensuEnv"] = sensuhandler.DefineSensuEnv(sensuEnv.Sensu.Environment)
	doc["tags"] = sensuEvent.Check.Tags
	doc["instance_address"] = sensuEvent.Client.Address
	doc["check_state_duration"] = sensuhandler.DefineCheckStateDuration()

	// Add a document to the Elasticsearch index
	_, err = client.Index().
		Index(esIndex).
		Type(esType).
		Id(docID).
		BodyJson(doc).
		Do()
	if err != nil {
		sensuutil.EHndlr(err)
	}

	// Log a successful document push to stdout. I don't add the id here as some id's are fixed but
	// the user has the ability to autogenerate an id if they don't want to provide one.
	fmt.Printf("Record added to ES\n")
}
<|endoftext|>


<commit_before>// Copyright © 2016 -2017 A-Frame authors.
// Use of this source code is governed by a MIT License
// that can be found in the LICENSE file.

package commands

import (
	"os"
	"os/signal"
	"time"

	"github.com/howi-ce/howi/addon/application/plugin/cli"
	"github.com/howi-ce/howi/addon/application/plugin/cli/flags"
	"github.com/mkungla/slackoverflow/cmd/slackoverflow/internal"
	"github.com/robfig/cron"
)

var fromDate time.Time

// StackExchange ...
func StackExchange(so *internal.SlackOverflow) cli.Command {
	cmd := cli.NewCommand("stackexchange")
	cmd.SetShortDesc("Stack Exchange related commands see slackoverflow stackexchange --help for more info.")
	cmd.AddSubcommand(StackExchangeQuestions(so))
	cmd.AddSubcommand(StackExchangeWatch(so))

	return cmd
}

// StackExchangeQuestions ...
func StackExchangeQuestions(so *internal.SlackOverflow) cli.Command {
	scmd := cli.NewCommand("questions")
	scmd.SetShortDesc("Work with stackexchange questions based on the config.")

	gnqFlag := flags.NewBoolFlag("get")
	gnqFlag.SetUsage("Get new questions from configured Stack Exchange Site")
	scmd.AddFlag(gnqFlag)

	uqFlag := flags.NewBoolFlag("update")
	uqFlag.SetUsage("Update information about existing questions")
	scmd.AddFlag(uqFlag)

	sFlag := flags.NewBoolFlag("sync")
	sFlag.SetUsage("Get new questions and update information about existing questions")
	scmd.AddFlag(sFlag)

	scmd.Do(func(w *cli.Worker) {
		if err := so.Session(w); err != nil {
			w.Fail(err.Error())
			return
		}
		get, err := w.Flag("get")
		if err != nil {
			w.Fail(err.Error())
			return
		}

		update, err := w.Flag("update")
		if err != nil {
			w.Fail(err.Error())
			return
		}
		sync, err := w.Flag("sync")
		if err != nil {
			w.Fail(err.Error())
			return
		}

		if sync.Present() {
			getNewQuestions(w, so)
			updateQuestions(w, so)
		} else if get.Present() {
			getNewQuestions(w, so)
		} else if update.Present() {
			updateQuestions(w, so)
		} else {
			w.Fail("atleast one flag must be provided")
			return
		}
	})
	scmd.AfterAlways(func(w *cli.Worker) {
		if err := so.DB.Close(); err != nil {
			w.Log.Error(err)
		}
		w.Log.Infof(
			"Stack Exchange Quota usage (%d/%d)",
			so.StackExchange.GetQuotaRemaining(),
			so.StackExchange.GetQuotaMax(),
		)
	})
	return scmd
}

// StackExchangeWatch ...
func StackExchangeWatch(so *internal.SlackOverflow) cli.Command {
	scmd := cli.NewCommand("watch")
	scmd.SetShortDesc("Watch new questions from Stack Exchange site (updated every minute nothing stored to db or posted to slack)")
	scmd.Do(func(w *cli.Worker) {
		if err := so.Session(w); err != nil {
			w.Fail(err.Error())
			return
		}
		w.Log.Ok("Waiting for new questions!")
		fromDate = time.Now().UTC().Add(-30 * time.Minute)
		startWatching(w, so)
		watch := cron.New()
		watch.AddFunc("@every 1m", func() {
			startWatching(w, so)
		})
		go watch.Start()
		sig := make(chan os.Signal)
		signal.Notify(sig, os.Interrupt, os.Kill)
		<-sig
	})
	scmd.AfterAlways(func(w *cli.Worker) {
		if err := so.DB.Close(); err != nil {
			w.Log.Error(err)
		}
		w.Log.Infof(
			"Stack Exchange Quota usage (%d/%d)",
			so.StackExchange.GetQuotaRemaining(),
			so.StackExchange.GetQuotaMax(),
		)
	})
	return scmd
}

func getNewQuestions(w *cli.Worker, so *internal.SlackOverflow) {
	w.Log.Info("Stack Exchange: Checking for new questions.")

	var empty bool
	latest, _ := so.DB.LatestStackExchangeQuestion()
	if latest.QID == 0 {
		empty = true
	}

	if empty {
		w.Log.Info("There are no questions in database,")
		w.Log.Info("That is ok if current execution is first time you run slackoverflow")
		w.Log.Infof("Or there has been no questions tagged with %q on site %q",
			so.Config.StackExchange.SearchAdvanced["tagged"],
			so.Config.StackExchange.Site,
		)
		latest.CreationDate = time.Now().UTC().Add(-4 * time.Hour)
	}

	now := time.Now().UTC().Unix()
	diff := latest.CreationDate.Unix() - now
	// Allow maximum 4 h old question as from date otherwise we may exhaust
	// rate limit if last tracked questions fromdata is to far past
	// and slackoverflow has not been running for a while.
	if diff > (4 * 60) {
		latest.CreationDate = time.Now().UTC().Add(-4 * time.Hour)
	}

	w.Log.Infof("Checking new questions since %s", latest.CreationDate.String())

	// Check for New Questions from Stack Exchange
	searchAdvanced := so.StackExchange.SearchAdvanced()

	// Set it here so that it is allowed to override by config
	searchAdvanced.Parameters.Set("site", so.Config.StackExchange.Site)

	// Set all parameters from config
	for param, value := range so.Config.StackExchange.SearchAdvanced {
		searchAdvanced.Parameters.Set(param, value)
	}

	searchAdvanced.Parameters.Set("fromdate", latest.CreationDate.Unix()+1)

	// Output query as table
	d, _ := w.Flag("debug")
	debbuging := d.Present()
	if debbuging {
		searchAdvanced.DrawQuery(w)
	}

	fetchQuestions := true
	var lastQuestion internal.QuestionObj
	for fetchQuestions {
		w.Log.Debugf("Fetching page %d", searchAdvanced.GetCurrentPageNr())
		if results, err := searchAdvanced.Get(); results {
			// Questions received
			for _, q := range searchAdvanced.Result.Items {

				w.Log.Infof("Question: %s", q.Title)
				w.Log.Infof("Url:      %s", q.ShareLink)

				if debbuging {
					newq := internal.NewTable("Question ID", "Time", "Answers", "Comments", "Score", "Views", "Username")
					newq.AddRow(
						q.QID,
						time.Unix(q.CreationDate, 0).UTC().Format("15:04:05 Mon Jan _2 2006"),
						q.AnswerCount,
						q.CommentCount,
						q.Score,
						q.ViewCount,
						q.Owner.DisplayName,
					)
					newq.Print()
				}
				// Skip sync if there are locally no questions
				if empty {
					lastQuestion = q
					continue
				}
				so.SyncQuestion(w, q)
			}
			if err != nil {
				fetchQuestions = false
				w.Log.Error(err)
			}
		}

		// Done go to next page
		if searchAdvanced.GetCurrentPageNr() > 10 {
			fetchQuestions = false
		} else if searchAdvanced.HasMore() {
			searchAdvanced.NextPage()
		} else {
			fetchQuestions = false
			w.Log.Debug("There are no more new questions.")
		}
	}
	if empty && lastQuestion.QID > 0 {
		so.SyncQuestion(w, lastQuestion)
	}
}

func updateQuestions(w *cli.Worker, so *internal.SlackOverflow) {
	w.Log.Info("Stack Exchange: Updating existing questions.")

	questionIds, questionIdsCount := so.DB.StackExchangeQuestionTrackedIds(
		so.Config.StackExchange.QuestionsToWatch)

	// Check do we already have some questions do obtain from_date for next request
	if questionIdsCount == 0 {
		w.Log.Info("There are no questions in database,")
		w.Log.Info("That is ok if current execution is first time you run slackoverflow")
		w.Log.Info("and this case run 'slackoverflow stackechange guestions' --get first")
	}

	w.Log.Infof("Checking updates for %d questions. Max to be tracked (%d)",
		questionIdsCount, so.Config.StackExchange.QuestionsToWatch)

	// Check for New Questions from Stack Exchange
	updateQuestions := so.StackExchange.Questions()

	// Set it here so that it is allowed to override by config
	updateQuestions.Parameters.Set("site", so.Config.StackExchange.Site)

	for param, value := range so.Config.StackExchange.Questions {
		updateQuestions.Parameters.Set(param, value)
	}

	// Output query as table
	d, _ := w.Flag("debug")
	debbuging := d.Present()
	if debbuging {
		updateQuestions.DrawQuery(w, questionIds)
	}

	fetchQuestions := true
	for fetchQuestions {
		w.Log.Debugf("Fetching page %d", updateQuestions.GetCurrentPageNr())
		if results, err := updateQuestions.Get(questionIds); results {
			// Questions received
			for _, q := range updateQuestions.Result.Items {

				w.Log.Infof("Question: %s", q.Title)
				w.Log.Infof("Url:      %s", q.ShareLink)
				if debbuging {
					newq := internal.NewTable("Question ID", "Time", "Answers", "Comments", "Score", "Views", "Username")
					newq.AddRow(
						q.QID,
						time.Unix(q.CreationDate, 0).UTC().Format("15:04:05 Mon Jan _2 2006"),
						q.AnswerCount,
						q.CommentCount,
						q.Score,
						q.ViewCount,
						q.Owner.DisplayName,
					)
					newq.Print()
				}

				so.SyncQuestion(w, q)
			}
			if err != nil {
				fetchQuestions = false
				w.Log.Error(err.Error())
			}
		}

		// Done go to next page
		if updateQuestions.HasMore() {
			updateQuestions.NextPage()
		} else {
			fetchQuestions = false
			w.Log.Debug("There are no more questions to update.")
		}
	}
}

func startWatching(w *cli.Worker, so *internal.SlackOverflow) {
	// Check for New Questions from Stack Exchange
	searchAdvanced := so.StackExchange.SearchAdvanced()

	// Set it here so that it is allowed to override by config
	searchAdvanced.Parameters.Set("site", so.Config.StackExchange.Site)

	// Set all parameters from config
	for param, value := range so.Config.StackExchange.SearchAdvanced {
		searchAdvanced.Parameters.Set(param, value)
	}
	searchAdvanced.Parameters.Set("tagged", "php")
	searchAdvanced.Parameters.Set("fromdate", fromDate.Unix()+1)

	fetchQuestions := true
	for fetchQuestions {
		w.Log.Debugf("Fetching page %d", searchAdvanced.GetCurrentPageNr())
		if results, err := searchAdvanced.Get(); results {
			// Questions received
			for _, q := range searchAdvanced.Result.Items {
				fromDate = time.Unix(q.CreationDate, 0).UTC()
				w.Log.Linef("Question: %s", q.Title)
				w.Log.Linef("Url:      %s", q.ShareLink)
				newq := internal.NewTable("Question ID", "Time", "Answers", "Comments", "Score", "Views", "Username")
				newq.AddRow(
					q.QID,
					time.Unix(q.CreationDate, 0).Local().Format("15:04:05 Mon Jan _2 2006"),
					q.AnswerCount,
					q.CommentCount,
					q.Score,
					q.ViewCount,
					q.Owner.DisplayName,
				)
				newq.Print()
			}
			if err != nil {
				fetchQuestions = false
				w.Log.Error(err.Error())
			}

			if len(searchAdvanced.Result.Items) > 0 {
				w.Log.Ok("Waiting for new questions!")
			}
		}

		// Done go to next page
		if searchAdvanced.HasMore() || searchAdvanced.GetCurrentPageNr() > 10 {
			searchAdvanced.NextPage()
		} else {
			fetchQuestions = false
		}
	}

	w.Log.Infof(
		"Stack Exchange Quota usage (%d/%d)",
		so.StackExchange.GetQuotaRemaining(),
		so.StackExchange.GetQuotaMax(),
	)
}
<commit_msg>Move duplicated code to printQuestion<commit_after>// Copyright © 2016 -2017 A-Frame authors.
// Use of this source code is governed by a MIT License
// that can be found in the LICENSE file.

package commands

import (
	"os"
	"os/signal"
	"time"

	"github.com/howi-ce/howi/addon/application/plugin/cli"
	"github.com/howi-ce/howi/addon/application/plugin/cli/flags"
	"github.com/mkungla/slackoverflow/cmd/slackoverflow/internal"
	"github.com/robfig/cron"
)

var fromDate time.Time

// StackExchange ...
func StackExchange(so *internal.SlackOverflow) cli.Command {
	cmd := cli.NewCommand("stackexchange")
	cmd.SetShortDesc("Stack Exchange related commands see slackoverflow stackexchange --help for more info.")
	cmd.AddSubcommand(StackExchangeQuestions(so))
	cmd.AddSubcommand(StackExchangeWatch(so))

	return cmd
}

// StackExchangeQuestions ...
func StackExchangeQuestions(so *internal.SlackOverflow) cli.Command {
	scmd := cli.NewCommand("questions")
	scmd.SetShortDesc("Work with stackexchange questions based on the config.")

	gnqFlag := flags.NewBoolFlag("get")
	gnqFlag.SetUsage("Get new questions from configured Stack Exchange Site")
	scmd.AddFlag(gnqFlag)

	uqFlag := flags.NewBoolFlag("update")
	uqFlag.SetUsage("Update information about existing questions")
	scmd.AddFlag(uqFlag)

	sFlag := flags.NewBoolFlag("sync")
	sFlag.SetUsage("Get new questions and update information about existing questions")
	scmd.AddFlag(sFlag)

	scmd.Do(func(w *cli.Worker) {
		if err := so.Session(w); err != nil {
			w.Fail(err.Error())
			return
		}
		get, err := w.Flag("get")
		if err != nil {
			w.Fail(err.Error())
			return
		}

		update, err := w.Flag("update")
		if err != nil {
			w.Fail(err.Error())
			return
		}
		sync, err := w.Flag("sync")
		if err != nil {
			w.Fail(err.Error())
			return
		}

		if sync.Present() {
			getNewQuestions(w, so)
			updateQuestions(w, so)
		} else if get.Present() {
			getNewQuestions(w, so)
		} else if update.Present() {
			updateQuestions(w, so)
		} else {
			w.Fail("atleast one flag must be provided")
			return
		}
	})
	scmd.AfterAlways(func(w *cli.Worker) {
		if err := so.DB.Close(); err != nil {
			w.Log.Error(err)
		}
		w.Log.Infof(
			"Stack Exchange Quota usage (%d/%d)",
			so.StackExchange.GetQuotaRemaining(),
			so.StackExchange.GetQuotaMax(),
		)
	})
	return scmd
}

// StackExchangeWatch ...
func StackExchangeWatch(so *internal.SlackOverflow) cli.Command {
	scmd := cli.NewCommand("watch")
	scmd.SetShortDesc("Watch new questions from Stack Exchange site (updated every minute nothing stored to db or posted to slack)")
	scmd.Do(func(w *cli.Worker) {
		if err := so.Session(w); err != nil {
			w.Fail(err.Error())
			return
		}
		w.Log.Ok("Waiting for new questions!")
		fromDate = time.Now().UTC().Add(-30 * time.Minute)
		startWatching(w, so)
		watch := cron.New()
		watch.AddFunc("@every 1m", func() {
			startWatching(w, so)
		})
		go watch.Start()
		sig := make(chan os.Signal)
		signal.Notify(sig, os.Interrupt, os.Kill)
		<-sig
	})
	scmd.AfterAlways(func(w *cli.Worker) {
		if err := so.DB.Close(); err != nil {
			w.Log.Error(err)
		}
		w.Log.Infof(
			"Stack Exchange Quota usage (%d/%d)",
			so.StackExchange.GetQuotaRemaining(),
			so.StackExchange.GetQuotaMax(),
		)
	})
	return scmd
}

func getNewQuestions(w *cli.Worker, so *internal.SlackOverflow) {
	w.Log.Info("Stack Exchange: Checking for new questions.")

	var empty bool
	latest, _ := so.DB.LatestStackExchangeQuestion()
	if latest.QID == 0 {
		empty = true
	}

	if empty {
		w.Log.Info("There are no questions in database,")
		w.Log.Info("That is ok if current execution is first time you run slackoverflow")
		w.Log.Infof("Or there has been no questions tagged with %q on site %q",
			so.Config.StackExchange.SearchAdvanced["tagged"],
			so.Config.StackExchange.Site,
		)
		latest.CreationDate = time.Now().UTC().Add(-4 * time.Hour)
	}

	now := time.Now().UTC().Unix()
	diff := latest.CreationDate.Unix() - now
	// Allow maximum 4 h old question as from date otherwise we may exhaust
	// rate limit if last tracked questions fromdata is to far past
	// and slackoverflow has not been running for a while.
	if diff > (4 * 60) {
		latest.CreationDate = time.Now().UTC().Add(-4 * time.Hour)
	}

	w.Log.Infof("Checking new questions since %s", latest.CreationDate.String())

	// Check for New Questions from Stack Exchange
	searchAdvanced := so.StackExchange.SearchAdvanced()

	// Set it here so that it is allowed to override by config
	searchAdvanced.Parameters.Set("site", so.Config.StackExchange.Site)

	// Set all parameters from config
	for param, value := range so.Config.StackExchange.SearchAdvanced {
		searchAdvanced.Parameters.Set(param, value)
	}

	searchAdvanced.Parameters.Set("fromdate", latest.CreationDate.Unix()+1)

	// Output query as table
	d, _ := w.Flag("debug")
	debbuging := d.Present()
	if debbuging {
		searchAdvanced.DrawQuery(w)
	}

	fetchQuestions := true
	var lastQuestion internal.QuestionObj
	for fetchQuestions {
		w.Log.Debugf("Fetching page %d", searchAdvanced.GetCurrentPageNr())
		if results, err := searchAdvanced.Get(); results {
			// Questions received
			for _, q := range searchAdvanced.Result.Items {

				w.Log.Infof("Question: %s", q.Title)
				w.Log.Infof("Url:      %s", q.ShareLink)

				if debbuging {
					printQuestion(q)
				}
				// Skip sync if there are locally no questions
				if empty {
					lastQuestion = q
					continue
				}
				so.SyncQuestion(w, q)
			}
			if err != nil {
				fetchQuestions = false
				w.Log.Error(err)
			}
		}

		// Done go to next page
		if searchAdvanced.GetCurrentPageNr() > 10 {
			fetchQuestions = false
		} else if searchAdvanced.HasMore() {
			searchAdvanced.NextPage()
		} else {
			fetchQuestions = false
			w.Log.Debug("There are no more new questions.")
		}
	}
	if empty && lastQuestion.QID > 0 {
		so.SyncQuestion(w, lastQuestion)
	}
}

func updateQuestions(w *cli.Worker, so *internal.SlackOverflow) {
	w.Log.Info("Stack Exchange: Updating existing questions.")

	questionIds, questionIdsCount := so.DB.StackExchangeQuestionTrackedIds(
		so.Config.StackExchange.QuestionsToWatch)

	// Check do we already have some questions do obtain from_date for next request
	if questionIdsCount == 0 {
		w.Log.Info("There are no questions in database,")
		w.Log.Info("That is ok if current execution is first time you run slackoverflow")
		w.Log.Info("and this case run 'slackoverflow stackechange guestions' --get first")
	}

	w.Log.Infof("Checking updates for %d questions. Max to be tracked (%d)",
		questionIdsCount, so.Config.StackExchange.QuestionsToWatch)

	// Check for New Questions from Stack Exchange
	updateQuestions := so.StackExchange.Questions()

	// Set it here so that it is allowed to override by config
	updateQuestions.Parameters.Set("site", so.Config.StackExchange.Site)

	for param, value := range so.Config.StackExchange.Questions {
		updateQuestions.Parameters.Set(param, value)
	}

	// Output query as table
	d, _ := w.Flag("debug")
	debbuging := d.Present()
	if debbuging {
		updateQuestions.DrawQuery(w, questionIds)
	}

	fetchQuestions := true
	for fetchQuestions {
		w.Log.Debugf("Fetching page %d", updateQuestions.GetCurrentPageNr())
		if results, err := updateQuestions.Get(questionIds); results {
			// Questions received
			for _, q := range updateQuestions.Result.Items {

				w.Log.Infof("Question: %s", q.Title)
				w.Log.Infof("Url:      %s", q.ShareLink)
				if debbuging {
					printQuestion(q)
				}

				so.SyncQuestion(w, q)
			}
			if err != nil {
				fetchQuestions = false
				w.Log.Error(err.Error())
			}
		}

		// Done go to next page
		if updateQuestions.HasMore() {
			updateQuestions.NextPage()
		} else {
			fetchQuestions = false
			w.Log.Debug("There are no more questions to update.")
		}
	}
}

func startWatching(w *cli.Worker, so *internal.SlackOverflow) {
	// Check for New Questions from Stack Exchange
	searchAdvanced := so.StackExchange.SearchAdvanced()

	// Set it here so that it is allowed to override by config
	searchAdvanced.Parameters.Set("site", so.Config.StackExchange.Site)

	// Set all parameters from config
	for param, value := range so.Config.StackExchange.SearchAdvanced {
		searchAdvanced.Parameters.Set(param, value)
	}
	searchAdvanced.Parameters.Set("tagged", "php")
	searchAdvanced.Parameters.Set("fromdate", fromDate.Unix()+1)

	fetchQuestions := true
	for fetchQuestions {
		w.Log.Debugf("Fetching page %d", searchAdvanced.GetCurrentPageNr())
		if results, err := searchAdvanced.Get(); results {
			// Questions received
			for _, q := range searchAdvanced.Result.Items {
				fromDate = time.Unix(q.CreationDate, 0).UTC()
				w.Log.Linef("Question: %s", q.Title)
				w.Log.Linef("Url:      %s", q.ShareLink)
				newq := internal.NewTable("Question ID", "Time", "Answers", "Comments", "Score", "Views", "Username")
				newq.AddRow(
					q.QID,
					time.Unix(q.CreationDate, 0).Local().Format("15:04:05 Mon Jan _2 2006"),
					q.AnswerCount,
					q.CommentCount,
					q.Score,
					q.ViewCount,
					q.Owner.DisplayName,
				)
				newq.Print()
			}
			if err != nil {
				fetchQuestions = false
				w.Log.Error(err.Error())
			}

			if len(searchAdvanced.Result.Items) > 0 {
				w.Log.Ok("Waiting for new questions!")
			}
		}

		// Done go to next page
		if searchAdvanced.HasMore() || searchAdvanced.GetCurrentPageNr() > 10 {
			searchAdvanced.NextPage()
		} else {
			fetchQuestions = false
		}
	}

	w.Log.Infof(
		"Stack Exchange Quota usage (%d/%d)",
		so.StackExchange.GetQuotaRemaining(),
		so.StackExchange.GetQuotaMax(),
	)
}

func printQuestion(q internal.QuestionObj) {
	newq := internal.NewTable("Question ID", "Time", "Answers", "Comments", "Score", "Views", "Username")
	newq.AddRow(
		q.QID,
		time.Unix(q.CreationDate, 0).UTC().Format("15:04:05 Mon Jan _2 2006"),
		q.AnswerCount,
		q.CommentCount,
		q.Score,
		q.ViewCount,
		q.Owner.DisplayName,
	)
	newq.Print()
}
<|endoftext|>


<commit_before>// Copyright 2014 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package core

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/big"
	"strings"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/state"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/logger/glog"
	"github.com/ethereum/go-ethereum/params"
)

// WriteGenesisBlock writes the genesis block to the database as block number 0
func WriteGenesisBlock(stateDb, blockDb common.Database, reader io.Reader) (*types.Block, error) {
	contents, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}

	var genesis struct {
		Nonce      string
		Timestamp  string
		ParentHash string
		ExtraData  string
		GasLimit   string
		Difficulty string
		Mixhash    string
		Coinbase   string
		Alloc      map[string]struct {
			Code    string
			Storage map[string]string
			Balance string
		}
	}

	if err := json.Unmarshal(contents, &genesis); err != nil {
		return nil, err
	}

	statedb := state.New(common.Hash{}, stateDb)
	for addr, account := range genesis.Alloc {
		address := common.HexToAddress(addr)
		statedb.AddBalance(address, common.String2Big(account.Balance))
		statedb.SetCode(address, common.Hex2Bytes(account.Code))
		for key, value := range account.Storage {
			statedb.SetState(address, common.HexToHash(key), common.HexToHash(value))
		}
	}
	statedb.SyncObjects()

	difficulty := common.String2Big(genesis.Difficulty)
	block := types.NewBlock(&types.Header{
		Nonce:      types.EncodeNonce(common.String2Big(genesis.Nonce).Uint64()),
		Time:       common.String2Big(genesis.Timestamp).Uint64(),
		ParentHash: common.HexToHash(genesis.ParentHash),
		Extra:      common.Hex2Bytes(genesis.ExtraData),
		GasLimit:   common.String2Big(genesis.GasLimit),
		Difficulty: difficulty,
		MixDigest:  common.HexToHash(genesis.Mixhash),
		Coinbase:   common.HexToAddress(genesis.Coinbase),
		Root:       statedb.Root(),
	}, nil, nil, nil)
	block.Td = difficulty

	if block := GetBlockByHash(blockDb, block.Hash()); block != nil {
		glog.V(logger.Info).Infoln("Genesis block already in chain. Writing canonical number")
		err := WriteCanonNumber(blockDb, block)
		if err != nil {
			return nil, err
		}
		return block, nil
	}

	statedb.Sync()

	err = WriteBlock(blockDb, block)
	if err != nil {
		return nil, err
	}
	err = WriteHead(blockDb, block)
	if err != nil {
		return nil, err
	}

	return block, nil
}

// GenesisBlockForTesting creates a block in which addr has the given wei balance.
// The state trie of the block is written to db.
func GenesisBlockForTesting(db common.Database, addr common.Address, balance *big.Int) *types.Block {
	statedb := state.New(common.Hash{}, db)
	obj := statedb.GetOrNewStateObject(addr)
	obj.SetBalance(balance)
	statedb.SyncObjects()
	statedb.Sync()
	block := types.NewBlock(&types.Header{
		Difficulty: params.GenesisDifficulty,
		GasLimit:   params.GenesisGasLimit,
		Root:       statedb.Root(),
	}, nil, nil, nil)
	block.Td = params.GenesisDifficulty
	return block
}

func WriteGenesisBlockForTesting(db common.Database, addr common.Address, balance *big.Int) *types.Block {
	testGenesis := fmt.Sprintf(`{
	"nonce":"0x%x",
	"gasLimit":"0x%x",
	"difficulty":"0x%x",
	"alloc": {
		"0x%x":{"balance":"0x%x"}
	}
}`, types.EncodeNonce(0), params.GenesisGasLimit.Bytes(), params.GenesisDifficulty.Bytes(), addr, balance.Bytes())
	block, _ := WriteGenesisBlock(db, db, strings.NewReader(testGenesis))
	return block
}

func WriteTestNetGenesisBlock(stateDb, blockDb common.Database, nonce uint64) (*types.Block, error) {
	testGenesis := fmt.Sprintf(`{
	"nonce":"0x%x",
	"gasLimit":"0x%x",
	"difficulty":"0x%x",
	"alloc": {
		"0000000000000000000000000000000000000001": {"balance": "1"},
		"0000000000000000000000000000000000000002": {"balance": "1"},
		"0000000000000000000000000000000000000003": {"balance": "1"},
		"0000000000000000000000000000000000000004": {"balance": "1"},
		"dbdbdb2cbd23b783741e8d7fcf51e459b497e4a6": {"balance": "1606938044258990275541962092341162602522202993782792835301376"},
		"e4157b34ea9615cfbde6b4fda419828124b70c78": {"balance": "1606938044258990275541962092341162602522202993782792835301376"},
		"b9c015918bdaba24b4ff057a92a3873d6eb201be": {"balance": "1606938044258990275541962092341162602522202993782792835301376"},
		"6c386a4b26f73c802f34673f7248bb118f97424a": {"balance": "1606938044258990275541962092341162602522202993782792835301376"},
		"cd2a3d9f938e13cd947ec05abc7fe734df8dd826": {"balance": "1606938044258990275541962092341162602522202993782792835301376"},
		"2ef47100e0787b915105fd5e3f4ff6752079d5cb": {"balance": "1606938044258990275541962092341162602522202993782792835301376"},
		"e6716f9544a56c530d868e4bfbacb172315bdead": {"balance": "1606938044258990275541962092341162602522202993782792835301376"},
		"1a26338f0d905e295fccb71fa9ea849ffa12aaf4": {"balance": "1606938044258990275541962092341162602522202993782792835301376"}
	}
}`, types.EncodeNonce(nonce), params.GenesisGasLimit.Bytes(), params.GenesisDifficulty.Bytes())
	return WriteGenesisBlock(stateDb, blockDb, strings.NewReader(testGenesis))
}
<commit_msg>core: genesis extra data field fix<commit_after>// Copyright 2014 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package core

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/big"
	"strings"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/state"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/logger/glog"
	"github.com/ethereum/go-ethereum/params"
)

// WriteGenesisBlock writes the genesis block to the database as block number 0
func WriteGenesisBlock(stateDb, blockDb common.Database, reader io.Reader) (*types.Block, error) {
	contents, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}

	var genesis struct {
		Nonce      string
		Timestamp  string
		ParentHash string
		ExtraData  string
		GasLimit   string
		Difficulty string
		Mixhash    string
		Coinbase   string
		Alloc      map[string]struct {
			Code    string
			Storage map[string]string
			Balance string
		}
	}

	if err := json.Unmarshal(contents, &genesis); err != nil {
		return nil, err
	}

	statedb := state.New(common.Hash{}, stateDb)
	for addr, account := range genesis.Alloc {
		address := common.HexToAddress(addr)
		statedb.AddBalance(address, common.String2Big(account.Balance))
		statedb.SetCode(address, common.Hex2Bytes(account.Code))
		for key, value := range account.Storage {
			statedb.SetState(address, common.HexToHash(key), common.HexToHash(value))
		}
	}
	statedb.SyncObjects()

	difficulty := common.String2Big(genesis.Difficulty)
	block := types.NewBlock(&types.Header{
		Nonce:      types.EncodeNonce(common.String2Big(genesis.Nonce).Uint64()),
		Time:       common.String2Big(genesis.Timestamp).Uint64(),
		ParentHash: common.HexToHash(genesis.ParentHash),
		Extra:      common.FromHex(genesis.ExtraData),
		GasLimit:   common.String2Big(genesis.GasLimit),
		Difficulty: difficulty,
		MixDigest:  common.HexToHash(genesis.Mixhash),
		Coinbase:   common.HexToAddress(genesis.Coinbase),
		Root:       statedb.Root(),
	}, nil, nil, nil)
	block.Td = difficulty

	if block := GetBlockByHash(blockDb, block.Hash()); block != nil {
		glog.V(logger.Info).Infoln("Genesis block already in chain. Writing canonical number")
		err := WriteCanonNumber(blockDb, block)
		if err != nil {
			return nil, err
		}
		return block, nil
	}

	statedb.Sync()

	err = WriteBlock(blockDb, block)
	if err != nil {
		return nil, err
	}
	err = WriteHead(blockDb, block)
	if err != nil {
		return nil, err
	}

	return block, nil
}

// GenesisBlockForTesting creates a block in which addr has the given wei balance.
// The state trie of the block is written to db.
func GenesisBlockForTesting(db common.Database, addr common.Address, balance *big.Int) *types.Block {
	statedb := state.New(common.Hash{}, db)
	obj := statedb.GetOrNewStateObject(addr)
	obj.SetBalance(balance)
	statedb.SyncObjects()
	statedb.Sync()
	block := types.NewBlock(&types.Header{
		Difficulty: params.GenesisDifficulty,
		GasLimit:   params.GenesisGasLimit,
		Root:       statedb.Root(),
	}, nil, nil, nil)
	block.Td = params.GenesisDifficulty
	return block
}

func WriteGenesisBlockForTesting(db common.Database, addr common.Address, balance *big.Int) *types.Block {
	testGenesis := fmt.Sprintf(`{
	"nonce":"0x%x",
	"gasLimit":"0x%x",
	"difficulty":"0x%x",
	"alloc": {
		"0x%x":{"balance":"0x%x"}
	}
}`, types.EncodeNonce(0), params.GenesisGasLimit.Bytes(), params.GenesisDifficulty.Bytes(), addr, balance.Bytes())
	block, _ := WriteGenesisBlock(db, db, strings.NewReader(testGenesis))
	return block
}

func WriteTestNetGenesisBlock(stateDb, blockDb common.Database, nonce uint64) (*types.Block, error) {
	testGenesis := fmt.Sprintf(`{
	"nonce":"0x%x",
	"gasLimit":"0x%x",
	"difficulty":"0x%x",
	"alloc": {
		"0000000000000000000000000000000000000001": {"balance": "1"},
		"0000000000000000000000000000000000000002": {"balance": "1"},
		"0000000000000000000000000000000000000003": {"balance": "1"},
		"0000000000000000000000000000000000000004": {"balance": "1"},
		"dbdbdb2cbd23b783741e8d7fcf51e459b497e4a6": {"balance": "1606938044258990275541962092341162602522202993782792835301376"},
		"e4157b34ea9615cfbde6b4fda419828124b70c78": {"balance": "1606938044258990275541962092341162602522202993782792835301376"},
		"b9c015918bdaba24b4ff057a92a3873d6eb201be": {"balance": "1606938044258990275541962092341162602522202993782792835301376"},
		"6c386a4b26f73c802f34673f7248bb118f97424a": {"balance": "1606938044258990275541962092341162602522202993782792835301376"},
		"cd2a3d9f938e13cd947ec05abc7fe734df8dd826": {"balance": "1606938044258990275541962092341162602522202993782792835301376"},
		"2ef47100e0787b915105fd5e3f4ff6752079d5cb": {"balance": "1606938044258990275541962092341162602522202993782792835301376"},
		"e6716f9544a56c530d868e4bfbacb172315bdead": {"balance": "1606938044258990275541962092341162602522202993782792835301376"},
		"1a26338f0d905e295fccb71fa9ea849ffa12aaf4": {"balance": "1606938044258990275541962092341162602522202993782792835301376"}
	}
}`, types.EncodeNonce(nonce), params.GenesisGasLimit.Bytes(), params.GenesisDifficulty.Bytes())
	return WriteGenesisBlock(stateDb, blockDb, strings.NewReader(testGenesis))
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/multierror"

	"github.com/hashicorp/aws-sdk-go/aws"
	"github.com/hashicorp/aws-sdk-go/gen/autoscaling"
	"github.com/hashicorp/aws-sdk-go/gen/ec2"
	"github.com/hashicorp/aws-sdk-go/gen/elb"
	"github.com/hashicorp/aws-sdk-go/gen/iam"
	"github.com/hashicorp/aws-sdk-go/gen/rds"
	"github.com/hashicorp/aws-sdk-go/gen/route53"
	"github.com/hashicorp/aws-sdk-go/gen/s3"

	awsSDK "github.com/awslabs/aws-sdk-go/aws"
	awsEC2 "github.com/awslabs/aws-sdk-go/service/ec2"
)

type Config struct {
	AccessKey string
	SecretKey string
	Token     string
	Region    string
}

type AWSClient struct {
	ec2conn         *ec2.EC2
	elbconn         *elb.ELB
	autoscalingconn *autoscaling.AutoScaling
	s3conn          *s3.S3
	r53conn         *route53.Route53
	region          string
	rdsconn         *rds.RDS
	iamconn         *iam.IAM
	ec2SDKconn      *awsEC2.EC2
}

// Client configures and returns a fully initailized AWSClient
func (c *Config) Client() (interface{}, error) {
	var client AWSClient

	// Get the auth and region. This can fail if keys/regions were not
	// specified and we're attempting to use the environment.
	var errs []error

	log.Println("[INFO] Building AWS region structure")
	err := c.ValidateRegion()
	if err != nil {
		errs = append(errs, err)
	}

	if len(errs) == 0 {
		// store AWS region in client struct, for region specific operations such as
		// bucket storage in S3
		client.region = c.Region

		log.Println("[INFO] Building AWS auth structure")
		creds := aws.Creds(c.AccessKey, c.SecretKey, c.Token)

		log.Println("[INFO] Initializing ELB connection")
		client.elbconn = elb.New(creds, c.Region, nil)
		log.Println("[INFO] Initializing AutoScaling connection")
		client.autoscalingconn = autoscaling.New(creds, c.Region, nil)
		log.Println("[INFO] Initializing S3 connection")
		client.s3conn = s3.New(creds, c.Region, nil)
		log.Println("[INFO] Initializing RDS connection")
		client.rdsconn = rds.New(creds, c.Region, nil)

		// aws-sdk-go uses v4 for signing requests, which requires all global
		// endpoints to use 'us-east-1'.
		// See http://docs.aws.amazon.com/general/latest/gr/sigv4_changes.html
		log.Println("[INFO] Initializing Route53 connection")
		client.r53conn = route53.New(creds, "us-east-1", nil)
		log.Println("[INFO] Initializing EC2 Connection")
		client.ec2conn = ec2.New(creds, c.Region, nil)

		client.iamconn = iam.New(creds, c.Region, nil)
		client.ec2SDKconn = awsEC2.New(&awsSDK.Config{Region: c.Region})
	}

	if len(errs) > 0 {
		return nil, &multierror.Error{Errors: errs}
	}

	return &client, nil
}

// IsValidRegion returns true if the configured region is a valid AWS
// region and false if it's not
func (c *Config) ValidateRegion() error {
	var regions = [11]string{"us-east-1", "us-west-2", "us-west-1", "eu-west-1",
		"eu-central-1", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1",
		"sa-east-1", "cn-north-1", "us-gov-west-1"}

	for _, valid := range regions {
		if c.Region == valid {
			return nil
		}
	}
	return fmt.Errorf("Not a valid region: %s", c.Region)
}
<commit_msg>provider/aws: Detect creds in AWS config<commit_after>package aws

import (
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/multierror"

	"github.com/hashicorp/aws-sdk-go/aws"
	"github.com/hashicorp/aws-sdk-go/gen/autoscaling"
	"github.com/hashicorp/aws-sdk-go/gen/ec2"
	"github.com/hashicorp/aws-sdk-go/gen/elb"
	"github.com/hashicorp/aws-sdk-go/gen/iam"
	"github.com/hashicorp/aws-sdk-go/gen/rds"
	"github.com/hashicorp/aws-sdk-go/gen/route53"
	"github.com/hashicorp/aws-sdk-go/gen/s3"

	awsSDK "github.com/awslabs/aws-sdk-go/aws"
	awsEC2 "github.com/awslabs/aws-sdk-go/service/ec2"
)

type Config struct {
	AccessKey string
	SecretKey string
	Token     string
	Region    string
}

type AWSClient struct {
	ec2conn         *ec2.EC2
	elbconn         *elb.ELB
	autoscalingconn *autoscaling.AutoScaling
	s3conn          *s3.S3
	r53conn         *route53.Route53
	region          string
	rdsconn         *rds.RDS
	iamconn         *iam.IAM
	ec2SDKconn      *awsEC2.EC2
}

// Client configures and returns a fully initailized AWSClient
func (c *Config) Client() (interface{}, error) {
	var client AWSClient

	// Get the auth and region. This can fail if keys/regions were not
	// specified and we're attempting to use the environment.
	var errs []error

	log.Println("[INFO] Building AWS region structure")
	err := c.ValidateRegion()
	if err != nil {
		errs = append(errs, err)
	}

	if len(errs) == 0 {
		// store AWS region in client struct, for region specific operations such as
		// bucket storage in S3
		client.region = c.Region

		log.Println("[INFO] Building AWS auth structure")
		creds := aws.DetectCreds(c.AccessKey, c.SecretKey, c.Token)

		log.Println("[INFO] Initializing ELB connection")
		client.elbconn = elb.New(creds, c.Region, nil)
		log.Println("[INFO] Initializing AutoScaling connection")
		client.autoscalingconn = autoscaling.New(creds, c.Region, nil)
		log.Println("[INFO] Initializing S3 connection")
		client.s3conn = s3.New(creds, c.Region, nil)
		log.Println("[INFO] Initializing RDS connection")
		client.rdsconn = rds.New(creds, c.Region, nil)

		// aws-sdk-go uses v4 for signing requests, which requires all global
		// endpoints to use 'us-east-1'.
		// See http://docs.aws.amazon.com/general/latest/gr/sigv4_changes.html
		log.Println("[INFO] Initializing Route53 connection")
		client.r53conn = route53.New(creds, "us-east-1", nil)
		log.Println("[INFO] Initializing EC2 Connection")
		client.ec2conn = ec2.New(creds, c.Region, nil)
		client.iamconn = iam.New(creds, c.Region, nil)

		sdkCreds := awsSDK.DetectCreds(c.AccessKey, c.SecretKey, c.Token)
		client.ec2SDKconn = awsEC2.New(&awsSDK.Config{
			Credentials: sdkCreds,
			Region:      c.Region,
		})
	}

	if len(errs) > 0 {
		return nil, &multierror.Error{Errors: errs}
	}

	return &client, nil
}

// IsValidRegion returns true if the configured region is a valid AWS
// region and false if it's not
func (c *Config) ValidateRegion() error {
	var regions = [11]string{"us-east-1", "us-west-2", "us-west-1", "eu-west-1",
		"eu-central-1", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1",
		"sa-east-1", "cn-north-1", "us-gov-west-1"}

	for _, valid := range regions {
		if c.Region == valid {
			return nil
		}
	}
	return fmt.Errorf("Not a valid region: %s", c.Region)
}
<|endoftext|>


<commit_before>package adt

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"math"
	"os"
	"path/filepath"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
)

const (
	HeaderLength           = 400
	ColumnDescriptorLength = 200
	MagicHeader            = "Advantage Table"
)

var (
	ErrMagicHeaderNotFound = errors.New("adt: magic header missing")
	ErrMultiplePKs         = errors.New("adt: multiple primary keys")
)

type Table struct {
	Name         string
	RecordCount  uint32
	DataOffset   uint16
	RecordLength uint32
	Columns      []*Column
	data         io.ReadSeeker
	memoData     io.ReadSeeker
}

func TableFromPath(filePath string) (*Table, error) {
	adt, err := os.Open(filePath)
	if err != nil {
		return nil, err
	}
	ext := filepath.Ext(filePath)
	admPath := filePath[:len(filePath)-len(ext)] + ".ADM"
	adm, _ := os.Open(admPath)
	// adm isn't required.
	table, err := FromReaders(adt, adm)
	table.Name = filepath.Base(filePath)
	return table, err
}

func FromReaders(adtContent io.ReadSeeker, admContent io.ReadSeeker) (*Table, error) {
	header := make([]byte, HeaderLength)
	if _, err := io.ReadAtLeast(adtContent, header, HeaderLength); err != nil {
		return nil, err
	}
	if string(header[:len(MagicHeader)]) != MagicHeader {
		return nil, ErrMagicHeaderNotFound
	}
	table := &Table{
		Columns:  []*Column{},
		data:     adtContent,
		memoData: admContent,
	}
	if err := readLE(header[24:], &table.RecordCount); err != nil {
		return nil, err
	}
	if err := readLE(header[32:], &table.DataOffset); err != nil {
		return nil, err
	}
	if err := readLE(header[36:], &table.RecordLength); err != nil {
		return nil, err
	}
	for i := 0; i < table.columnCount(); i++ {
		c, err := ColumnFromReader(adtContent)
		if err != nil {
			return nil, err
		}
		table.Columns = append(table.Columns, c)
	}
	return table, nil
}

func (t *Table) columnCount() int {
	return int((t.DataOffset - HeaderLength) / 200)
}

func readLE(src []byte, dest interface{}) error {
	return binary.Read(bytes.NewReader(src), binary.LittleEndian, dest)
}

func (t *Table) GetPK() (*Column, error) {
	var result *Column
	for _, c := range t.Columns {
		if c.Type == ColumnTypeAutoIncrement {
			if result != nil {
				return nil, ErrMultiplePKs
			}
			result = c
		}
	}
	return result, nil
}

func (t *Table) Get(record int) (Record, error) {
	t.data.Seek(int64(int(t.DataOffset)+int(t.RecordLength)*record), 0)
	return t.readRecord()
}

func (t *Table) readRecord() (Record, error) {
	r := Record{}
	bytes := make([]byte, t.RecordLength)
	if r, err := io.ReadFull(t.data, bytes); err != nil {
		log.Warn("didn't read enough: ", r, err)
		return nil, err
	}
	if string(bytes[:len(RecordMagicHeader)]) != RecordMagicHeader {
		//return nil, ErrMagicHeaderNotFound
	}
	for _, column := range t.Columns {
		value, err := ReadValue(bytes, column)
		// dbg:
		//valueBytes := bytes[column.Offset : column.Offset+column.Length]

		if asMemo, ok := value.(MemoField); ok {
			t.memoData.Seek(int64(asMemo.BlockOffset)*8, 0)
			data := make([]byte, asMemo.Length)
			if _, err := io.ReadFull(t.memoData, data); err != nil {
				log.Warnln("didn't read enough for memo field", column.Name, err)
				return nil, nil
			}
			value = data
		}

		if err != nil {
			return nil, err
		}
		r[column.Name] = value
	}
	return r, nil
}

func ReadValue(src []byte, column *Column) (interface{}, error) {
	valueBytes := src[column.Offset : column.Offset+column.Length]
	switch column.Type {
	case ColumnTypeCharacter:
		runes := make([]rune, 0, len(valueBytes))
		for _, b := range valueBytes {
			runes = append(runes, rune(b))
		}
		return strings.Trim(string(runes), " \u0000"), nil
	case ColumnTypeShortInt:
		var value int16
		err := binary.Read(bytes.NewReader(valueBytes), binary.LittleEndian, &value)
		// null encoded as minint
		if value == math.MinInt16 {
			return nil, nil
		}
		return value, err
	case ColumnTypeInt:
		var value int32
		err := binary.Read(bytes.NewReader(valueBytes), binary.LittleEndian, &value)
		if value == math.MinInt32 {
			return nil, nil
		}
		return value, err
	case ColumnTypeBlob:
		buf := src[column.Offset : column.Offset+column.Length]
		return buf, nil
	case ColumnTypeMemo:
		var value MemoField
		err := binary.Read(bytes.NewReader(valueBytes), binary.LittleEndian, &value)
		return value, err
	case ColumnTypeAutoIncrement:
		var value uint32
		err := binary.Read(bytes.NewReader(valueBytes), binary.LittleEndian, &value)
		return value, err
	case ColumnTypeBool:
		var value bool
		if src[column.Offset : column.Offset+column.Length][0] == 'T' {
			value = true
		}
		return value, nil
	case ColumnTypeTime:
		buf := src[column.Offset : column.Offset+column.Length]
		i := binary.LittleEndian.Uint32(buf)
		n := int32(i)
		if n == -1 {
			return time.Duration(0), nil
		}
		return time.Millisecond * time.Duration(n), nil
	case ColumnTypeTimestamp:
		buf := src[column.Offset : column.Offset+column.Length]
		i := binary.LittleEndian.Uint32(buf[:4])
		j := binary.LittleEndian.Uint32(buf[4:])
		value := adtDatetimeToTime(int32(i), int32(j))
		if i == 0 {
			return nil, nil
		}
		return value, nil
	case ColumnTypeDate:
		buf := src[column.Offset : column.Offset+column.Length]
		i := binary.LittleEndian.Uint32(buf)
		value := adtDateToTime(int32(i))
		if i == 0 {
			return nil, nil
		}
		return value, nil
	case ColumnTypeCurrency:
		fallthrough
	case ColumnTypeDouble:
		buf := src[column.Offset : column.Offset+column.Length]
		var value float64
		err := binary.Read(bytes.NewReader(buf), binary.LittleEndian, &value)
		if value == -1.6e-322 {
			return nil, err
		}
		return value, err
	default:
		value := make([]byte, column.Length)
		copy(value, src[column.Offset:column.Offset+column.Length])
		return nil, fmt.Errorf("adt ReadValue: %s not implemented", column.Type)
		return value, nil
	}
}
<commit_msg>logrus: use new import path<commit_after>package adt

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"math"
	"os"
	"path/filepath"
	"strings"
	"time"

	log "github.com/sirupsen/logrus"
)

const (
	HeaderLength           = 400
	ColumnDescriptorLength = 200
	MagicHeader            = "Advantage Table"
)

var (
	ErrMagicHeaderNotFound = errors.New("adt: magic header missing")
	ErrMultiplePKs         = errors.New("adt: multiple primary keys")
)

type Table struct {
	Name         string
	RecordCount  uint32
	DataOffset   uint16
	RecordLength uint32
	Columns      []*Column
	data         io.ReadSeeker
	memoData     io.ReadSeeker
}

func TableFromPath(filePath string) (*Table, error) {
	adt, err := os.Open(filePath)
	if err != nil {
		return nil, err
	}
	ext := filepath.Ext(filePath)
	admPath := filePath[:len(filePath)-len(ext)] + ".ADM"
	adm, _ := os.Open(admPath)
	// adm isn't required.
	table, err := FromReaders(adt, adm)
	table.Name = filepath.Base(filePath)
	return table, err
}

func FromReaders(adtContent io.ReadSeeker, admContent io.ReadSeeker) (*Table, error) {
	header := make([]byte, HeaderLength)
	if _, err := io.ReadAtLeast(adtContent, header, HeaderLength); err != nil {
		return nil, err
	}
	if string(header[:len(MagicHeader)]) != MagicHeader {
		return nil, ErrMagicHeaderNotFound
	}
	table := &Table{
		Columns:  []*Column{},
		data:     adtContent,
		memoData: admContent,
	}
	if err := readLE(header[24:], &table.RecordCount); err != nil {
		return nil, err
	}
	if err := readLE(header[32:], &table.DataOffset); err != nil {
		return nil, err
	}
	if err := readLE(header[36:], &table.RecordLength); err != nil {
		return nil, err
	}
	for i := 0; i < table.columnCount(); i++ {
		c, err := ColumnFromReader(adtContent)
		if err != nil {
			return nil, err
		}
		table.Columns = append(table.Columns, c)
	}
	return table, nil
}

func (t *Table) columnCount() int {
	return int((t.DataOffset - HeaderLength) / 200)
}

func readLE(src []byte, dest interface{}) error {
	return binary.Read(bytes.NewReader(src), binary.LittleEndian, dest)
}

func (t *Table) GetPK() (*Column, error) {
	var result *Column
	for _, c := range t.Columns {
		if c.Type == ColumnTypeAutoIncrement {
			if result != nil {
				return nil, ErrMultiplePKs
			}
			result = c
		}
	}
	return result, nil
}

func (t *Table) Get(record int) (Record, error) {
	t.data.Seek(int64(int(t.DataOffset)+int(t.RecordLength)*record), 0)
	return t.readRecord()
}

func (t *Table) readRecord() (Record, error) {
	r := Record{}
	bytes := make([]byte, t.RecordLength)
	if r, err := io.ReadFull(t.data, bytes); err != nil {
		log.Warn("didn't read enough: ", r, err)
		return nil, err
	}
	if string(bytes[:len(RecordMagicHeader)]) != RecordMagicHeader {
		//return nil, ErrMagicHeaderNotFound
	}
	for _, column := range t.Columns {
		value, err := ReadValue(bytes, column)
		// dbg:
		//valueBytes := bytes[column.Offset : column.Offset+column.Length]

		if asMemo, ok := value.(MemoField); ok {
			t.memoData.Seek(int64(asMemo.BlockOffset)*8, 0)
			data := make([]byte, asMemo.Length)
			if _, err := io.ReadFull(t.memoData, data); err != nil {
				log.Warnln("didn't read enough for memo field", column.Name, err)
				return nil, nil
			}
			value = data
		}

		if err != nil {
			return nil, err
		}
		r[column.Name] = value
	}
	return r, nil
}

func ReadValue(src []byte, column *Column) (interface{}, error) {
	valueBytes := src[column.Offset : column.Offset+column.Length]
	switch column.Type {
	case ColumnTypeCharacter:
		runes := make([]rune, 0, len(valueBytes))
		for _, b := range valueBytes {
			runes = append(runes, rune(b))
		}
		return strings.Trim(string(runes), " \u0000"), nil
	case ColumnTypeShortInt:
		var value int16
		err := binary.Read(bytes.NewReader(valueBytes), binary.LittleEndian, &value)
		// null encoded as minint
		if value == math.MinInt16 {
			return nil, nil
		}
		return value, err
	case ColumnTypeInt:
		var value int32
		err := binary.Read(bytes.NewReader(valueBytes), binary.LittleEndian, &value)
		if value == math.MinInt32 {
			return nil, nil
		}
		return value, err
	case ColumnTypeBlob:
		buf := src[column.Offset : column.Offset+column.Length]
		return buf, nil
	case ColumnTypeMemo:
		var value MemoField
		err := binary.Read(bytes.NewReader(valueBytes), binary.LittleEndian, &value)
		return value, err
	case ColumnTypeAutoIncrement:
		var value uint32
		err := binary.Read(bytes.NewReader(valueBytes), binary.LittleEndian, &value)
		return value, err
	case ColumnTypeBool:
		var value bool
		if src[column.Offset : column.Offset+column.Length][0] == 'T' {
			value = true
		}
		return value, nil
	case ColumnTypeTime:
		buf := src[column.Offset : column.Offset+column.Length]
		i := binary.LittleEndian.Uint32(buf)
		n := int32(i)
		if n == -1 {
			return time.Duration(0), nil
		}
		return time.Millisecond * time.Duration(n), nil
	case ColumnTypeTimestamp:
		buf := src[column.Offset : column.Offset+column.Length]
		i := binary.LittleEndian.Uint32(buf[:4])
		j := binary.LittleEndian.Uint32(buf[4:])
		value := adtDatetimeToTime(int32(i), int32(j))
		if i == 0 {
			return nil, nil
		}
		return value, nil
	case ColumnTypeDate:
		buf := src[column.Offset : column.Offset+column.Length]
		i := binary.LittleEndian.Uint32(buf)
		value := adtDateToTime(int32(i))
		if i == 0 {
			return nil, nil
		}
		return value, nil
	case ColumnTypeCurrency:
		fallthrough
	case ColumnTypeDouble:
		buf := src[column.Offset : column.Offset+column.Length]
		var value float64
		err := binary.Read(bytes.NewReader(buf), binary.LittleEndian, &value)
		if value == -1.6e-322 {
			return nil, err
		}
		return value, err
	default:
		value := make([]byte, column.Length)
		copy(value, src[column.Offset:column.Offset+column.Length])
		return nil, fmt.Errorf("adt ReadValue: %s not implemented", column.Type)
		return value, nil
	}
}
<|endoftext|>


<commit_before>package client

import (
	"errors"
	"net/http"

	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
	"github.com/Azure/azure-sdk-for-go/services/dns/mgmt/2017-10-01/dns"
	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2018-05-01/resources"
	"github.com/Azure/go-autorest/autorest"
	"github.com/Azure/go-autorest/autorest/adal"
	"github.com/Azure/go-autorest/autorest/azure"
	"github.com/giantswarm/microerror"
)

type AzureClientSetConfig struct {
	// ClientID is the ID of the Active Directory Service Principal.
	ClientID string
	// ClientSecret is the secret of the Active Directory Service Principal.
	ClientSecret string
	// The cloud environment identifier. Takes values from https://github.com/Azure/go-autorest/blob/ec5f4903f77ed9927ac95b19ab8e44ada64c1356/autorest/azure/environments.go#L13
	Cloud string
	// SubscriptionID is the ID of the Azure subscription.
	SubscriptionID string
	// TenantID is the ID of the Active Directory tenant.
	TenantID string
}

// clientConfig contains all essential information to create an Azure client.
type clientConfig struct {
	subscriptionID          string
	resourceManagerEndpoint string
	servicePrincipalToken   *adal.ServicePrincipalToken
}

func (c AzureClientSetConfig) Validate() error {
	if c.ClientID == "" {
		return errors.New("ClientID must not be empty")
	}
	if c.ClientSecret == "" {
		return errors.New("ClientSecret must not be empty")
	}
	if c.SubscriptionID == "" {
		return errors.New("SubscriptionID must not be empty")
	}
	if c.TenantID == "" {
		return errors.New("TenantID must not be empty")
	}

	return nil
}

// AzureClientSet is the collection of Azure API clients.
type AzureClientSet struct {
	// DeploymentsClient manages deployments of ARM templates.
	DeploymentsClient *resources.DeploymentsClient
	// GroupsClient manages ARM resource groups.
	GroupsClient *resources.GroupsClient
	// DNSRecordSetsClient manages DNS zones' records.
	DNSRecordSetsClient *dns.RecordSetsClient
	// DNSRecordSetsClient manages DNS zones.
	DNSZonesClient *dns.ZonesClient
	// InterfacesClient manages virtual network interfaces.
	InterfacesClient *network.InterfacesClient
	// VirtualNetworkClient manages virtual networks.
	VirtualNetworkClient *network.VirtualNetworksClient
	// VirtualNetworkGatewayConnectionsClient manages virtual network gateway connections.
	VirtualNetworkGatewayConnectionsClient *network.VirtualNetworkGatewayConnectionsClient
	// VirtualNetworkGatewaysClient manages virtual network gateways.
	VirtualNetworkGatewaysClient *network.VirtualNetworkGatewaysClient
	// VirtualMachineScaleSetsClient manages virtual machine scale sets.
	VirtualMachineScaleSetsClient *compute.VirtualMachineScaleSetsClient
	// VirtualMachineScaleSetVMsClient manages virtual machine scale set VMs.
	VirtualMachineScaleSetVMsClient *compute.VirtualMachineScaleSetVMsClient
	// VnetPeeringClient manages virtual network peerings.
	VnetPeeringClient *network.VirtualNetworkPeeringsClient
}

// NewAzureClientSet returns the Azure API clients.
func NewAzureClientSet(config AzureClientSetConfig) (*AzureClientSet, error) {
	if err := config.Validate(); err != nil {
		return nil, microerror.Maskf(invalidConfigError, "config.%s", err)
	}

	// Returns environment object contains all API endpoints for specific Azure cloud.
	// For empty config.Cloud returns Azure public cloud.
	env, err := parseAzureEnvironment(config.Cloud)
	if err != nil {
		return nil, microerror.Maskf(err, "parsing Azure environment")
	}

	servicePrincipalToken, err := newServicePrincipalToken(config, env)
	if err != nil {
		return nil, microerror.Maskf(err, "creating service principal token")
	}

	c := &clientConfig{
		subscriptionID:          config.SubscriptionID,
		resourceManagerEndpoint: env.ResourceManagerEndpoint,
		servicePrincipalToken:   servicePrincipalToken,
	}

	clientSet := &AzureClientSet{
		DeploymentsClient:                      newDeploymentsClient(c),
		GroupsClient:                           newGroupsClient(c),
		DNSRecordSetsClient:                    newDNSRecordSetsClient(c),
		DNSZonesClient:                         newDNSZonesClient(c),
		InterfacesClient:                       newInterfacesClient(c),
		VirtualNetworkClient:                   newVirtualNetworkClient(c),
		VirtualNetworkGatewayConnectionsClient: newVirtualNetworkGatewayConnectionsClient(c),
		VirtualNetworkGatewaysClient:           newVirtualNetworkGatewaysClient(c),
		VirtualMachineScaleSetVMsClient:        newVirtualMachineScaleSetVMsClient(c),
		VirtualMachineScaleSetsClient:          newVirtualMachineScaleSetsClient(c),
		VnetPeeringClient:                      newVnetPeeringClient(c),
	}

	return clientSet, nil
}

// ResponseWasNotFound returns true if the response code from the Azure API
// was a 404.
func ResponseWasNotFound(resp autorest.Response) bool {
	if resp.Response != nil && resp.StatusCode == http.StatusNotFound {
		return true
	}

	return false
}

func newDeploymentsClient(config *clientConfig) *resources.DeploymentsClient {
	c := resources.NewDeploymentsClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newGroupsClient(config *clientConfig) *resources.GroupsClient {
	c := resources.NewGroupsClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newDNSRecordSetsClient(config *clientConfig) *dns.RecordSetsClient {
	c := dns.NewRecordSetsClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newDNSZonesClient(config *clientConfig) *dns.ZonesClient {
	c := dns.NewZonesClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newInterfacesClient(config *clientConfig) *network.InterfacesClient {
	c := network.NewInterfacesClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newVirtualNetworkClient(config *clientConfig) *network.VirtualNetworksClient {
	c := network.NewVirtualNetworksClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newVirtualNetworkGatewayConnectionsClient(config *clientConfig) *network.VirtualNetworkGatewayConnectionsClient {
	c := network.NewVirtualNetworkGatewayConnectionsClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newVirtualNetworkGatewaysClient(config *clientConfig) *network.VirtualNetworkGatewaysClient {
	c := network.NewVirtualNetworkGatewaysClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newVirtualMachineScaleSetsClient(config *clientConfig) *compute.VirtualMachineScaleSetsClient {
	c := compute.NewVirtualMachineScaleSetsClient(config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newVirtualMachineScaleSetVMsClient(config *clientConfig) *compute.VirtualMachineScaleSetVMsClient {
	c := compute.NewVirtualMachineScaleSetVMsClient(config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newVnetPeeringClient(config *clientConfig) *network.VirtualNetworkPeeringsClient {
	c := network.NewVirtualNetworkPeeringsClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newServicePrincipalToken(config AzureClientSetConfig, env azure.Environment) (*adal.ServicePrincipalToken, error) {
	oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID)
	if err != nil {
		return nil, microerror.Maskf(err, "creating OAuth config")
	}

	token, err := adal.NewServicePrincipalToken(*oauthConfig, config.ClientID, config.ClientSecret, env.ServiceManagementEndpoint)
	if err != nil {
		return nil, microerror.Maskf(err, "getting token")
	}

	return token, nil
}

// parseAzureEnvironment returns azure environment by name.
func parseAzureEnvironment(cloudName string) (azure.Environment, error) {
	var env azure.Environment
	var err error
	if cloudName == "" {
		env = azure.PublicCloud
	} else {
		env, err = azure.EnvironmentFromName(cloudName)
		if err != nil {
			return env, microerror.Maskf(err, "parsing Azure environment")
		}
	}
	return env, microerror.Maskf(err, "parsing Azure environment")
}
<commit_msg>Use with URI function (#331)<commit_after>package client

import (
	"errors"
	"net/http"

	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
	"github.com/Azure/azure-sdk-for-go/services/dns/mgmt/2017-10-01/dns"
	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network"
	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2018-05-01/resources"
	"github.com/Azure/go-autorest/autorest"
	"github.com/Azure/go-autorest/autorest/adal"
	"github.com/Azure/go-autorest/autorest/azure"
	"github.com/giantswarm/microerror"
)

type AzureClientSetConfig struct {
	// ClientID is the ID of the Active Directory Service Principal.
	ClientID string
	// ClientSecret is the secret of the Active Directory Service Principal.
	ClientSecret string
	// The cloud environment identifier. Takes values from https://github.com/Azure/go-autorest/blob/ec5f4903f77ed9927ac95b19ab8e44ada64c1356/autorest/azure/environments.go#L13
	Cloud string
	// SubscriptionID is the ID of the Azure subscription.
	SubscriptionID string
	// TenantID is the ID of the Active Directory tenant.
	TenantID string
}

// clientConfig contains all essential information to create an Azure client.
type clientConfig struct {
	subscriptionID          string
	resourceManagerEndpoint string
	servicePrincipalToken   *adal.ServicePrincipalToken
}

func (c AzureClientSetConfig) Validate() error {
	if c.ClientID == "" {
		return errors.New("ClientID must not be empty")
	}
	if c.ClientSecret == "" {
		return errors.New("ClientSecret must not be empty")
	}
	if c.SubscriptionID == "" {
		return errors.New("SubscriptionID must not be empty")
	}
	if c.TenantID == "" {
		return errors.New("TenantID must not be empty")
	}

	return nil
}

// AzureClientSet is the collection of Azure API clients.
type AzureClientSet struct {
	// DeploymentsClient manages deployments of ARM templates.
	DeploymentsClient *resources.DeploymentsClient
	// GroupsClient manages ARM resource groups.
	GroupsClient *resources.GroupsClient
	// DNSRecordSetsClient manages DNS zones' records.
	DNSRecordSetsClient *dns.RecordSetsClient
	// DNSRecordSetsClient manages DNS zones.
	DNSZonesClient *dns.ZonesClient
	// InterfacesClient manages virtual network interfaces.
	InterfacesClient *network.InterfacesClient
	// VirtualNetworkClient manages virtual networks.
	VirtualNetworkClient *network.VirtualNetworksClient
	// VirtualNetworkGatewayConnectionsClient manages virtual network gateway connections.
	VirtualNetworkGatewayConnectionsClient *network.VirtualNetworkGatewayConnectionsClient
	// VirtualNetworkGatewaysClient manages virtual network gateways.
	VirtualNetworkGatewaysClient *network.VirtualNetworkGatewaysClient
	// VirtualMachineScaleSetsClient manages virtual machine scale sets.
	VirtualMachineScaleSetsClient *compute.VirtualMachineScaleSetsClient
	// VirtualMachineScaleSetVMsClient manages virtual machine scale set VMs.
	VirtualMachineScaleSetVMsClient *compute.VirtualMachineScaleSetVMsClient
	// VnetPeeringClient manages virtual network peerings.
	VnetPeeringClient *network.VirtualNetworkPeeringsClient
}

// NewAzureClientSet returns the Azure API clients.
func NewAzureClientSet(config AzureClientSetConfig) (*AzureClientSet, error) {
	if err := config.Validate(); err != nil {
		return nil, microerror.Maskf(invalidConfigError, "config.%s", err)
	}

	// Returns environment object contains all API endpoints for specific Azure cloud.
	// For empty config.Cloud returns Azure public cloud.
	env, err := parseAzureEnvironment(config.Cloud)
	if err != nil {
		return nil, microerror.Maskf(err, "parsing Azure environment")
	}

	servicePrincipalToken, err := newServicePrincipalToken(config, env)
	if err != nil {
		return nil, microerror.Maskf(err, "creating service principal token")
	}

	c := &clientConfig{
		subscriptionID:          config.SubscriptionID,
		resourceManagerEndpoint: env.ResourceManagerEndpoint,
		servicePrincipalToken:   servicePrincipalToken,
	}

	clientSet := &AzureClientSet{
		DeploymentsClient:                      newDeploymentsClient(c),
		GroupsClient:                           newGroupsClient(c),
		DNSRecordSetsClient:                    newDNSRecordSetsClient(c),
		DNSZonesClient:                         newDNSZonesClient(c),
		InterfacesClient:                       newInterfacesClient(c),
		VirtualNetworkClient:                   newVirtualNetworkClient(c),
		VirtualNetworkGatewayConnectionsClient: newVirtualNetworkGatewayConnectionsClient(c),
		VirtualNetworkGatewaysClient:           newVirtualNetworkGatewaysClient(c),
		VirtualMachineScaleSetVMsClient:        newVirtualMachineScaleSetVMsClient(c),
		VirtualMachineScaleSetsClient:          newVirtualMachineScaleSetsClient(c),
		VnetPeeringClient:                      newVnetPeeringClient(c),
	}

	return clientSet, nil
}

// ResponseWasNotFound returns true if the response code from the Azure API
// was a 404.
func ResponseWasNotFound(resp autorest.Response) bool {
	if resp.Response != nil && resp.StatusCode == http.StatusNotFound {
		return true
	}

	return false
}

func newDeploymentsClient(config *clientConfig) *resources.DeploymentsClient {
	c := resources.NewDeploymentsClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newGroupsClient(config *clientConfig) *resources.GroupsClient {
	c := resources.NewGroupsClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newDNSRecordSetsClient(config *clientConfig) *dns.RecordSetsClient {
	c := dns.NewRecordSetsClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newDNSZonesClient(config *clientConfig) *dns.ZonesClient {
	c := dns.NewZonesClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newInterfacesClient(config *clientConfig) *network.InterfacesClient {
	c := network.NewInterfacesClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newVirtualNetworkClient(config *clientConfig) *network.VirtualNetworksClient {
	c := network.NewVirtualNetworksClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newVirtualNetworkGatewayConnectionsClient(config *clientConfig) *network.VirtualNetworkGatewayConnectionsClient {
	c := network.NewVirtualNetworkGatewayConnectionsClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newVirtualNetworkGatewaysClient(config *clientConfig) *network.VirtualNetworkGatewaysClient {
	c := network.NewVirtualNetworkGatewaysClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newVirtualMachineScaleSetsClient(config *clientConfig) *compute.VirtualMachineScaleSetsClient {
	c := compute.NewVirtualMachineScaleSetsClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newVirtualMachineScaleSetVMsClient(config *clientConfig) *compute.VirtualMachineScaleSetVMsClient {
	c := compute.NewVirtualMachineScaleSetVMsClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newVnetPeeringClient(config *clientConfig) *network.VirtualNetworkPeeringsClient {
	c := network.NewVirtualNetworkPeeringsClientWithBaseURI(config.resourceManagerEndpoint, config.subscriptionID)
	c.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)

	return &c
}

func newServicePrincipalToken(config AzureClientSetConfig, env azure.Environment) (*adal.ServicePrincipalToken, error) {
	oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID)
	if err != nil {
		return nil, microerror.Maskf(err, "creating OAuth config")
	}

	token, err := adal.NewServicePrincipalToken(*oauthConfig, config.ClientID, config.ClientSecret, env.ServiceManagementEndpoint)
	if err != nil {
		return nil, microerror.Maskf(err, "getting token")
	}

	return token, nil
}

// parseAzureEnvironment returns azure environment by name.
func parseAzureEnvironment(cloudName string) (azure.Environment, error) {
	var env azure.Environment
	var err error
	if cloudName == "" {
		env = azure.PublicCloud
	} else {
		env, err = azure.EnvironmentFromName(cloudName)
		if err != nil {
			return env, microerror.Maskf(err, "parsing Azure environment")
		}
	}
	return env, microerror.Maskf(err, "parsing Azure environment")
}
<|endoftext|>


<commit_before>package fs

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"sync"

	"github.com/docker/libcontainer/cgroups"
	"github.com/docker/libcontainer/configs"
)

var (
	subsystems = map[string]subsystem{
		"devices":    &DevicesGroup{},
		"memory":     &MemoryGroup{},
		"cpu":        &CpuGroup{},
		"cpuset":     &CpusetGroup{},
		"cpuacct":    &CpuacctGroup{},
		"blkio":      &BlkioGroup{},
		"perf_event": &PerfEventGroup{},
		"freezer":    &FreezerGroup{},
	}
	CgroupProcesses = "cgroup.procs"
)

type subsystem interface {
	// Returns the stats, as 'stats', corresponding to the cgroup under 'path'.
	GetStats(path string, stats *cgroups.Stats) error
	// Removes the cgroup represented by 'data'.
	Remove(*data) error
	// Creates and joins the cgroup represented by data.
	Apply(*data) error
	// Set the cgroup represented by cgroup.
	Set(path string, cgroup *configs.Cgroup) error
}

type Manager struct {
	Cgroups *configs.Cgroup
	Paths   map[string]string
}

// The absolute path to the root of the cgroup hierarchies.
var cgroupRootLock sync.Mutex
var cgroupRoot string

// Gets the cgroupRoot.
func getCgroupRoot() (string, error) {
	cgroupRootLock.Lock()
	defer cgroupRootLock.Unlock()

	if cgroupRoot != "" {
		return cgroupRoot, nil
	}

	root, err := cgroups.FindCgroupMountpointDir()
	if err != nil {
		return "", err
	}

	if _, err := os.Stat(root); err != nil {
		return "", err
	}

	cgroupRoot = root
	return cgroupRoot, nil
}

type data struct {
	root   string
	cgroup string
	c      *configs.Cgroup
	pid    int
}

func (m *Manager) Apply(pid int) error {
	if m.Cgroups == nil {
		return nil
	}

	d, err := getCgroupData(m.Cgroups, pid)
	if err != nil {
		return err
	}

	paths := make(map[string]string)
	defer func() {
		if err != nil {
			cgroups.RemovePaths(paths)
		}
	}()
	for name, sys := range subsystems {
		if err := sys.Apply(d); err != nil {
			return err
		}
		// TODO: Apply should, ideally, be reentrant or be broken up into a separate
		// create and join phase so that the cgroup hierarchy for a container can be
		// created then join consists of writing the process pids to cgroup.procs
		p, err := d.path(name)
		if err != nil {
			if cgroups.IsNotFound(err) {
				continue
			}
			return err
		}
		paths[name] = p
	}
	m.Paths = paths

	return nil
}

func (m *Manager) Destroy() error {
	return cgroups.RemovePaths(m.Paths)
}

func (m *Manager) GetPaths() map[string]string {
	return m.Paths
}

// Symmetrical public function to update device based cgroups.  Also available
// in the systemd implementation.
func ApplyDevices(c *configs.Cgroup, pid int) error {
	d, err := getCgroupData(c, pid)
	if err != nil {
		return err
	}

	devices := subsystems["devices"]

	return devices.Apply(d)
}

func (m *Manager) GetStats() (*cgroups.Stats, error) {
	stats := cgroups.NewStats()
	for name, path := range m.Paths {
		sys, ok := subsystems[name]
		if !ok || !cgroups.PathExists(path) {
			continue
		}
		if err := sys.GetStats(path, stats); err != nil {
			return nil, err
		}
	}

	return stats, nil
}

func (m *Manager) Set(container *configs.Config) error {
	for name, path := range m.Paths {
		sys, ok := subsystems[name]
		if !ok || !cgroups.PathExists(path) {
			continue
		}
		if err := sys.Set(path, container.Cgroups); err != nil {
			return err
		}
	}

	return nil
}

// Freeze toggles the container's freezer cgroup depending on the state
// provided
func (m *Manager) Freeze(state configs.FreezerState) error {
	d, err := getCgroupData(m.Cgroups, 0)
	if err != nil {
		return err
	}

	dir, err := d.path("freezer")
	if err != nil {
		return err
	}
	if !cgroups.PathExists(dir) {
		return cgroups.NewNotFoundError("freezer")
	}

	prevState := m.Cgroups.Freezer
	m.Cgroups.Freezer = state

	freezer := subsystems["freezer"]
	err = freezer.Set(dir, m.Cgroups)
	if err != nil {
		m.Cgroups.Freezer = prevState
		return err
	}

	return nil
}

func (m *Manager) GetPids() ([]int, error) {
	d, err := getCgroupData(m.Cgroups, 0)
	if err != nil {
		return nil, err
	}

	dir, err := d.path("devices")
	if err != nil {
		return nil, err
	}
	if !cgroups.PathExists(dir) {
		return nil, cgroups.NewNotFoundError("devices")
	}

	return cgroups.ReadProcsFile(dir)
}

func getCgroupData(c *configs.Cgroup, pid int) (*data, error) {
	root, err := getCgroupRoot()
	if err != nil {
		return nil, err
	}

	cgroup := c.Name
	if c.Parent != "" {
		cgroup = filepath.Join(c.Parent, cgroup)
	}

	return &data{
		root:   root,
		cgroup: cgroup,
		c:      c,
		pid:    pid,
	}, nil
}

func (raw *data) parent(subsystem string) (string, error) {
	initPath, err := cgroups.GetInitCgroupDir(subsystem)
	if err != nil {
		return "", err
	}
	return filepath.Join(raw.root, subsystem, initPath), nil
}

func (raw *data) path(subsystem string) (string, error) {
	// If the cgroup name/path is absolute do not look relative to the cgroup of the init process.
	if filepath.IsAbs(raw.cgroup) {
		return filepath.Join(raw.root, subsystem, raw.cgroup), nil
	}

	parent, err := raw.parent(subsystem)
	if err != nil {
		return "", err
	}

	return filepath.Join(parent, raw.cgroup), nil
}

func (raw *data) join(subsystem string) (string, error) {
	path, err := raw.path(subsystem)
	if err != nil {
		return "", err
	}
	if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) {
		return "", err
	}
	if err := writeFile(path, CgroupProcesses, strconv.Itoa(raw.pid)); err != nil {
		return "", err
	}
	return path, nil
}

func writeFile(dir, file, data string) error {
	return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700)
}

func readFile(dir, file string) (string, error) {
	data, err := ioutil.ReadFile(filepath.Join(dir, file))
	return string(data), err
}

func removePath(p string, err error) error {
	if err != nil {
		return err
	}
	if p != "" {
		return os.RemoveAll(p)
	}
	return nil
}
<commit_msg>Add back check for mountpoint to return consistent error<commit_after>package fs

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"sync"

	"github.com/docker/libcontainer/cgroups"
	"github.com/docker/libcontainer/configs"
)

var (
	subsystems = map[string]subsystem{
		"devices":    &DevicesGroup{},
		"memory":     &MemoryGroup{},
		"cpu":        &CpuGroup{},
		"cpuset":     &CpusetGroup{},
		"cpuacct":    &CpuacctGroup{},
		"blkio":      &BlkioGroup{},
		"perf_event": &PerfEventGroup{},
		"freezer":    &FreezerGroup{},
	}
	CgroupProcesses = "cgroup.procs"
)

type subsystem interface {
	// Returns the stats, as 'stats', corresponding to the cgroup under 'path'.
	GetStats(path string, stats *cgroups.Stats) error
	// Removes the cgroup represented by 'data'.
	Remove(*data) error
	// Creates and joins the cgroup represented by data.
	Apply(*data) error
	// Set the cgroup represented by cgroup.
	Set(path string, cgroup *configs.Cgroup) error
}

type Manager struct {
	Cgroups *configs.Cgroup
	Paths   map[string]string
}

// The absolute path to the root of the cgroup hierarchies.
var cgroupRootLock sync.Mutex
var cgroupRoot string

// Gets the cgroupRoot.
func getCgroupRoot() (string, error) {
	cgroupRootLock.Lock()
	defer cgroupRootLock.Unlock()

	if cgroupRoot != "" {
		return cgroupRoot, nil
	}

	root, err := cgroups.FindCgroupMountpointDir()
	if err != nil {
		return "", err
	}

	if _, err := os.Stat(root); err != nil {
		return "", err
	}

	cgroupRoot = root
	return cgroupRoot, nil
}

type data struct {
	root   string
	cgroup string
	c      *configs.Cgroup
	pid    int
}

func (m *Manager) Apply(pid int) error {
	if m.Cgroups == nil {
		return nil
	}

	d, err := getCgroupData(m.Cgroups, pid)
	if err != nil {
		return err
	}

	paths := make(map[string]string)
	defer func() {
		if err != nil {
			cgroups.RemovePaths(paths)
		}
	}()
	for name, sys := range subsystems {
		if err := sys.Apply(d); err != nil {
			return err
		}
		// TODO: Apply should, ideally, be reentrant or be broken up into a separate
		// create and join phase so that the cgroup hierarchy for a container can be
		// created then join consists of writing the process pids to cgroup.procs
		p, err := d.path(name)
		if err != nil {
			if cgroups.IsNotFound(err) {
				continue
			}
			return err
		}
		paths[name] = p
	}
	m.Paths = paths

	return nil
}

func (m *Manager) Destroy() error {
	return cgroups.RemovePaths(m.Paths)
}

func (m *Manager) GetPaths() map[string]string {
	return m.Paths
}

// Symmetrical public function to update device based cgroups.  Also available
// in the systemd implementation.
func ApplyDevices(c *configs.Cgroup, pid int) error {
	d, err := getCgroupData(c, pid)
	if err != nil {
		return err
	}

	devices := subsystems["devices"]

	return devices.Apply(d)
}

func (m *Manager) GetStats() (*cgroups.Stats, error) {
	stats := cgroups.NewStats()
	for name, path := range m.Paths {
		sys, ok := subsystems[name]
		if !ok || !cgroups.PathExists(path) {
			continue
		}
		if err := sys.GetStats(path, stats); err != nil {
			return nil, err
		}
	}

	return stats, nil
}

func (m *Manager) Set(container *configs.Config) error {
	for name, path := range m.Paths {
		sys, ok := subsystems[name]
		if !ok || !cgroups.PathExists(path) {
			continue
		}
		if err := sys.Set(path, container.Cgroups); err != nil {
			return err
		}
	}

	return nil
}

// Freeze toggles the container's freezer cgroup depending on the state
// provided
func (m *Manager) Freeze(state configs.FreezerState) error {
	d, err := getCgroupData(m.Cgroups, 0)
	if err != nil {
		return err
	}

	dir, err := d.path("freezer")
	if err != nil {
		return err
	}
	if !cgroups.PathExists(dir) {
		return cgroups.NewNotFoundError("freezer")
	}

	prevState := m.Cgroups.Freezer
	m.Cgroups.Freezer = state

	freezer := subsystems["freezer"]
	err = freezer.Set(dir, m.Cgroups)
	if err != nil {
		m.Cgroups.Freezer = prevState
		return err
	}

	return nil
}

func (m *Manager) GetPids() ([]int, error) {
	d, err := getCgroupData(m.Cgroups, 0)
	if err != nil {
		return nil, err
	}

	dir, err := d.path("devices")
	if err != nil {
		return nil, err
	}
	if !cgroups.PathExists(dir) {
		return nil, cgroups.NewNotFoundError("devices")
	}

	return cgroups.ReadProcsFile(dir)
}

func getCgroupData(c *configs.Cgroup, pid int) (*data, error) {
	root, err := getCgroupRoot()
	if err != nil {
		return nil, err
	}

	cgroup := c.Name
	if c.Parent != "" {
		cgroup = filepath.Join(c.Parent, cgroup)
	}

	return &data{
		root:   root,
		cgroup: cgroup,
		c:      c,
		pid:    pid,
	}, nil
}

func (raw *data) parent(subsystem string) (string, error) {
	initPath, err := cgroups.GetInitCgroupDir(subsystem)
	if err != nil {
		return "", err
	}
	return filepath.Join(raw.root, subsystem, initPath), nil
}

func (raw *data) path(subsystem string) (string, error) {
	_, err := cgroups.FindCgroupMountpoint(subsystem)
	// If we didn't mount the subsystem, there is no point we make the path.
	if err != nil {
		return "", err
	}

	// If the cgroup name/path is absolute do not look relative to the cgroup of the init process.
	if filepath.IsAbs(raw.cgroup) {
		return filepath.Join(raw.root, subsystem, raw.cgroup), nil
	}

	parent, err := raw.parent(subsystem)
	if err != nil {
		return "", err
	}

	return filepath.Join(parent, raw.cgroup), nil
}

func (raw *data) join(subsystem string) (string, error) {
	path, err := raw.path(subsystem)
	if err != nil {
		return "", err
	}
	if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) {
		return "", err
	}
	if err := writeFile(path, CgroupProcesses, strconv.Itoa(raw.pid)); err != nil {
		return "", err
	}
	return path, nil
}

func writeFile(dir, file, data string) error {
	return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700)
}

func readFile(dir, file string) (string, error) {
	data, err := ioutil.ReadFile(filepath.Join(dir, file))
	return string(data), err
}

func removePath(p string, err error) error {
	if err != nil {
		return err
	}
	if p != "" {
		return os.RemoveAll(p)
	}
	return nil
}
<|endoftext|>