<commit_before>// Copyright ©2015 The corridor Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"runtime"
	"time"

	"github.com/ericdfournier/corridor"
)

func main() {
	///////////////////////////////////////////////////////////////////////////////////

	// set max processing units
	cpuCount := runtime.NumCPU()
	runtime.GOMAXPROCS(cpuCount)

	///////////////////////////////////////////////////////////////////////////////////

	// start clock
	start := time.Now()

	///////////////////////////////////////////////////////////////////////////////////

	// import source subscripts
	source := corridor.CsvToSubs("sourceSubs.csv")

	// import destination subscripts
	destination := corridor.CsvToSubs("destinationSubs.csv")

	///////////////////////////////////////////////////////////////////////////////////

	// import domain
	searchDomain := corridor.CsvToDomain("searchDomain.csv")

	///////////////////////////////////////////////////////////////////////////////////

	// initialize objectives
	searchObjectives := corridor.CsvToMultiObjective(
		"accessibility.csv",
		"slope.csv",
		"disturbance.csv")

	///////////////////////////////////////////////////////////////////////////////////

	// initialize parameters
	populationSize := 1000
	evolutionSize := 1000
	randomness := 1.0

	// generate parameter structure
	searchParameters := corridor.NewParameters(
		source,
		destination,
		populationSize,
		evolutionSize,
		randomness)

	//////////////////////////////////////////////////////////////////////////////////

	// evolve populations
	searchEvolution := corridor.NewEvolution(
		searchParameters,
		searchDomain,
		searchObjectives)

	///////////////////////////////////////////////////////////////////////////////////

	// initialize elite count
	eliteCount := 100

	// extract elite set
	eliteSet := corridor.NewEliteSet(eliteCount, <-searchEvolution.Populations)

	///////////////////////////////////////////////////////////////////////////////////

	// write elite set to file
	corridor.EliteSetToCsv(eliteSet, "santaBarbara_p-1000_e-1000_eliteSet.csv")

	///////////////////////////////////////////////////////////////////////////////////

	// stop clock and print runtime
	fmt.Printf("Elapsed Time: %s\n", time.Since(start))

	///////////////////////////////////////////////////////////////////////////////////
}
<commit_msg>Modified SB randomness coefficient to avoid and infinite loop condition on the large run. More investigation needed.<commit_after>// Copyright ©2015 The corridor Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"runtime"
	"time"

	"github.com/ericdfournier/corridor"
)

func main() {
	///////////////////////////////////////////////////////////////////////////////////

	// set max processing units
	cpuCount := runtime.NumCPU()
	runtime.GOMAXPROCS(cpuCount)

	///////////////////////////////////////////////////////////////////////////////////

	// start clock
	start := time.Now()

	///////////////////////////////////////////////////////////////////////////////////

	// import source subscripts
	source := corridor.CsvToSubs("sourceSubs.csv")

	// import destination subscripts
	destination := corridor.CsvToSubs("destinationSubs.csv")

	///////////////////////////////////////////////////////////////////////////////////

	// import domain
	searchDomain := corridor.CsvToDomain("blankDomain.csv")

	///////////////////////////////////////////////////////////////////////////////////

	// initialize objectives
	searchObjectives := corridor.CsvToMultiObjective(
		"accessibility.csv",
		"slope.csv",
		"disturbance.csv")

	///////////////////////////////////////////////////////////////////////////////////

	// initialize parameters
	populationSize := 10000
	evolutionSize := 1000
	randomness := 2.0

	// generate parameter structure
	searchParameters := corridor.NewParameters(
		source,
		destination,
		populationSize,
		evolutionSize,
		randomness)

	//////////////////////////////////////////////////////////////////////////////////

	// evolve populations
	searchEvolution := corridor.NewEvolution(
		searchParameters,
		searchDomain,
		searchObjectives)

	///////////////////////////////////////////////////////////////////////////////////

	// initialize elite count
	eliteCount := 100

	// extract elite set
	eliteSet := corridor.NewEliteSet(eliteCount, <-searchEvolution.Populations)

	///////////////////////////////////////////////////////////////////////////////////

	// write elite set to file
	corridor.EliteSetToCsv(eliteSet, "santaBarbara_p-1000_e-1000_eliteSet.csv")

	///////////////////////////////////////////////////////////////////////////////////

	// stop clock and print runtime
	fmt.Printf("Elapsed Time: %s\n", time.Since(start))

	///////////////////////////////////////////////////////////////////////////////////
}
<|endoftext|>


<commit_before>package containerd

import (
	"encoding/json"
	"os"
	"path/filepath"

	"github.com/Sirupsen/logrus"
)

type entry struct {
	Event *Event `json:"event"`
}

func newJournal(path string) (*journal, error) {
	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
		return nil, err
	}
	f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0755)
	if err != nil {
		return nil, err
	}
	j := &journal{
		f:   f,
		enc: json.NewEncoder(f),
		wc:  make(chan *Event, 2048),
	}
	go j.start()
	return j, nil
}

type journal struct {
	f   *os.File
	enc *json.Encoder
	wc  chan *Event
}

func (j *journal) start() {
	for e := range j.wc {
		et := &entry{
			Event: e,
		}
		if err := j.enc.Encode(et); err != nil {
			logrus.WithField("error", err).Error("write event to journal")
		}
	}
}

func (j *journal) write(e *Event) {
	j.wc <- e
}

func (j *journal) Close() error {
	// TODO: add waitgroup to make sure journal is flushed
	close(j.wc)
	return j.f.Close()
}
<commit_msg>add waitgroup to journal<commit_after>package containerd

import (
	"encoding/json"
	"os"
	"path/filepath"
	"sync"

	"github.com/Sirupsen/logrus"
)

type entry struct {
	Event *Event `json:"event"`
}

func newJournal(path string) (*journal, error) {
	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
		return nil, err
	}
	f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0755)
	if err != nil {
		return nil, err
	}
	j := &journal{
		f:   f,
		enc: json.NewEncoder(f),
		wc:  make(chan *Event, 2048),
	}
	j.wg.Add(1)
	go j.start()
	return j, nil
}

type journal struct {
	f   *os.File
	enc *json.Encoder
	wc  chan *Event
	wg  sync.WaitGroup
}

func (j *journal) start() {
	defer j.wg.Done()
	for e := range j.wc {
		et := &entry{
			Event: e,
		}
		if err := j.enc.Encode(et); err != nil {
			logrus.WithField("error", err).Error("write event to journal")
		}
	}
}

func (j *journal) write(e *Event) {
	j.wc <- e
}

func (j *journal) Close() error {
	close(j.wc)
	j.wg.Wait()
	return j.f.Close()
}
<|endoftext|>


<commit_before>// NOT READY

// Need to update this to final code.
<commit_msg>Filled out keypair.go<commit_after>// Every file in go is a part of some package. Since we want to create a program
// from this file we use 'package main'
package main

import (
	"encoding/hex"
	"fmt"
	"log"

	"github.com/btcsuite/btcec"
	"github.com/btcsuite/btcnet"
	"github.com/btcsuite/btcutil"
)

// generateKeyPair creates a key pair based on the eliptic curve
// secp256k1. The key pair returned by this function is two points
// on this curve. Bitcoin requires that the public and private keys
// used to generate signatures are generated using this curve.

func generateKeyPair() (*btcec.PublicKey, *btcec.PrivateKey) {

	// Generate a private key, use the curve secpc256k1 and kill the program on
	// any errors
	priv, err := btcec.NewPrivateKey(btcec.S256())
	if err != nil {
		// There was an error. Log it and bail out
		log.Fatal(err)
	}

	return priv.PubKey(), priv
}

// generateAddr computes the associated bitcon address from the provided
// public key. We compute ripemd160(sha256(b)) of the pubkey and then
// shimmy the hashed bytes into btcsuite's AddressPubKeyHash type
func generateAddr(pub *btcec.PublicKey) *btcutil.AddressPubKeyHash {

	net := &btcnet.MainNetParams

	// Serialize the public key into bytes and then run ripemd160(sha256(b)) on it
	b := btcutil.Hash160(pub.SerializeCompressed())

	// Convert the hashed public key into the btcsuite type so that the library
	// will handle the base58 encoding when we call addr.String()
	addr, err := btcutil.NewAddressPubKeyHash(b, net)
	if err != nil {
		log.Fatal(err)
	}

	return addr
}

func main() {

	// In order to recieve coins we must generate a public/private key pair.
	pub, priv := generateKeyPair()

	// To use this address we must store our private key somewhere. Everything
	// else can be recovered from the private key.
	fmt.Printf("This is a private key in hex:\t[%s]\n",
		hex.EncodeToString(priv.Serialize()))

	fmt.Printf("This is a public key in hex:\t[%s]\n",
		hex.EncodeToString(pub.SerializeCompressed()))

	addr := generateAddr(pub)

	// Output the bitcoin address derived from the public key
	fmt.Printf("This is the associated Bitcoin address:\t[%s]\n", addr.String())
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package gce_test

import (
	"github.com/juju/errors"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/constraints"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/provider/gce"
	"github.com/juju/juju/provider/gce/google"
)

type environInstSuite struct {
	gce.BaseSuite
}

var _ = gc.Suite(&environInstSuite{})

func (s *environInstSuite) TestInstances(c *gc.C) {
	s.FakeEnviron.Insts = []instance.Instance{s.Instance}

	ids := []instance.Id{"spam"}
	insts, err := s.Env.Instances(ids)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(insts, jc.DeepEquals, []instance.Instance{s.Instance})
}

func (s *environInstSuite) TestInstancesEmptyArg(c *gc.C) {
	_, err := s.Env.Instances(nil)

	c.Check(err, gc.Equals, environs.ErrNoInstances)
}

func (s *environInstSuite) TestInstancesInstancesFailed(c *gc.C) {
	failure := errors.New("<unknown>")
	s.FakeEnviron.Err = failure

	ids := []instance.Id{"spam"}
	insts, err := s.Env.Instances(ids)

	c.Check(insts, jc.DeepEquals, []instance.Instance{nil})
	c.Check(errors.Cause(err), gc.Equals, failure)
}

func (s *environInstSuite) TestInstancesPartialMatch(c *gc.C) {
	s.FakeEnviron.Insts = []instance.Instance{s.Instance}

	ids := []instance.Id{"spam", "eggs"}
	insts, err := s.Env.Instances(ids)

	c.Check(insts, jc.DeepEquals, []instance.Instance{s.Instance, nil})
	c.Check(errors.Cause(err), gc.Equals, environs.ErrPartialInstances)
}

func (s *environInstSuite) TestInstancesNoMatch(c *gc.C) {
	s.FakeEnviron.Insts = []instance.Instance{s.Instance}

	ids := []instance.Id{"eggs"}
	insts, err := s.Env.Instances(ids)

	c.Check(insts, jc.DeepEquals, []instance.Instance{nil})
	c.Check(errors.Cause(err), gc.Equals, environs.ErrNoInstances)
}

func (s *environInstSuite) TestBasicInstances(c *gc.C) {
	s.FakeConn.Insts = []google.Instance{*s.BaseInstance}

	insts, err := gce.GetInstances(s.Env)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(insts, jc.DeepEquals, []instance.Instance{s.Instance})
}

func (s *environInstSuite) TestBasicInstancesAPI(c *gc.C) {
	s.FakeConn.Insts = []google.Instance{*s.BaseInstance}

	_, err := gce.GetInstances(s.Env)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(s.FakeConn.Calls, gc.HasLen, 1)
	c.Check(s.FakeConn.Calls[0].FuncName, gc.Equals, "Instances")
	c.Check(s.FakeConn.Calls[0].Prefix, gc.Equals, s.Prefix+"machine-")
	c.Check(s.FakeConn.Calls[0].Statuses, jc.DeepEquals, []string{google.StatusPending, google.StatusStaging, google.StatusRunning})
}

func (s *environInstSuite) TestStateServerInstances(c *gc.C) {
	s.FakeConn.Insts = []google.Instance{*s.BaseInstance}

	ids, err := s.Env.StateServerInstances()
	c.Assert(err, jc.ErrorIsNil)

	c.Check(ids, jc.DeepEquals, []instance.Id{"spam"})
}

func (s *environInstSuite) TestStateServerInstancesAPI(c *gc.C) {
	s.FakeConn.Insts = []google.Instance{*s.BaseInstance}

	_, err := s.Env.StateServerInstances()
	c.Assert(err, jc.ErrorIsNil)

	c.Check(s.FakeConn.Calls, gc.HasLen, 1)
	c.Check(s.FakeConn.Calls[0].FuncName, gc.Equals, "Instances")
	c.Check(s.FakeConn.Calls[0].Prefix, gc.Equals, s.Prefix+"machine-")
	c.Check(s.FakeConn.Calls[0].Statuses, jc.DeepEquals, []string{google.StatusPending, google.StatusStaging, google.StatusRunning})
}

func (s *environInstSuite) TestStateServerInstancesNotBootstrapped(c *gc.C) {
	_, err := s.Env.StateServerInstances()

	c.Check(err, gc.Equals, environs.ErrNotBootstrapped)
}

func (s *environInstSuite) TestStateServerInstancesMixed(c *gc.C) {
	other := google.NewInstance(google.InstanceSummary{}, nil)
	s.FakeConn.Insts = []google.Instance{*s.BaseInstance, *other}

	ids, err := s.Env.StateServerInstances()
	c.Assert(err, jc.ErrorIsNil)

	c.Check(ids, jc.DeepEquals, []instance.Id{"spam"})
}

func (s *environInstSuite) TestParsePlacement(c *gc.C) {
	zone := google.NewZone("a-zone", google.StatusUp)
	s.FakeConn.Zones = []google.AvailabilityZone{zone}

	placement, err := gce.ParsePlacement(s.Env, "zone=a-zone")
	c.Assert(err, jc.ErrorIsNil)

	c.Check(placement.Zone, jc.DeepEquals, &zone)
}

func (s *environInstSuite) TestParsePlacementZoneFailure(c *gc.C) {
	failure := errors.New("<unknown>")
	s.FakeConn.Err = failure

	_, err := gce.ParsePlacement(s.Env, "zone=a-zone")

	c.Check(errors.Cause(err), gc.Equals, failure)
}

func (s *environInstSuite) TestParsePlacementMissingDirective(c *gc.C) {
	_, err := gce.ParsePlacement(s.Env, "a-zone")

	c.Check(err, gc.ErrorMatches, `.*unknown placement directive: .*`)
}

func (s *environInstSuite) TestParsePlacementUnknownDirective(c *gc.C) {
	_, err := gce.ParsePlacement(s.Env, "inst=spam")

	c.Check(err, gc.ErrorMatches, `.*unknown placement directive: .*`)
}

func (s *environInstSuite) TestCheckInstanceType(c *gc.C) {
	typ := "n1-standard-1"
	cons := constraints.Value{
		InstanceType: &typ,
	}
	matched := gce.CheckInstanceType(cons)

	c.Check(matched, jc.IsTrue)
}

func (s *environInstSuite) TestCheckInstanceTypeUnknown(c *gc.C) {
	typ := "n1-standard-1.unknown"
	cons := constraints.Value{
		InstanceType: &typ,
	}
	matched := gce.CheckInstanceType(cons)

	c.Check(matched, jc.IsFalse)
}
<commit_msg>Test with multiple instances.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package gce_test

import (
	"github.com/juju/errors"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/constraints"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/provider/gce"
	"github.com/juju/juju/provider/gce/google"
)

type environInstSuite struct {
	gce.BaseSuite
}

var _ = gc.Suite(&environInstSuite{})

func (s *environInstSuite) TestInstances(c *gc.C) {
	spam := s.NewInstance(c, "spam")
	ham := s.NewInstance(c, "ham")
	eggs := s.NewInstance(c, "eggs")
	s.FakeEnviron.Insts = []instance.Instance{spam, ham, eggs}

	ids := []instance.Id{"spam", "eggs", "ham"}
	insts, err := s.Env.Instances(ids)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(insts, jc.DeepEquals, []instance.Instance{spam, eggs, ham})
}

func (s *environInstSuite) TestInstancesEmptyArg(c *gc.C) {
	_, err := s.Env.Instances(nil)

	c.Check(err, gc.Equals, environs.ErrNoInstances)
}

func (s *environInstSuite) TestInstancesInstancesFailed(c *gc.C) {
	failure := errors.New("<unknown>")
	s.FakeEnviron.Err = failure

	ids := []instance.Id{"spam"}
	insts, err := s.Env.Instances(ids)

	c.Check(insts, jc.DeepEquals, []instance.Instance{nil})
	c.Check(errors.Cause(err), gc.Equals, failure)
}

func (s *environInstSuite) TestInstancesPartialMatch(c *gc.C) {
	s.FakeEnviron.Insts = []instance.Instance{s.Instance}

	ids := []instance.Id{"spam", "eggs"}
	insts, err := s.Env.Instances(ids)

	c.Check(insts, jc.DeepEquals, []instance.Instance{s.Instance, nil})
	c.Check(errors.Cause(err), gc.Equals, environs.ErrPartialInstances)
}

func (s *environInstSuite) TestInstancesNoMatch(c *gc.C) {
	s.FakeEnviron.Insts = []instance.Instance{s.Instance}

	ids := []instance.Id{"eggs"}
	insts, err := s.Env.Instances(ids)

	c.Check(insts, jc.DeepEquals, []instance.Instance{nil})
	c.Check(errors.Cause(err), gc.Equals, environs.ErrNoInstances)
}

func (s *environInstSuite) TestBasicInstances(c *gc.C) {
	spam := s.NewBaseInstance(c, "spam")
	ham := s.NewBaseInstance(c, "ham")
	eggs := s.NewBaseInstance(c, "eggs")
	s.FakeConn.Insts = []google.Instance{*spam, *ham, *eggs}

	insts, err := gce.GetInstances(s.Env)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(insts, jc.DeepEquals, []instance.Instance{
		s.NewInstance(c, "spam"),
		s.NewInstance(c, "ham"),
		s.NewInstance(c, "eggs"),
	})
}

func (s *environInstSuite) TestBasicInstancesAPI(c *gc.C) {
	s.FakeConn.Insts = []google.Instance{*s.BaseInstance}

	_, err := gce.GetInstances(s.Env)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(s.FakeConn.Calls, gc.HasLen, 1)
	c.Check(s.FakeConn.Calls[0].FuncName, gc.Equals, "Instances")
	c.Check(s.FakeConn.Calls[0].Prefix, gc.Equals, s.Prefix+"machine-")
	c.Check(s.FakeConn.Calls[0].Statuses, jc.DeepEquals, []string{google.StatusPending, google.StatusStaging, google.StatusRunning})
}

func (s *environInstSuite) TestStateServerInstances(c *gc.C) {
	s.FakeConn.Insts = []google.Instance{*s.BaseInstance}

	ids, err := s.Env.StateServerInstances()
	c.Assert(err, jc.ErrorIsNil)

	c.Check(ids, jc.DeepEquals, []instance.Id{"spam"})
}

func (s *environInstSuite) TestStateServerInstancesAPI(c *gc.C) {
	s.FakeConn.Insts = []google.Instance{*s.BaseInstance}

	_, err := s.Env.StateServerInstances()
	c.Assert(err, jc.ErrorIsNil)

	c.Check(s.FakeConn.Calls, gc.HasLen, 1)
	c.Check(s.FakeConn.Calls[0].FuncName, gc.Equals, "Instances")
	c.Check(s.FakeConn.Calls[0].Prefix, gc.Equals, s.Prefix+"machine-")
	c.Check(s.FakeConn.Calls[0].Statuses, jc.DeepEquals, []string{google.StatusPending, google.StatusStaging, google.StatusRunning})
}

func (s *environInstSuite) TestStateServerInstancesNotBootstrapped(c *gc.C) {
	_, err := s.Env.StateServerInstances()

	c.Check(err, gc.Equals, environs.ErrNotBootstrapped)
}

func (s *environInstSuite) TestStateServerInstancesMixed(c *gc.C) {
	other := google.NewInstance(google.InstanceSummary{}, nil)
	s.FakeConn.Insts = []google.Instance{*s.BaseInstance, *other}

	ids, err := s.Env.StateServerInstances()
	c.Assert(err, jc.ErrorIsNil)

	c.Check(ids, jc.DeepEquals, []instance.Id{"spam"})
}

func (s *environInstSuite) TestParsePlacement(c *gc.C) {
	zone := google.NewZone("a-zone", google.StatusUp)
	s.FakeConn.Zones = []google.AvailabilityZone{zone}

	placement, err := gce.ParsePlacement(s.Env, "zone=a-zone")
	c.Assert(err, jc.ErrorIsNil)

	c.Check(placement.Zone, jc.DeepEquals, &zone)
}

func (s *environInstSuite) TestParsePlacementZoneFailure(c *gc.C) {
	failure := errors.New("<unknown>")
	s.FakeConn.Err = failure

	_, err := gce.ParsePlacement(s.Env, "zone=a-zone")

	c.Check(errors.Cause(err), gc.Equals, failure)
}

func (s *environInstSuite) TestParsePlacementMissingDirective(c *gc.C) {
	_, err := gce.ParsePlacement(s.Env, "a-zone")

	c.Check(err, gc.ErrorMatches, `.*unknown placement directive: .*`)
}

func (s *environInstSuite) TestParsePlacementUnknownDirective(c *gc.C) {
	_, err := gce.ParsePlacement(s.Env, "inst=spam")

	c.Check(err, gc.ErrorMatches, `.*unknown placement directive: .*`)
}

func (s *environInstSuite) TestCheckInstanceType(c *gc.C) {
	typ := "n1-standard-1"
	cons := constraints.Value{
		InstanceType: &typ,
	}
	matched := gce.CheckInstanceType(cons)

	c.Check(matched, jc.IsTrue)
}

func (s *environInstSuite) TestCheckInstanceTypeUnknown(c *gc.C) {
	typ := "n1-standard-1.unknown"
	cons := constraints.Value{
		InstanceType: &typ,
	}
	matched := gce.CheckInstanceType(cons)

	c.Check(matched, jc.IsFalse)
}
<|endoftext|>


<commit_before>// Copyright 2016 The fer Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mq_test

import (
	"fmt"
	"sync"
	"testing"

	"github.com/sbinet-alice/fer/mq"
	_ "github.com/sbinet-alice/fer/mq/nanomsg"
	_ "github.com/sbinet-alice/fer/mq/zeromq"
)

func TestOpen(t *testing.T) {
	_, err := mq.Open("no-such-driver")
	if err == nil {
		t.Fatalf("expected a no such-driver error")
	}

	drv1, err := mq.Open("nanomsg")
	if err != nil {
		t.Fatal(err)
	}

	drv2, err := mq.Open("nanomsg")
	if err != nil {
		t.Fatal(err)
	}

	if drv1 != drv2 {
		t.Fatalf("Open is not idem-potent")
	}
}

func TestPushPullNN(t *testing.T) {
	const (
		N    = 5
		tmpl = "data-%02d"
		port = "6666"
	)

	drv, err := mq.Open("nanomsg")
	if err != nil {
		t.Fatal(err)
	}
	pull, err := drv.NewSocket(mq.Pull)
	if err != nil {
		t.Fatal(err)
	}
	defer pull.Close()

	push, err := drv.NewSocket(mq.Push)
	if err != nil {
		t.Fatal(err)
	}
	defer push.Close()

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		err := push.Dial("tcp://localhost:" + port)
		if err != nil {
			t.Fatal(err)
		}
		for i := 0; i < N; i++ {
			err = push.Send([]byte(fmt.Sprintf(tmpl, i)))
			if err != nil {
				t.Fatalf("error sending data[%d]: %v\n", i, err)
			}
		}
		err = push.Close()
		if err != nil {
			t.Fatal(err)
		}
		wg.Done()
	}()

	err = pull.Listen("tcp://*:" + port)
	if err != nil {
		t.Fatal(err)
	}
	for i := 0; i < N; i++ {
		msg, err := pull.Recv()
		if err != nil {
			t.Fatal(err)
		}
		if got, want := string(msg), fmt.Sprintf(tmpl, i); got != want {
			t.Errorf("push-pull[%d]: got=%q want=%q\n", i, got, want)
		}
	}
	err = pull.Close()
	if err != nil {
		t.Fatal(err)
	}
	wg.Wait()
}

func TestPushPullZMQ(t *testing.T) {
	const (
		N    = 5
		tmpl = "data-%02d"
		port = "5555"
	)

	drv, err := mq.Open("zeromq")
	if err != nil {
		t.Fatal(err)
	}
	pull, err := drv.NewSocket(mq.Pull)
	if err != nil {
		t.Fatal(err)
	}
	defer pull.Close()

	push, err := drv.NewSocket(mq.Push)
	if err != nil {
		t.Fatal(err)
	}
	defer push.Close()

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		err := push.Dial("tcp://localhost:" + port)
		if err != nil {
			t.Fatal(err)
		}
		for i := 0; i < N; i++ {
			err = push.Send([]byte(fmt.Sprintf(tmpl, i)))
			if err != nil {
				t.Fatalf("error sending data[%d]: %v\n", i, err)
			}
		}
		err = push.Close()
		if err != nil {
			t.Fatal(err)
		}
		wg.Done()
	}()

	err = pull.Listen("tcp://*:" + port)
	if err != nil {
		t.Fatal(err)
	}
	for i := 0; i < N; i++ {
		msg, err := pull.Recv()
		if err != nil {
			t.Fatal(err)
		}
		if got, want := string(msg), fmt.Sprintf(tmpl, i); got != want {
			t.Errorf("push-pull[%d]: got=%q want=%q\n", i, got, want)
		}
	}
	err = pull.Close()
	if err != nil {
		t.Fatal(err)
	}
	wg.Wait()
}
<commit_msg>mq: use sub-tests<commit_after>// Copyright 2016 The fer Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mq_test

import (
	"fmt"
	"net"
	"strconv"
	"sync"
	"testing"

	"github.com/sbinet-alice/fer/mq"
	_ "github.com/sbinet-alice/fer/mq/nanomsg"
	_ "github.com/sbinet-alice/fer/mq/zeromq"
)

func TestOpen(t *testing.T) {
	_, err := mq.Open("no-such-driver")
	if err == nil {
		t.Fatalf("expected a no such-driver error")
	}

	drv1, err := mq.Open("nanomsg")
	if err != nil {
		t.Fatal(err)
	}

	drv2, err := mq.Open("nanomsg")
	if err != nil {
		t.Fatal(err)
	}

	if drv1 != drv2 {
		t.Fatalf("Open is not idem-potent")
	}
}

var drivers = []string{"zeromq", "nanomsg"}

func getTCPPort() (string, error) {
	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
	if err != nil {
		return "", err
	}
	l, err := net.ListenTCP("tcp", addr)
	if err != nil {
		return "", err
	}
	defer l.Close()
	return strconv.Itoa(l.Addr().(*net.TCPAddr).Port), nil
}

func TestPushPull(t *testing.T) {
	for i := range drivers {
		transport := drivers[i]
		t.Run("transport="+transport, func(t *testing.T) {

			const (
				N    = 5
				tmpl = "data-%02d"
			)

			port, err := getTCPPort()
			if err != nil {
				t.Fatalf("error getting free TCP port: %v\n", err)
			}

			drv, err := mq.Open(transport)
			if err != nil {
				t.Fatal(err)
			}
			pull, err := drv.NewSocket(mq.Pull)
			if err != nil {
				t.Fatal(err)
			}
			defer pull.Close()

			push, err := drv.NewSocket(mq.Push)
			if err != nil {
				t.Fatal(err)
			}
			defer push.Close()

			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				err := push.Dial("tcp://localhost:" + port)
				if err != nil {
					t.Fatal(err)
				}
				for i := 0; i < N; i++ {
					err = push.Send([]byte(fmt.Sprintf(tmpl, i)))
					if err != nil {
						t.Fatalf("error sending data[%d]: %v\n", i, err)
					}
				}
				err = push.Close()
				if err != nil {
					t.Fatal(err)
				}
				wg.Done()
			}()

			err = pull.Listen("tcp://*:" + port)
			if err != nil {
				t.Fatal(err)
			}
			for i := 0; i < N; i++ {
				msg, err := pull.Recv()
				if err != nil {
					t.Fatal(err)
				}
				if got, want := string(msg), fmt.Sprintf(tmpl, i); got != want {
					t.Errorf("push-pull[%d]: got=%q want=%q\n", i, got, want)
				}
			}
			err = pull.Close()
			if err != nil {
				t.Fatal(err)
			}
			wg.Wait()
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"regexp"
	"strings"

	"code.google.com/p/go.text/encoding/japanese"
	"code.google.com/p/go.text/transform"
)

var ImgUrlRegexp *regexp.Regexp = regexp.MustCompile("h?ttp://[0-9a-zA-Z/\\-.%]+?\\.(jpg|jpeg|gif|png)")

func Dat(url string) {
	resp, err := http.Get(url)
	if err != nil {
		log.Fatalf("failed to fetch: %v, err: %v\n", url, err)
		return
	}
	defer resp.Body.Close()

	reader := transform.NewReader(resp.Body, japanese.ShiftJIS.NewDecoder())
	bufr := bufio.NewReader(reader)
	for {
		lineb, err := bufr.ReadString('\n')
		if err == io.EOF {
			return
		}
		if err != nil {
			log.Fatalf("failed to read content. err: %v\n", err)
			return
		}
		line := strings.Split(string(lineb), "<>")
		if len(line) >= 4 {
			body := line[3]
			matched := ImgUrlRegexp.FindAllString(body, -1)
			fmt.Println(matched)
		}
	}
}

func DatQueue(dat chan string, done chan bool) {
	for {
		url, more := <-dat
		if more && url != "" {
			Dat(url)
		} else {
			done <- true
		}
	}
}

func main() {
	dat := make(chan string)
	done := make(chan bool)

	go DatQueue(dat, done)

	if len(os.Args) > 1 {
		for i := 1; i < len(os.Args); i++ {
			dat <- os.Args[i]
		}
	} else {
		in := bufio.NewReader(os.Stdin)
		for {
			input, err := in.ReadString('\n')
			if err != nil {
				break
			}
			dat <- input
		}
	}
	close(dat)

	<-done
}
<commit_msg>download img. use log.Printf<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"code.google.com/p/go.text/encoding/japanese"
	"code.google.com/p/go.text/transform"
)

// FIXME: brash up regexp
var ImgUrlRegexp *regexp.Regexp = regexp.MustCompile("h?ttp://[0-9a-zA-Z/\\-.%]+?\\.(jpg|jpeg|gif|png)")

func Img(url string) {
	log.Printf("downloading %v...\n", url)

	out, err := os.Create(filepath.Base(url))
	if err != nil {
		log.Printf("failed to create download file: %v\n", err)
		return
	}
	defer out.Close()

	resp, err := http.Get(url)
	if err != nil {
		log.Printf("failed to download img: %v, err: \n", url, err)
		return
	}
	defer resp.Body.Close()

	n, err := io.Copy(out, resp.Body)
	if err != nil {
		log.Printf("failed to copy img: %v\n", err)
		return
	}
	log.Printf("saved %v (%v bytes)\n", url, n)
}

func ImgQueue(ch chan string, done chan bool) {
	for {
		url, more := <-ch
		if more && url != "" {
			Img(url)
		} else {
			done <- true
		}
	}
}

func Dat(url string) {
	log.Printf("reading %v...\n", url)

	resp, err := http.Get(url)
	if err != nil {
		log.Printf("failed to fetch: %v, err: %v\n", url, err)
		return
	}
	defer resp.Body.Close()

	imgCh := make(chan string)
	done := make(chan bool)
	go ImgQueue(imgCh, done)

	reader := transform.NewReader(resp.Body, japanese.ShiftJIS.NewDecoder())
	bufr := bufio.NewReader(reader)
	for {
		lineb, err := bufr.ReadString('\n')
		if err == io.EOF {
			return
		}
		if err != nil {
			log.Printf("failed to read content. err: %v\n", err)
			return
		}
		line := strings.Split(string(lineb), "<>")
		if len(line) >= 4 {
			body := line[3]
			matched := ImgUrlRegexp.FindAllString(body, -1)
			for i := 0; i < len(matched); i++ {
				imgCh <- matched[i]
			}
		}
	}
	close(imgCh)

	<-done
}

func DatQueue(ch chan string, done chan bool) {
	for {
		url, more := <-ch
		if more && url != "" {
			Dat(url)
		} else {
			done <- true
		}
	}
}

func main() {
	datCh := make(chan string)
	done := make(chan bool)

	go DatQueue(datCh, done)

	if len(os.Args) > 1 {
		for i := 1; i < len(os.Args); i++ {
			datCh <- os.Args[i]
		}
	} else {
		in := bufio.NewReader(os.Stdin)
		for {
			input, err := in.ReadString('\n')
			if err != nil {
				break
			}
			datCh <- input
		}
	}
	close(datCh)

	<-done

	fmt.Println("OK")
}
<|endoftext|>


<commit_before>package cell_test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"code.cloudfoundry.org/archiver/extractor/test_helper"
	"code.cloudfoundry.org/bbs/models"
	"code.cloudfoundry.org/inigo/fixtures"
	"code.cloudfoundry.org/inigo/helpers"
	repconfig "code.cloudfoundry.org/rep/cmd/rep/config"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
	"github.com/tedsuo/ifrit/grouper"
)

var _ = Describe("Network Environment Variables", func() {
	var (
		guid                string
		modifyRepConfig     func(*repconfig.RepConfig)
		fileServerStaticDir string
		fileServer          ifrit.Runner
		runtime             ifrit.Process
	)

	BeforeEach(func() {
		fileServer, fileServerStaticDir = componentMaker.FileServer()
		modifyRepConfig = func(*repconfig.RepConfig) {}
		guid = helpers.GenerateGuid()
	})

	JustBeforeEach(func() {
		runtime = ginkgomon.Invoke(grouper.NewParallel(os.Kill, grouper.Members{
			{"rep", componentMaker.Rep(modifyRepConfig)},
			{"auctioneer", componentMaker.Auctioneer()},
			{"router", componentMaker.Router()},
			{"route-emitter", componentMaker.RouteEmitter()},
			{"file-server", fileServer},
		}))
	})

	AfterEach(func() {
		helpers.StopProcesses(runtime)
	})

	Describe("tasks", func() {
		var task *models.Task

		JustBeforeEach(func() {
			taskToDesire := helpers.TaskCreateRequest(
				guid,
				&models.RunAction{
					User: "vcap",
					Path: "sh",
					Args: []string{"-c", "/usr/bin/env | grep 'CF_INSTANCE' > /home/vcap/env"},
				},
			)
			taskToDesire.ResultFile = "/home/vcap/env"

			err := bbsClient.DesireTask(logger, taskToDesire.TaskGuid, taskToDesire.Domain, taskToDesire.TaskDefinition)
			Expect(err).NotTo(HaveOccurred())

			Eventually(func() interface{} {
				var err error
				task, err = bbsClient.TaskByGuid(logger, guid)
				Expect(err).ShouldNot(HaveOccurred())

				return task.State
			}).Should(Equal(models.Task_Completed))
		})

		Context("when ExportNetworkEnvVars is false", func() {
			BeforeEach(func() {
				modifyRepConfig = func(config *repconfig.RepConfig) { config.ExportNetworkEnvVars = false }
			})

			It("does not set the networking environment variables", func() {
				Expect(task.Result).To(Equal(""))
			})
		})

		Context("when ExportNetworkEnvVars is true", func() {
			BeforeEach(func() {
				modifyRepConfig = func(config *repconfig.RepConfig) { config.ExportNetworkEnvVars = true }
			})

			It("sets the networking environment variables", func() {
				Expect(task.Result).To(ContainSubstring("CF_INSTANCE_ADDR=\n"))
				Expect(task.Result).To(ContainSubstring("CF_INSTANCE_PORT=\n"))
				Expect(task.Result).To(ContainSubstring("CF_INSTANCE_PORTS=[]\n"))
				Expect(task.Result).To(ContainSubstring(fmt.Sprintf("CF_INSTANCE_IP=%s\n", componentMaker.ExternalAddress)))
				Expect(task.Result).To(ContainSubstring("CF_INSTANCE_INTERNAL_IP="))
			})
		})
	})

	Describe("LRPs", func() {
		var response []byte
		var actualLRP *models.ActualLRP

		BeforeEach(func() {
			test_helper.CreateZipArchive(
				filepath.Join(fileServerStaticDir, "lrp.zip"),
				fixtures.GoServerApp(),
			)
		})

		JustBeforeEach(func() {
			lrp := helpers.DefaultLRPCreateRequest(guid, guid, 1)
			lrp.Setup = models.WrapAction(&models.DownloadAction{
				User: "vcap",
				From: fmt.Sprintf("http://%s/v1/static/%s", componentMaker.Addresses.FileServer, "lrp.zip"),
				To:   "/tmp/diego",
			})
			lrp.Action = models.WrapAction(&models.RunAction{
				User: "vcap",
				Path: "/tmp/diego/go-server",
				Env:  []*models.EnvironmentVariable{{"PORT", "8080"}},
			})

			err := bbsClient.DesireLRP(logger, lrp)
			Expect(err).NotTo(HaveOccurred())

			Eventually(helpers.LRPStatePoller(logger, bbsClient, guid, nil)).Should(Equal(models.ActualLRPStateRunning))
			Eventually(helpers.ResponseCodeFromHostPoller(componentMaker.Addresses.Router, helpers.DefaultHost)).Should(Equal(http.StatusOK))

			lrps, err := bbsClient.ActualLRPGroupsByProcessGuid(logger, guid)
			Expect(err).NotTo(HaveOccurred())

			Expect(lrps).To(HaveLen(1))
			actualLRP = lrps[0].Instance

			var status int
			response, status, err = helpers.ResponseBodyAndStatusCodeFromHost(componentMaker.Addresses.Router, helpers.DefaultHost, "env")
			Expect(status).To(Equal(http.StatusOK))
			Expect(err).NotTo(HaveOccurred())
		})

		Context("when ExportNetworkEnvVars is false", func() {
			BeforeEach(func() {
				modifyRepConfig = func(config *repconfig.RepConfig) { config.ExportNetworkEnvVars = false }
			})

			It("does not set the networking environment variables", func() {
				Expect(response).NotTo(ContainSubstring("CF_INSTANCE_IP="))
				Expect(response).NotTo(ContainSubstring("CF_INSTANCE_INTERNAL_IP="))
				Expect(response).NotTo(ContainSubstring("CF_INSTANCE_ADDR="))
				Expect(response).NotTo(ContainSubstring("CF_INSTANCE_PORT="))
				Expect(response).NotTo(ContainSubstring("CF_INSTANCE_PORTS="))
			})
		})

		Context("when ExportNetworkEnvVars is true", func() {
			BeforeEach(func() {
				modifyRepConfig = func(config *repconfig.RepConfig) { config.ExportNetworkEnvVars = true }
			})

			It("sets the networking environment variables", func() {
				netInfo := actualLRP.ActualLRPNetInfo
				Expect(response).To(ContainSubstring(fmt.Sprintf("CF_INSTANCE_ADDR=%s:%d\n", netInfo.Address, netInfo.Ports[0].HostPort)))
				Expect(response).To(ContainSubstring(fmt.Sprintf("CF_INSTANCE_IP=%s\n", componentMaker.ExternalAddress)))
				Expect(response).To(ContainSubstring("CF_INSTANCE_INTERNAL_IP="))
				Expect(response).To(ContainSubstring(fmt.Sprintf("CF_INSTANCE_PORT=%d\n", netInfo.Ports[0].HostPort)))

				type portMapping struct {
					External uint32 `json:"external"`
					Internal uint32 `json:"internal"`
				}
				ports := []portMapping{}

				buf := bytes.NewBuffer(response)
				for {
					line, err := buf.ReadString('\n')
					if err != nil {
						break
					}
					if strings.HasPrefix(line, "CF_INSTANCE_PORTS=") {
						err := json.Unmarshal([]byte(strings.TrimPrefix(line, "CF_INSTANCE_PORTS=")), &ports)
						Expect(err).NotTo(HaveOccurred())
						break
					}
				}

				Expect(ports).To(Equal([]portMapping{
					{External: netInfo.Ports[0].HostPort, Internal: netInfo.Ports[0].ContainerPort},
				}))
			})
		})
	})
})
<commit_msg>tests the instance address in the CF_INSTANCE_INTERNAL_IP<commit_after>package cell_test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"code.cloudfoundry.org/archiver/extractor/test_helper"
	"code.cloudfoundry.org/bbs/models"
	"code.cloudfoundry.org/inigo/fixtures"
	"code.cloudfoundry.org/inigo/helpers"
	repconfig "code.cloudfoundry.org/rep/cmd/rep/config"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
	"github.com/tedsuo/ifrit/grouper"
)

var _ = Describe("Network Environment Variables", func() {
	var (
		guid                string
		modifyRepConfig     func(*repconfig.RepConfig)
		fileServerStaticDir string
		fileServer          ifrit.Runner
		runtime             ifrit.Process
	)

	BeforeEach(func() {
		fileServer, fileServerStaticDir = componentMaker.FileServer()
		modifyRepConfig = func(*repconfig.RepConfig) {}
		guid = helpers.GenerateGuid()
	})

	JustBeforeEach(func() {
		runtime = ginkgomon.Invoke(grouper.NewParallel(os.Kill, grouper.Members{
			{"rep", componentMaker.Rep(modifyRepConfig)},
			{"auctioneer", componentMaker.Auctioneer()},
			{"router", componentMaker.Router()},
			{"route-emitter", componentMaker.RouteEmitter()},
			{"file-server", fileServer},
		}))
	})

	AfterEach(func() {
		helpers.StopProcesses(runtime)
	})

	Describe("tasks", func() {
		var task *models.Task

		JustBeforeEach(func() {
			taskToDesire := helpers.TaskCreateRequest(
				guid,
				&models.RunAction{
					User: "vcap",
					Path: "sh",
					Args: []string{"-c", "/usr/bin/env | grep 'CF_INSTANCE' > /home/vcap/env"},
				},
			)
			taskToDesire.ResultFile = "/home/vcap/env"

			err := bbsClient.DesireTask(logger, taskToDesire.TaskGuid, taskToDesire.Domain, taskToDesire.TaskDefinition)
			Expect(err).NotTo(HaveOccurred())

			Eventually(func() interface{} {
				var err error
				task, err = bbsClient.TaskByGuid(logger, guid)
				Expect(err).ShouldNot(HaveOccurred())

				return task.State
			}).Should(Equal(models.Task_Completed))
		})

		Context("when ExportNetworkEnvVars is false", func() {
			BeforeEach(func() {
				modifyRepConfig = func(config *repconfig.RepConfig) { config.ExportNetworkEnvVars = false }
			})

			It("does not set the networking environment variables", func() {
				Expect(task.Result).To(Equal(""))
			})
		})

		Context("when ExportNetworkEnvVars is true", func() {
			BeforeEach(func() {
				modifyRepConfig = func(config *repconfig.RepConfig) { config.ExportNetworkEnvVars = true }
			})

			It("sets the networking environment variables", func() {
				Expect(task.Result).To(ContainSubstring("CF_INSTANCE_ADDR=\n"))
				Expect(task.Result).To(ContainSubstring("CF_INSTANCE_PORT=\n"))
				Expect(task.Result).To(ContainSubstring("CF_INSTANCE_PORTS=[]\n"))
				Expect(task.Result).To(ContainSubstring(fmt.Sprintf("CF_INSTANCE_IP=%s\n", componentMaker.ExternalAddress)))
				Expect(task.Result).To(ContainSubstring("CF_INSTANCE_INTERNAL_IP="))
			})
		})
	})

	Describe("LRPs", func() {
		var response []byte
		var actualLRP *models.ActualLRP

		BeforeEach(func() {
			test_helper.CreateZipArchive(
				filepath.Join(fileServerStaticDir, "lrp.zip"),
				fixtures.GoServerApp(),
			)
		})

		JustBeforeEach(func() {
			lrp := helpers.DefaultLRPCreateRequest(guid, guid, 1)
			lrp.Setup = models.WrapAction(&models.DownloadAction{
				User: "vcap",
				From: fmt.Sprintf("http://%s/v1/static/%s", componentMaker.Addresses.FileServer, "lrp.zip"),
				To:   "/tmp/diego",
			})
			lrp.Action = models.WrapAction(&models.RunAction{
				User: "vcap",
				Path: "/tmp/diego/go-server",
				Env:  []*models.EnvironmentVariable{{"PORT", "8080"}},
			})

			err := bbsClient.DesireLRP(logger, lrp)
			Expect(err).NotTo(HaveOccurred())

			Eventually(helpers.LRPStatePoller(logger, bbsClient, guid, nil)).Should(Equal(models.ActualLRPStateRunning))
			Eventually(helpers.ResponseCodeFromHostPoller(componentMaker.Addresses.Router, helpers.DefaultHost)).Should(Equal(http.StatusOK))

			lrps, err := bbsClient.ActualLRPGroupsByProcessGuid(logger, guid)
			Expect(err).NotTo(HaveOccurred())

			Expect(lrps).To(HaveLen(1))
			actualLRP = lrps[0].Instance

			var status int
			response, status, err = helpers.ResponseBodyAndStatusCodeFromHost(componentMaker.Addresses.Router, helpers.DefaultHost, "env")
			Expect(status).To(Equal(http.StatusOK))
			Expect(err).NotTo(HaveOccurred())
		})

		Context("when ExportNetworkEnvVars is false", func() {
			BeforeEach(func() {
				modifyRepConfig = func(config *repconfig.RepConfig) { config.ExportNetworkEnvVars = false }
			})

			It("does not set the networking environment variables", func() {
				Expect(response).NotTo(ContainSubstring("CF_INSTANCE_IP="))
				Expect(response).NotTo(ContainSubstring("CF_INSTANCE_INTERNAL_IP="))
				Expect(response).NotTo(ContainSubstring("CF_INSTANCE_ADDR="))
				Expect(response).NotTo(ContainSubstring("CF_INSTANCE_PORT="))
				Expect(response).NotTo(ContainSubstring("CF_INSTANCE_PORTS="))
			})
		})

		Context("when ExportNetworkEnvVars is true", func() {
			BeforeEach(func() {
				modifyRepConfig = func(config *repconfig.RepConfig) { config.ExportNetworkEnvVars = true }
			})

			It("sets the networking environment variables", func() {
				netInfo := actualLRP.ActualLRPNetInfo
				Expect(response).To(ContainSubstring(fmt.Sprintf("CF_INSTANCE_ADDR=%s:%d\n", netInfo.Address, netInfo.Ports[0].HostPort)))
				Expect(response).To(ContainSubstring(fmt.Sprintf("CF_INSTANCE_IP=%s\n", componentMaker.ExternalAddress)))
				Expect(response).To(ContainSubstring(fmt.Sprintf("CF_INSTANCE_INTERNAL_IP=%s\n", netInfo.InstanceAddress)))
				Expect(response).To(ContainSubstring(fmt.Sprintf("CF_INSTANCE_PORT=%d\n", netInfo.Ports[0].HostPort)))

				type portMapping struct {
					External uint32 `json:"external"`
					Internal uint32 `json:"internal"`
				}
				ports := []portMapping{}

				buf := bytes.NewBuffer(response)
				for {
					line, err := buf.ReadString('\n')
					if err != nil {
						break
					}
					if strings.HasPrefix(line, "CF_INSTANCE_PORTS=") {
						err := json.Unmarshal([]byte(strings.TrimPrefix(line, "CF_INSTANCE_PORTS=")), &ports)
						Expect(err).NotTo(HaveOccurred())
						break
					}
				}

				Expect(ports).To(Equal([]portMapping{
					{External: netInfo.Ports[0].HostPort, Internal: netInfo.Ports[0].ContainerPort},
				}))
			})
		})
	})
})
<|endoftext|>


<commit_before>package gae_host

import (
	"fmt"
	"github.com/strongo/app"
	"github.com/strongo/bots-framework/core"
	"github.com/strongo/bots-framework/platforms/telegram"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"net/http"
)

type GaeTelegramChatStore struct {
	GaeBotChatStore
}

var _ bots.BotChatStore = (*GaeTelegramChatStore)(nil) // Check for interface implementation at compile time

func NewGaeTelegramChatStore(log strongo.Logger, r *http.Request) *GaeTelegramChatStore {
	return &GaeTelegramChatStore{
		GaeBotChatStore: GaeBotChatStore{
			GaeBaseStore: NewGaeBaseStore(log, r, telegram_bot.TelegramChatKind),
			newBotChatEntity: func() bots.BotChat {
				telegramChat := telegram_bot.NewTelegramChat()
				return &telegramChat
			},
			validateBotChatEntityType: func(entity bots.BotChat) {
				if _, ok := entity.(*telegram_bot.TelegramChat); !ok {
					panic(fmt.Sprintf("Expected *telegram_bot.TelegramChat but received %T", entity))
				}
			},
			botChatKey: func(botChatId interface{}) *datastore.Key {
				if intId, ok := botChatId.(int64); ok {
					key := datastore.NewKey(appengine.NewContext(r), telegram_bot.TelegramChatKind, "", (int64)(intId), nil)
					return key
				} else if strId, ok := botChatId.(string); ok {
					key := datastore.NewKey(appengine.NewContext(r), telegram_bot.TelegramChatKind, strId, 0, nil)
					return key
				} else {
					panic(fmt.Sprintf("Expected botChatId as int, got: %T", botChatId))
				}
			},
		},
	}
}
<commit_msg>FixTransfers()<commit_after>package gae_host

import (
	"fmt"
	"github.com/strongo/app"
	"github.com/strongo/bots-framework/core"
	"github.com/strongo/bots-framework/platforms/telegram"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"net/http"
	"github.com/qedus/nds"
	"time"
	"golang.org/x/net/context"
)

type GaeTelegramChatStore struct {
	GaeBotChatStore
}

var _ bots.BotChatStore = (*GaeTelegramChatStore)(nil) // Check for interface implementation at compile time

func NewGaeTelegramChatStore(log strongo.Logger, r *http.Request) *GaeTelegramChatStore {
	return &GaeTelegramChatStore{
		GaeBotChatStore: GaeBotChatStore{
			GaeBaseStore: NewGaeBaseStore(log, r, telegram_bot.TelegramChatKind),
			newBotChatEntity: func() bots.BotChat {
				telegramChat := telegram_bot.NewTelegramChat()
				return &telegramChat
			},
			validateBotChatEntityType: func(entity bots.BotChat) {
				if _, ok := entity.(*telegram_bot.TelegramChat); !ok {
					panic(fmt.Sprintf("Expected *telegram_bot.TelegramChat but received %T", entity))
				}
			},
			botChatKey: func(botChatId interface{}) *datastore.Key {
				if intId, ok := botChatId.(int64); ok {
					key := datastore.NewKey(appengine.NewContext(r), telegram_bot.TelegramChatKind, "", (int64)(intId), nil)
					return key
				} else if strId, ok := botChatId.(string); ok {
					key := datastore.NewKey(appengine.NewContext(r), telegram_bot.TelegramChatKind, strId, 0, nil)
					return key
				} else {
					panic(fmt.Sprintf("Expected botChatId as int, got: %T", botChatId))
				}
			},
		},
	}
}

func MarkTelegramChatAsForbidden(c context.Context, tgChatID int64, dtForbidden time.Time) error {
	return nds.RunInTransaction(c, func(c context.Context) (err error) {
		key := datastore.NewKey(c, telegram_bot.TelegramChatKind, "", tgChatID, nil)
		var chat telegram_bot.TelegramChat
		if err = nds.Get(c, key, &chat); err != nil {
			return
		}
		var changed bool
		if chat.DtForbidden.IsZero() {
			chat.DtForbidden = dtForbidden
			changed = true
		}

		if chat.DtForbiddenLast.IsZero() || chat.DtForbiddenLast.Before(dtForbidden) {
			chat.DtForbiddenLast = dtForbidden
			changed = true
		}

		if changed {
			_, err = nds.Put(c, key, &chat)
		}
		return
	}, nil)
}<|endoftext|>


<commit_before>package apollostats

const VERSION = "0.4"

// Max rows DB will return for all queries.
const MAX_ROWS = 200

// DB connection timeout, in seconds.
const TIMEOUT = 30

// Need to adjust time because the main server is running GMT-5.
const TIMEZONE_ADJUST = "EST"

// Cache update every x minutes.
const CACHE_UPDATE = 60
<commit_msg>Bump version.<commit_after>package apollostats

const VERSION = "0.5"

// Max rows DB will return for all queries.
const MAX_ROWS = 200

// DB connection timeout, in seconds.
const TIMEOUT = 30

// Need to adjust time because the main server is running GMT-5.
const TIMEZONE_ADJUST = "EST"

// Cache update every x minutes.
const CACHE_UPDATE = 60
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"strconv"
	"strings"
)

// CountTheWays is a custom type that
// we'll read a flag into
type CountTheWays []int

func (c *CountTheWays) String() string {
	result := ""
	for _, v := range *c {
		if len(result) > 0 {
			result += " ... "
		}
		result += fmt.Sprint(v)
	}
	return result
}

// Set will be used by the flag package
func (c *CountTheWays) Set(value string) error {
	values := strings.Split(value, ",")

	for _, v := range values {
		i, err := strconv.Atoi(v)
		if err != nil {
			return err
		}
		*c = append(*c, i)
	}

	return nil
}

// Config will be the holder for our flags
type Config struct {
	subject      string
	isAwesome    bool
	howAwesome   int
	countTheWays CountTheWays
}

// Setup initializes a config from flags that
// are passed in
func (c *Config) Setup() {
	// you can set a flag directly like so:
	// var someVar = flag.String("flag_name", "default_val", "description")
	// but in practice putting it in a struct is generally better

	// longhand
	flag.StringVar(&c.subject, "subject", "", "subject is a string, it defaults to empty")
	// shorthand
	flag.StringVar(&c.subject, "s", "", "subject is a string, it defaults to empty (shorthand)")

	flag.BoolVar(&c.isAwesome, "isawesome", true, "is go awesome or what?")
	flag.IntVar(&c.howAwesome, "howawesome", 10, "how awesome out of 10?")

	// custom variable type
	flag.Var(&c.countTheWays, "c", "comma separated list of integers")
}

// GetMessage uses all of the internal
// config vars and returns a sentence
func (c *Config) GetMessage() string {
	msg := c.subject
	if c.isAwesome {
		msg += " is awesome"
	} else {
		msg += " is NOT awesome"
	}

	msg = fmt.Sprintf("%s with a certainty of %d out of 10. Let me count the ways %s", msg, c.howAwesome, c.countTheWays.String())
	return msg
}

func main() {
	// initialize our setup
	c := Config{}
	c.Setup()

	// generally call this from main
	flag.Parse()

	fmt.Println(c.GetMessage())

}
<commit_msg>defaulted bool to false<commit_after>package main

import (
	"flag"
	"fmt"
	"strconv"
	"strings"
)

// CountTheWays is a custom type that
// we'll read a flag into
type CountTheWays []int

func (c *CountTheWays) String() string {
	result := ""
	for _, v := range *c {
		if len(result) > 0 {
			result += " ... "
		}
		result += fmt.Sprint(v)
	}
	return result
}

// Set will be used by the flag package
func (c *CountTheWays) Set(value string) error {
	values := strings.Split(value, ",")

	for _, v := range values {
		i, err := strconv.Atoi(v)
		if err != nil {
			return err
		}
		*c = append(*c, i)
	}

	return nil
}

// Config will be the holder for our flags
type Config struct {
	subject      string
	isAwesome    bool
	howAwesome   int
	countTheWays CountTheWays
}

// Setup initializes a config from flags that
// are passed in
func (c *Config) Setup() {
	// you can set a flag directly like so:
	// var someVar = flag.String("flag_name", "default_val", "description")
	// but in practice putting it in a struct is generally better

	// longhand
	flag.StringVar(&c.subject, "subject", "", "subject is a string, it defaults to empty")
	// shorthand
	flag.StringVar(&c.subject, "s", "", "subject is a string, it defaults to empty (shorthand)")

	flag.BoolVar(&c.isAwesome, "isawesome", false, "is go awesome or what?")
	flag.IntVar(&c.howAwesome, "howawesome", 10, "how awesome out of 10?")

	// custom variable type
	flag.Var(&c.countTheWays, "c", "comma separated list of integers")
}

// GetMessage uses all of the internal
// config vars and returns a sentence
func (c *Config) GetMessage() string {
	msg := c.subject
	if c.isAwesome {
		msg += " is awesome"
	} else {
		msg += " is NOT awesome"
	}

	msg = fmt.Sprintf("%s with a certainty of %d out of 10. Let me count the ways %s", msg, c.howAwesome, c.countTheWays.String())
	return msg
}

func main() {
	// initialize our setup
	c := Config{}
	c.Setup()

	// generally call this from main
	flag.Parse()

	fmt.Println(c.GetMessage())

}
<|endoftext|>


<commit_before>// A wrapper package that attempts to map the OpenCL 1.1 C API to idiomatic Go.
//
// TODO describe how empty interfaces are interpreted.
package cl11

import (
	"errors"
	"fmt"
	"reflect"
	"strings"
	"unsafe"
)

var (
	float32Type = reflect.TypeOf(float32(0))
	float32Size = float32Type.Size()
)

// A reference counted OpenCL object.
type Object interface {
	Retain() error
	Release() error
	ReferenceCount() (int, error)
}

type Profile int

func toProfile(profile string) Profile {
	switch profile {
	case "FULL_PROFILE":
		return FullProfile
	case "EMBEDDED_PROFILE":
		return EmbeddedProfile
	}
	panic(errors.New("unknown profile"))
}

func (pp Profile) String() string {
	switch pp {
	case zeroProfile:
		return ""
	case FullProfile:
		return "full profile"
	case EmbeddedProfile:
		return "embedded profile"
	}
	panic("unreachable")
}

type Version struct {
	Major int
	Minor int
	Info  string
}

func toVersion(version string) Version {

	var result Version
	var err error

	if strings.HasPrefix(version, "OpenCL C") {
		_, err = fmt.Sscanf(version, "OpenCL C %d.%d", &result.Major, &result.Minor)
		result.Info = strings.TrimSpace(version[len(fmt.Sprintf("OpenCL C %d.%d", result.Major, result.Minor)):])

	} else if strings.HasPrefix(version, "OpenCL") {
		_, err = fmt.Sscanf(version, "OpenCL %d.%d", &result.Major, &result.Minor)
		result.Info = strings.TrimSpace(version[len(fmt.Sprintf("OpenCL %d.%d", result.Major, result.Minor)):])

	} else {
		_, err = fmt.Sscanf(version, "%d.%d", &result.Major, &result.Minor)
	}

	if err != nil {
		// Maybe a regexp to try and find a "\d.\d"? It works on nVidia, AMD,
		// and Intel atm.
		panic("could not parse version string \"" + version + "\"")
	}

	return result
}

func (v Version) String() string {
	if v.Info != "" {
		return fmt.Sprintf("%d.%d %s", v.Major, v.Minor, v.Info)
	}
	return fmt.Sprintf("%d.%d", v.Major, v.Minor)
}

func pointerSize(value interface{}) (unsafe.Pointer, uintptr, error) {
	v := reflect.ValueOf(value)
	switch v.Kind() {
	// case reflect.Ptr:
	case reflect.Slice:
		pointer := unsafe.Pointer(&(*reflect.SliceHeader)(unsafe.Pointer(v.Pointer())).Data)
		size := v.Type().Elem().Size() * uintptr(v.Len())
		return pointer, size, nil
	}
	return unsafe.Pointer(uintptr(0)), 0, NotAddressable
}
<commit_msg>Removed todo in package documentation.<commit_after>// A wrapper package that attempts to map the OpenCL 1.1 C API to idiomatic Go.
package cl11

import (
	"errors"
	"fmt"
	"reflect"
	"strings"
	"unsafe"
)

var (
	float32Type = reflect.TypeOf(float32(0))
	float32Size = float32Type.Size()
)

// A reference counted OpenCL object.
type Object interface {
	Retain() error
	Release() error
	ReferenceCount() (int, error)
}

type Profile int

func toProfile(profile string) Profile {
	switch profile {
	case "FULL_PROFILE":
		return FullProfile
	case "EMBEDDED_PROFILE":
		return EmbeddedProfile
	}
	panic(errors.New("unknown profile"))
}

func (pp Profile) String() string {
	switch pp {
	case zeroProfile:
		return ""
	case FullProfile:
		return "full profile"
	case EmbeddedProfile:
		return "embedded profile"
	}
	panic("unreachable")
}

type Version struct {
	Major int
	Minor int
	Info  string
}

func toVersion(version string) Version {

	var result Version
	var err error

	if strings.HasPrefix(version, "OpenCL C") {
		_, err = fmt.Sscanf(version, "OpenCL C %d.%d", &result.Major, &result.Minor)
		result.Info = strings.TrimSpace(version[len(fmt.Sprintf("OpenCL C %d.%d", result.Major, result.Minor)):])

	} else if strings.HasPrefix(version, "OpenCL") {
		_, err = fmt.Sscanf(version, "OpenCL %d.%d", &result.Major, &result.Minor)
		result.Info = strings.TrimSpace(version[len(fmt.Sprintf("OpenCL %d.%d", result.Major, result.Minor)):])

	} else {
		_, err = fmt.Sscanf(version, "%d.%d", &result.Major, &result.Minor)
	}

	if err != nil {
		// Maybe a regexp to try and find a "\d.\d"? It works on nVidia, AMD,
		// and Intel atm.
		panic("could not parse version string \"" + version + "\"")
	}

	return result
}

func (v Version) String() string {
	if v.Info != "" {
		return fmt.Sprintf("%d.%d %s", v.Major, v.Minor, v.Info)
	}
	return fmt.Sprintf("%d.%d", v.Major, v.Minor)
}

func pointerSize(value interface{}) (unsafe.Pointer, uintptr, error) {
	v := reflect.ValueOf(value)
	switch v.Kind() {
	// case reflect.Ptr:
	case reflect.Slice:
		pointer := unsafe.Pointer(&(*reflect.SliceHeader)(unsafe.Pointer(v.Pointer())).Data)
		size := v.Type().Elem().Size() * uintptr(v.Len())
		return pointer, size, nil
	}
	return unsafe.Pointer(uintptr(0)), 0, NotAddressable
}
<|endoftext|>


<commit_before>package sources

import (
	"context"
	"fmt"
	"io"
	"net/http"

	lua "github.com/anaminus/gopher-lua"
	"github.com/anaminus/rbxmk"
	"github.com/anaminus/rbxmk/rtypes"
	"github.com/robloxapi/types"
)

func init() { register(HTTP) }
func HTTP() rbxmk.Source {
	return rbxmk.Source{
		Name: "http",
		Read: func(s rbxmk.State) (b []byte, err error) {
			options := s.Pull(1, "HTTPOptions").(rtypes.HTTPOptions)
			options.Method = "GET"
			options.RequestFormat = rtypes.FormatSelector{}
			options.ResponseFormat = rtypes.FormatSelector{Format: "bin"}
			options.Body = nil
			request, err := doHTTPRequest(s, options)
			if err != nil {
				return nil, err
			}
			resp, err := request.Resolve()
			if err != nil {
				return nil, err
			}
			if !resp.Success {
				return nil, fmt.Errorf(resp.StatusMessage)
			}
			body := resp.Body.(types.Stringlike).Stringlike()
			return []byte(body), nil
		},
		Write: func(s rbxmk.State, b []byte) (err error) {
			options := s.Pull(1, "HTTPOptions").(rtypes.HTTPOptions)
			options.Method = "POST"
			options.RequestFormat = rtypes.FormatSelector{Format: "bin"}
			options.ResponseFormat = rtypes.FormatSelector{}
			options.Body = types.BinaryString(b)
			request, err := doHTTPRequest(s, options)
			if err != nil {
				return err
			}
			resp, err := request.Resolve()
			if err != nil {
				return err
			}
			if !resp.Success {
				return fmt.Errorf(resp.StatusMessage)
			}
			return nil
		},
		Library: rbxmk.Library{
			Open: func(s rbxmk.State) *lua.LTable {
				lib := s.L.CreateTable(0, 1)
				lib.RawSetString("request", s.WrapFunc(httpRequest))
				return lib
			},
		},
	}
}

type HTTPRequest struct {
	cancel context.CancelFunc

	respch chan *http.Response
	resp   *rtypes.HTTPResponse

	errch chan error
	err   error

	fmt rbxmk.Format
	sel rtypes.FormatSelector
	pr  *io.PipeReader
}

// Type returns a string identifying the type of the value.
func (*HTTPRequest) Type() string {
	return "HTTPRequest"
}

func (r *HTTPRequest) encode(w *io.PipeWriter, f rbxmk.Format, s rtypes.FormatSelector, v types.Value) {
	if err := f.Encode(s, w, v); err != nil {
		w.CloseWithError(err)
		return
	}
	w.Close()
}

func (r *HTTPRequest) do(client *http.Client, req *http.Request) {
	defer close(r.respch)
	defer close(r.errch)
	if client == nil {
		client = http.DefaultClient
	}
	resp, err := client.Do(req)
	if err != nil {
		r.errch <- err
		return
	}
	r.respch <- resp
}

func (r *HTTPRequest) Resolve() (*rtypes.HTTPResponse, error) {
	if r.resp != nil || r.err != nil {
		return r.resp, r.err
	}
	if r.pr != nil {
		defer r.pr.Close()
	}
	select {
	case resp := <-r.respch:
		defer resp.Body.Close()
		r.resp = &rtypes.HTTPResponse{
			Success:       200 <= resp.StatusCode && resp.StatusCode < 300,
			StatusCode:    resp.StatusCode,
			StatusMessage: resp.Status,
			Headers:       rtypes.HTTPHeaders(resp.Header),
		}
		if r.fmt.Name != "" {
			if r.resp.Body, r.err = r.fmt.Decode(r.sel, resp.Body); r.err != nil {
				return nil, r.err
			}
		}
		return r.resp, nil
	case r.err = <-r.errch:
		return nil, r.err
	}
}

func (r *HTTPRequest) Cancel() {
	if r.resp != nil || r.err != nil {
		return
	}
	r.cancel()
	defer close(r.respch)
	defer close(r.errch)
	r.err = <-r.errch
}

func doHTTPRequest(s rbxmk.State, options rtypes.HTTPOptions) (request *HTTPRequest, err error) {
	var r *io.PipeReader
	var w *io.PipeWriter
	var reqfmt rbxmk.Format
	var respfmt rbxmk.Format
	if options.RequestFormat.Format != "" {
		reqfmt = s.Format(options.RequestFormat.Format)
		if reqfmt.Encode == nil {
			return nil, fmt.Errorf("cannot encode with format %s", reqfmt.Name)
		}
		if options.Body != nil {
			r, w = io.Pipe()
		}
	}
	if options.ResponseFormat.Format != "" {
		respfmt = s.Format(options.ResponseFormat.Format)
		if respfmt.Decode == nil {
			return nil, fmt.Errorf("cannot decode with format %s", respfmt.Name)
		}
	}

	// Create request.
	ctx, cancel := context.WithCancel(context.TODO())
	var req *http.Request
	if r != nil {
		req, err = http.NewRequestWithContext(ctx, options.Method, options.URL, r)
	} else {
		req, err = http.NewRequestWithContext(ctx, options.Method, options.URL, nil)
	}
	if err != nil {
		cancel()
		return nil, err
	}
	req.Header = http.Header(options.Headers)

	// Push request object.
	request = &HTTPRequest{
		cancel: cancel,
		respch: make(chan *http.Response),
		errch:  make(chan error),
		fmt:    respfmt,
		sel:    options.ResponseFormat,
		pr:     r,
	}
	if w != nil {
		go request.encode(w, reqfmt, options.RequestFormat, options.Body)
	}
	go request.do(s.Client, req)
	return request, nil
}

func httpRequest(s rbxmk.State) int {
	options := s.Pull(1, "HTTPOptions").(rtypes.HTTPOptions)
	request, err := doHTTPRequest(s, options)
	if err != nil {
		return s.RaiseError("%w", err)
	}
	return s.Push(request)
}
<commit_msg>Encode request body to memory.<commit_after>package sources

import (
	"bytes"
	"context"
	"fmt"
	"net/http"

	lua "github.com/anaminus/gopher-lua"
	"github.com/anaminus/rbxmk"
	"github.com/anaminus/rbxmk/rtypes"
	"github.com/robloxapi/types"
)

func init() { register(HTTP) }
func HTTP() rbxmk.Source {
	return rbxmk.Source{
		Name: "http",
		Read: func(s rbxmk.State) (b []byte, err error) {
			options := s.Pull(1, "HTTPOptions").(rtypes.HTTPOptions)
			options.Method = "GET"
			options.RequestFormat = rtypes.FormatSelector{}
			options.ResponseFormat = rtypes.FormatSelector{Format: "bin"}
			options.Body = nil
			request, err := doHTTPRequest(s, options)
			if err != nil {
				return nil, err
			}
			resp, err := request.Resolve()
			if err != nil {
				return nil, err
			}
			if !resp.Success {
				return nil, fmt.Errorf(resp.StatusMessage)
			}
			body := resp.Body.(types.Stringlike).Stringlike()
			return []byte(body), nil
		},
		Write: func(s rbxmk.State, b []byte) (err error) {
			options := s.Pull(1, "HTTPOptions").(rtypes.HTTPOptions)
			options.Method = "POST"
			options.RequestFormat = rtypes.FormatSelector{Format: "bin"}
			options.ResponseFormat = rtypes.FormatSelector{}
			options.Body = types.BinaryString(b)
			request, err := doHTTPRequest(s, options)
			if err != nil {
				return err
			}
			resp, err := request.Resolve()
			if err != nil {
				return err
			}
			if !resp.Success {
				return fmt.Errorf(resp.StatusMessage)
			}
			return nil
		},
		Library: rbxmk.Library{
			Open: func(s rbxmk.State) *lua.LTable {
				lib := s.L.CreateTable(0, 1)
				lib.RawSetString("request", s.WrapFunc(httpRequest))
				return lib
			},
		},
	}
}

type HTTPRequest struct {
	cancel context.CancelFunc

	respch chan *http.Response
	resp   *rtypes.HTTPResponse

	errch chan error
	err   error

	fmt rbxmk.Format
	sel rtypes.FormatSelector
}

// Type returns a string identifying the type of the value.
func (*HTTPRequest) Type() string {
	return "HTTPRequest"
}

func (r *HTTPRequest) do(client *http.Client, req *http.Request) {
	defer close(r.respch)
	defer close(r.errch)
	if client == nil {
		client = http.DefaultClient
	}
	resp, err := client.Do(req)
	if err != nil {
		r.errch <- err
		return
	}
	r.respch <- resp
}

func (r *HTTPRequest) Resolve() (*rtypes.HTTPResponse, error) {
	if r.resp != nil || r.err != nil {
		return r.resp, r.err
	}
	select {
	case resp := <-r.respch:
		defer resp.Body.Close()
		r.resp = &rtypes.HTTPResponse{
			Success:       200 <= resp.StatusCode && resp.StatusCode < 300,
			StatusCode:    resp.StatusCode,
			StatusMessage: resp.Status,
			Headers:       rtypes.HTTPHeaders(resp.Header),
		}
		if r.fmt.Name != "" {
			if r.resp.Body, r.err = r.fmt.Decode(r.sel, resp.Body); r.err != nil {
				return nil, r.err
			}
		}
		return r.resp, nil
	case r.err = <-r.errch:
		return nil, r.err
	}
}

func (r *HTTPRequest) Cancel() {
	if r.resp != nil || r.err != nil {
		return
	}
	r.cancel()
	defer close(r.respch)
	defer close(r.errch)
	r.err = <-r.errch
}

func doHTTPRequest(s rbxmk.State, options rtypes.HTTPOptions) (request *HTTPRequest, err error) {
	var buf *bytes.Buffer
	if options.RequestFormat.Format != "" {
		reqfmt := s.Format(options.RequestFormat.Format)
		if reqfmt.Encode == nil {
			return nil, fmt.Errorf("cannot encode with format %s", reqfmt.Name)
		}
		if options.Body != nil {
			buf = new(bytes.Buffer)
			if err := reqfmt.Encode(options.RequestFormat, buf, options.Body); err != nil {
				return nil, fmt.Errorf("encode body: %w", err)
			}
		}
	}
	var respfmt rbxmk.Format
	if options.ResponseFormat.Format != "" {
		respfmt = s.Format(options.ResponseFormat.Format)
		if respfmt.Decode == nil {
			return nil, fmt.Errorf("cannot decode with format %s", respfmt.Name)
		}
	}

	// Create request.
	ctx, cancel := context.WithCancel(context.TODO())
	var req *http.Request
	if buf != nil {
		// Use of *bytes.Buffer guarantees that req.GetBody will be set.
		req, err = http.NewRequestWithContext(ctx, options.Method, options.URL, buf)
	} else {
		req, err = http.NewRequestWithContext(ctx, options.Method, options.URL, nil)
	}
	if err != nil {
		cancel()
		return nil, err
	}
	req.Header = http.Header(options.Headers)

	// Push request object.
	request = &HTTPRequest{
		cancel: cancel,
		respch: make(chan *http.Response),
		errch:  make(chan error),
		fmt:    respfmt,
		sel:    options.ResponseFormat,
	}
	go request.do(s.Client, req)
	return request, nil
}

func httpRequest(s rbxmk.State) int {
	options := s.Pull(1, "HTTPOptions").(rtypes.HTTPOptions)
	request, err := doHTTPRequest(s, options)
	if err != nil {
		return s.RaiseError("%w", err)
	}
	return s.Push(request)
}
<|endoftext|>


<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2017 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package cloud

import (
	"context"
	"encoding/json"
	"path/filepath"
	"sync"
	"time"

	"github.com/loadimpact/k6/lib/metrics"
	"github.com/loadimpact/k6/lib/netext"
	"github.com/pkg/errors"

	"gopkg.in/guregu/null.v3"

	"github.com/loadimpact/k6/lib"
	"github.com/loadimpact/k6/stats"
	log "github.com/sirupsen/logrus"
)

// TestName is the default Load Impact Cloud test name
const TestName = "k6 test"

// Collector sends result data to the Load Impact cloud service.
type Collector struct {
	config      Config
	referenceID string

	duration   int64
	thresholds map[string][]*stats.Threshold
	client     *Client

	anonymous bool

	bufferMutex      sync.Mutex
	bufferHTTPTrails []*netext.Trail
	bufferSamples    []*Sample

	opts lib.Options

	aggrBuckets map[int64]aggregationBucket
}

// Verify that Collector implements lib.Collector
var _ lib.Collector = &Collector{}

// New creates a new cloud collector
func New(conf Config, src *lib.SourceData, opts lib.Options, version string) (*Collector, error) {
	if val, ok := opts.External["loadimpact"]; ok {
		if err := json.Unmarshal(val, &conf); err != nil {
			return nil, err
		}
	}

	if conf.AggregationPeriod.Duration > 0 && (opts.SystemTags["vu"] || opts.SystemTags["iter"]) {
		return nil, errors.New("Aggregatoion cannot be enabled if the 'vu' or 'iter' system tag is also enabled")
	}

	if !conf.Name.Valid || conf.Name.String == "" {
		conf.Name = null.StringFrom(filepath.Base(src.Filename))
	}
	if conf.Name.String == "-" {
		conf.Name = null.StringFrom(TestName)
	}

	thresholds := make(map[string][]*stats.Threshold)
	for name, t := range opts.Thresholds {
		thresholds[name] = append(thresholds[name], t.Thresholds...)
	}

	// Sum test duration from options. -1 for unknown duration.
	var duration int64 = -1
	if len(opts.Stages) > 0 {
		duration = sumStages(opts.Stages)
	} else if opts.Duration.Valid {
		duration = int64(time.Duration(opts.Duration.Duration).Seconds())
	}

	if duration == -1 {
		return nil, errors.New("Tests with unspecified duration are not allowed when using Load Impact Insights")
	}

	if !conf.Token.Valid && conf.DeprecatedToken.Valid {
		log.Warn("K6CLOUD_TOKEN is deprecated and will be removed. Use K6_CLOUD_TOKEN instead.")
		conf.Token = conf.DeprecatedToken
	}

	return &Collector{
		config:      conf,
		thresholds:  thresholds,
		client:      NewClient(conf.Token.String, conf.Host.String, version),
		anonymous:   !conf.Token.Valid,
		duration:    duration,
		opts:        opts,
		aggrBuckets: map[int64]aggregationBucket{},
	}, nil
}

func (c *Collector) Init() error {
	thresholds := make(map[string][]string)

	for name, t := range c.thresholds {
		for _, threshold := range t {
			thresholds[name] = append(thresholds[name], threshold.Source)
		}
	}

	testRun := &TestRun{
		Name:       c.config.Name.String,
		ProjectID:  c.config.ProjectID.Int64,
		VUsMax:     c.opts.VUsMax.Int64,
		Thresholds: thresholds,
		Duration:   c.duration,
	}

	response, err := c.client.CreateTestRun(testRun)
	if err != nil {
		return err
	}
	c.referenceID = response.ReferenceID

	if response.ConfigOverride != nil {
		log.WithFields(log.Fields{
			"override": response.ConfigOverride,
		}).Debug("Cloud: overriding config options")
		c.config = c.config.Apply(*response.ConfigOverride)
	}

	log.WithFields(log.Fields{
		"name":        c.config.Name,
		"projectId":   c.config.ProjectID,
		"duration":    c.duration,
		"referenceId": c.referenceID,
	}).Debug("Cloud: Initialized")
	return nil
}

func (c *Collector) Link() string {
	return URLForResults(c.referenceID, c.config)
}

func (c *Collector) Run(ctx context.Context) {
	wg := sync.WaitGroup{}

	// If enabled, start periodically aggregating the collected HTTP trails
	if c.config.AggregationPeriod.Duration > 0 {
		wg.Add(1)
		aggregationTicker := time.NewTicker(time.Duration(c.config.AggregationCalcInterval.Duration))

		go func() {
			for {
				select {
				case <-aggregationTicker.C:
					c.aggregateHTTPTrails(time.Duration(c.config.AggregationWaitPeriod.Duration))
				case <-ctx.Done():
					c.aggregateHTTPTrails(0)
					c.flushHTTPTrails()
					c.pushMetrics()
					wg.Done()
					return
				}
			}
		}()
	}

	defer func() {
		wg.Wait()
		c.testFinished()
	}()

	pushTicker := time.NewTicker(time.Duration(c.config.MetricPushInterval.Duration))
	for {
		select {
		case <-pushTicker.C:
			c.pushMetrics()
		case <-ctx.Done():
			c.pushMetrics()
			return
		}
	}
}

func (c *Collector) IsReady() bool {
	return true
}

func (c *Collector) Collect(sampleContainers []stats.SampleContainer) {
	if c.referenceID == "" {
		return
	}

	newSamples := []*Sample{}
	newHTTPTrails := []*netext.Trail{}

	for _, sampleContainer := range sampleContainers {
		switch sc := sampleContainer.(type) {
		case *netext.Trail:
			// Check if aggregation is enabled,
			if c.config.AggregationPeriod.Duration > 0 {
				newHTTPTrails = append(newHTTPTrails, sc)
			} else {
				newSamples = append(newSamples, NewSampleFromTrail(sc))
			}
		case *netext.NetTrail:
			//TODO: aggregate?
			newSamples = append(newSamples, &Sample{
				Type:   DataTypeMap,
				Metric: "iter_li_all",
				Data: &SampleDataMap{
					Time: Timestamp(sc.GetTime()),
					Tags: sc.GetTags(),
					Values: map[string]float64{
						metrics.DataSent.Name:          float64(sc.BytesWritten),
						metrics.DataReceived.Name:      float64(sc.BytesRead),
						metrics.IterationDuration.Name: stats.D(sc.EndTime.Sub(sc.StartTime)),
					},
				}})
		default:
			for _, sample := range sampleContainer.GetSamples() {
				newSamples = append(newSamples, &Sample{
					Type:   DataTypeSingle,
					Metric: sample.Metric.Name,
					Data: &SampleDataSingle{
						Type:  sample.Metric.Type,
						Time:  Timestamp(sample.Time),
						Tags:  sample.Tags,
						Value: sample.Value,
					},
				})
			}

		}
	}

	if len(newSamples) > 0 || len(newHTTPTrails) > 0 {
		c.bufferMutex.Lock()
		c.bufferSamples = append(c.bufferSamples, newSamples...)
		c.bufferHTTPTrails = append(c.bufferHTTPTrails, newHTTPTrails...)
		c.bufferMutex.Unlock()
	}
}

func (c *Collector) aggregateHTTPTrails(waitPeriod time.Duration) {
	c.bufferMutex.Lock()
	newHTTPTrails := c.bufferHTTPTrails
	c.bufferHTTPTrails = nil
	c.bufferMutex.Unlock()

	aggrPeriod := int64(c.config.AggregationPeriod.Duration)

	// Distribute all newly buffered HTTP trails into buckets and sub-buckets
	for _, trail := range newHTTPTrails {
		trailTags := trail.GetTags()
		bucketID := trail.GetTime().UnixNano() / aggrPeriod

		// Get or create a time bucket for that trail period
		bucket, ok := c.aggrBuckets[bucketID]
		if !ok {
			bucket = aggregationBucket{}
			c.aggrBuckets[bucketID] = bucket
		}

		// Either use an existing subbucket key or use the trail tags as a new one
		subBucketKey := trailTags
		subBucket, ok := bucket[subBucketKey]
		if !ok {
			for sbTags, sb := range bucket {
				if trailTags.IsEqual(sbTags) {
					subBucketKey = sbTags
					subBucket = sb
				}
			}
		}
		bucket[subBucketKey] = append(subBucket, trail)
	}

	// Which buckets are still new and we'll wait for trails to accumulate before aggregating
	bucketCutoffID := time.Now().Add(-waitPeriod).UnixNano() / aggrPeriod
	iqrRadius := c.config.AggregationOutlierIqrRadius.Float64
	iqrLowerCoef := c.config.AggregationOutlierIqrCoefLower.Float64
	iqrUpperCoef := c.config.AggregationOutlierIqrCoefUpper.Float64
	newSamples := []*Sample{}

	// Handle all aggregation buckets older than bucketCutoffID
	for bucketID, subBuckets := range c.aggrBuckets {
		if bucketID > bucketCutoffID {
			continue
		}

		for tags, httpTrails := range subBuckets {
			trailCount := int64(len(httpTrails))
			if trailCount < c.config.AggregationMinSamples.Int64 {
				for _, trail := range httpTrails {
					newSamples = append(newSamples, NewSampleFromTrail(trail))
				}
				continue
			}

			connDurations := make(durations, trailCount)
			reqDurations := make(durations, trailCount)
			for i, trail := range httpTrails {
				connDurations[i] = trail.ConnDuration
				reqDurations[i] = trail.Duration
			}
			minConnDur, maxConnDur := connDurations.SelectGetNormalBounds(iqrRadius, iqrLowerCoef, iqrUpperCoef)
			minReqDur, maxReqDur := reqDurations.SelectGetNormalBounds(iqrRadius, iqrLowerCoef, iqrUpperCoef)

			aggrData := &SampleDataAggregatedHTTPReqs{
				Time: Timestamp(time.Unix(0, bucketID*aggrPeriod+aggrPeriod/2)),
				Type: "aggregated_trend",
				Tags: tags,
			}

			for _, trail := range httpTrails {
				if trail.ConnDuration < minConnDur ||
					trail.ConnDuration > maxConnDur ||
					trail.Duration < minReqDur ||
					trail.Duration > maxReqDur {

					newSamples = append(newSamples, NewSampleFromTrail(trail))
				} else {
					aggrData.Add(trail)
				}
			}
			aggrData.CalcAverages()

			if aggrData.Count > 0 {
				log.WithFields(log.Fields{
					"http_samples": aggrData.Count,
				}).Debug("Aggregated HTTP metrics")
				newSamples = append(newSamples, &Sample{
					Type:   DataTypeAggregatedHTTPReqs,
					Metric: "http_req_li_all",
					Data:   aggrData,
				})
			}
		}
		delete(c.aggrBuckets, bucketID)
	}

	if len(newSamples) > 0 {
		c.bufferMutex.Lock()
		c.bufferSamples = append(c.bufferSamples, newSamples...)
		c.bufferMutex.Unlock()
	}
}

func (c *Collector) flushHTTPTrails() {
	c.bufferMutex.Lock()
	defer c.bufferMutex.Unlock()

	newSamples := []*Sample{}
	for _, trail := range c.bufferHTTPTrails {
		newSamples = append(newSamples, NewSampleFromTrail(trail))
	}
	for _, bucket := range c.aggrBuckets {
		for _, trails := range bucket {
			for _, trail := range trails {
				newSamples = append(newSamples, NewSampleFromTrail(trail))
			}
		}
	}

	c.bufferHTTPTrails = nil
	c.aggrBuckets = map[int64]aggregationBucket{}
	c.bufferSamples = append(c.bufferSamples, newSamples...)
}
func (c *Collector) pushMetrics() {
	c.bufferMutex.Lock()
	if len(c.bufferSamples) == 0 {
		c.bufferMutex.Unlock()
		return
	}
	buffer := c.bufferSamples
	c.bufferSamples = nil
	c.bufferMutex.Unlock()

	log.WithFields(log.Fields{
		"samples": len(buffer),
	}).Debug("Pushing metrics to cloud")

	err := c.client.PushMetric(c.referenceID, c.config.NoCompress.Bool, buffer)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Warn("Failed to send metrics to cloud")
	}
}

func (c *Collector) testFinished() {
	if c.referenceID == "" {
		return
	}

	testTainted := false
	thresholdResults := make(ThresholdResult)
	for name, thresholds := range c.thresholds {
		thresholdResults[name] = make(map[string]bool)
		for _, t := range thresholds {
			thresholdResults[name][t.Source] = t.Failed
			if t.Failed {
				testTainted = true
			}
		}
	}

	log.WithFields(log.Fields{
		"ref":     c.referenceID,
		"tainted": testTainted,
	}).Debug("Sending test finished")

	err := c.client.TestFinished(c.referenceID, thresholdResults, testTainted)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Warn("Failed to send test finished to cloud")
	}
}

func sumStages(stages []lib.Stage) int64 {
	var total time.Duration
	for _, stage := range stages {
		total += time.Duration(stage.Duration.Duration)
	}

	return int64(total.Seconds())
}

// GetRequiredSystemTags returns which sample tags are needed by this collector
func (c *Collector) GetRequiredSystemTags() lib.TagSet {
	return lib.GetTagSet("name", "method", "status", "error", "check", "group")
}
<commit_msg>Fix a typo<commit_after>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2017 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package cloud

import (
	"context"
	"encoding/json"
	"path/filepath"
	"sync"
	"time"

	"github.com/loadimpact/k6/lib/metrics"
	"github.com/loadimpact/k6/lib/netext"
	"github.com/pkg/errors"

	"gopkg.in/guregu/null.v3"

	"github.com/loadimpact/k6/lib"
	"github.com/loadimpact/k6/stats"
	log "github.com/sirupsen/logrus"
)

// TestName is the default Load Impact Cloud test name
const TestName = "k6 test"

// Collector sends result data to the Load Impact cloud service.
type Collector struct {
	config      Config
	referenceID string

	duration   int64
	thresholds map[string][]*stats.Threshold
	client     *Client

	anonymous bool

	bufferMutex      sync.Mutex
	bufferHTTPTrails []*netext.Trail
	bufferSamples    []*Sample

	opts lib.Options

	aggrBuckets map[int64]aggregationBucket
}

// Verify that Collector implements lib.Collector
var _ lib.Collector = &Collector{}

// New creates a new cloud collector
func New(conf Config, src *lib.SourceData, opts lib.Options, version string) (*Collector, error) {
	if val, ok := opts.External["loadimpact"]; ok {
		if err := json.Unmarshal(val, &conf); err != nil {
			return nil, err
		}
	}

	if conf.AggregationPeriod.Duration > 0 && (opts.SystemTags["vu"] || opts.SystemTags["iter"]) {
		return nil, errors.New("Aggregation cannot be enabled if the 'vu' or 'iter' system tag is also enabled")
	}

	if !conf.Name.Valid || conf.Name.String == "" {
		conf.Name = null.StringFrom(filepath.Base(src.Filename))
	}
	if conf.Name.String == "-" {
		conf.Name = null.StringFrom(TestName)
	}

	thresholds := make(map[string][]*stats.Threshold)
	for name, t := range opts.Thresholds {
		thresholds[name] = append(thresholds[name], t.Thresholds...)
	}

	// Sum test duration from options. -1 for unknown duration.
	var duration int64 = -1
	if len(opts.Stages) > 0 {
		duration = sumStages(opts.Stages)
	} else if opts.Duration.Valid {
		duration = int64(time.Duration(opts.Duration.Duration).Seconds())
	}

	if duration == -1 {
		return nil, errors.New("Tests with unspecified duration are not allowed when using Load Impact Insights")
	}

	if !conf.Token.Valid && conf.DeprecatedToken.Valid {
		log.Warn("K6CLOUD_TOKEN is deprecated and will be removed. Use K6_CLOUD_TOKEN instead.")
		conf.Token = conf.DeprecatedToken
	}

	return &Collector{
		config:      conf,
		thresholds:  thresholds,
		client:      NewClient(conf.Token.String, conf.Host.String, version),
		anonymous:   !conf.Token.Valid,
		duration:    duration,
		opts:        opts,
		aggrBuckets: map[int64]aggregationBucket{},
	}, nil
}

func (c *Collector) Init() error {
	thresholds := make(map[string][]string)

	for name, t := range c.thresholds {
		for _, threshold := range t {
			thresholds[name] = append(thresholds[name], threshold.Source)
		}
	}

	testRun := &TestRun{
		Name:       c.config.Name.String,
		ProjectID:  c.config.ProjectID.Int64,
		VUsMax:     c.opts.VUsMax.Int64,
		Thresholds: thresholds,
		Duration:   c.duration,
	}

	response, err := c.client.CreateTestRun(testRun)
	if err != nil {
		return err
	}
	c.referenceID = response.ReferenceID

	if response.ConfigOverride != nil {
		log.WithFields(log.Fields{
			"override": response.ConfigOverride,
		}).Debug("Cloud: overriding config options")
		c.config = c.config.Apply(*response.ConfigOverride)
	}

	log.WithFields(log.Fields{
		"name":        c.config.Name,
		"projectId":   c.config.ProjectID,
		"duration":    c.duration,
		"referenceId": c.referenceID,
	}).Debug("Cloud: Initialized")
	return nil
}

func (c *Collector) Link() string {
	return URLForResults(c.referenceID, c.config)
}

func (c *Collector) Run(ctx context.Context) {
	wg := sync.WaitGroup{}

	// If enabled, start periodically aggregating the collected HTTP trails
	if c.config.AggregationPeriod.Duration > 0 {
		wg.Add(1)
		aggregationTicker := time.NewTicker(time.Duration(c.config.AggregationCalcInterval.Duration))

		go func() {
			for {
				select {
				case <-aggregationTicker.C:
					c.aggregateHTTPTrails(time.Duration(c.config.AggregationWaitPeriod.Duration))
				case <-ctx.Done():
					c.aggregateHTTPTrails(0)
					c.flushHTTPTrails()
					c.pushMetrics()
					wg.Done()
					return
				}
			}
		}()
	}

	defer func() {
		wg.Wait()
		c.testFinished()
	}()

	pushTicker := time.NewTicker(time.Duration(c.config.MetricPushInterval.Duration))
	for {
		select {
		case <-pushTicker.C:
			c.pushMetrics()
		case <-ctx.Done():
			c.pushMetrics()
			return
		}
	}
}

func (c *Collector) IsReady() bool {
	return true
}

func (c *Collector) Collect(sampleContainers []stats.SampleContainer) {
	if c.referenceID == "" {
		return
	}

	newSamples := []*Sample{}
	newHTTPTrails := []*netext.Trail{}

	for _, sampleContainer := range sampleContainers {
		switch sc := sampleContainer.(type) {
		case *netext.Trail:
			// Check if aggregation is enabled,
			if c.config.AggregationPeriod.Duration > 0 {
				newHTTPTrails = append(newHTTPTrails, sc)
			} else {
				newSamples = append(newSamples, NewSampleFromTrail(sc))
			}
		case *netext.NetTrail:
			//TODO: aggregate?
			newSamples = append(newSamples, &Sample{
				Type:   DataTypeMap,
				Metric: "iter_li_all",
				Data: &SampleDataMap{
					Time: Timestamp(sc.GetTime()),
					Tags: sc.GetTags(),
					Values: map[string]float64{
						metrics.DataSent.Name:          float64(sc.BytesWritten),
						metrics.DataReceived.Name:      float64(sc.BytesRead),
						metrics.IterationDuration.Name: stats.D(sc.EndTime.Sub(sc.StartTime)),
					},
				}})
		default:
			for _, sample := range sampleContainer.GetSamples() {
				newSamples = append(newSamples, &Sample{
					Type:   DataTypeSingle,
					Metric: sample.Metric.Name,
					Data: &SampleDataSingle{
						Type:  sample.Metric.Type,
						Time:  Timestamp(sample.Time),
						Tags:  sample.Tags,
						Value: sample.Value,
					},
				})
			}

		}
	}

	if len(newSamples) > 0 || len(newHTTPTrails) > 0 {
		c.bufferMutex.Lock()
		c.bufferSamples = append(c.bufferSamples, newSamples...)
		c.bufferHTTPTrails = append(c.bufferHTTPTrails, newHTTPTrails...)
		c.bufferMutex.Unlock()
	}
}

func (c *Collector) aggregateHTTPTrails(waitPeriod time.Duration) {
	c.bufferMutex.Lock()
	newHTTPTrails := c.bufferHTTPTrails
	c.bufferHTTPTrails = nil
	c.bufferMutex.Unlock()

	aggrPeriod := int64(c.config.AggregationPeriod.Duration)

	// Distribute all newly buffered HTTP trails into buckets and sub-buckets
	for _, trail := range newHTTPTrails {
		trailTags := trail.GetTags()
		bucketID := trail.GetTime().UnixNano() / aggrPeriod

		// Get or create a time bucket for that trail period
		bucket, ok := c.aggrBuckets[bucketID]
		if !ok {
			bucket = aggregationBucket{}
			c.aggrBuckets[bucketID] = bucket
		}

		// Either use an existing subbucket key or use the trail tags as a new one
		subBucketKey := trailTags
		subBucket, ok := bucket[subBucketKey]
		if !ok {
			for sbTags, sb := range bucket {
				if trailTags.IsEqual(sbTags) {
					subBucketKey = sbTags
					subBucket = sb
				}
			}
		}
		bucket[subBucketKey] = append(subBucket, trail)
	}

	// Which buckets are still new and we'll wait for trails to accumulate before aggregating
	bucketCutoffID := time.Now().Add(-waitPeriod).UnixNano() / aggrPeriod
	iqrRadius := c.config.AggregationOutlierIqrRadius.Float64
	iqrLowerCoef := c.config.AggregationOutlierIqrCoefLower.Float64
	iqrUpperCoef := c.config.AggregationOutlierIqrCoefUpper.Float64
	newSamples := []*Sample{}

	// Handle all aggregation buckets older than bucketCutoffID
	for bucketID, subBuckets := range c.aggrBuckets {
		if bucketID > bucketCutoffID {
			continue
		}

		for tags, httpTrails := range subBuckets {
			trailCount := int64(len(httpTrails))
			if trailCount < c.config.AggregationMinSamples.Int64 {
				for _, trail := range httpTrails {
					newSamples = append(newSamples, NewSampleFromTrail(trail))
				}
				continue
			}

			connDurations := make(durations, trailCount)
			reqDurations := make(durations, trailCount)
			for i, trail := range httpTrails {
				connDurations[i] = trail.ConnDuration
				reqDurations[i] = trail.Duration
			}
			minConnDur, maxConnDur := connDurations.SelectGetNormalBounds(iqrRadius, iqrLowerCoef, iqrUpperCoef)
			minReqDur, maxReqDur := reqDurations.SelectGetNormalBounds(iqrRadius, iqrLowerCoef, iqrUpperCoef)

			aggrData := &SampleDataAggregatedHTTPReqs{
				Time: Timestamp(time.Unix(0, bucketID*aggrPeriod+aggrPeriod/2)),
				Type: "aggregated_trend",
				Tags: tags,
			}

			for _, trail := range httpTrails {
				if trail.ConnDuration < minConnDur ||
					trail.ConnDuration > maxConnDur ||
					trail.Duration < minReqDur ||
					trail.Duration > maxReqDur {

					newSamples = append(newSamples, NewSampleFromTrail(trail))
				} else {
					aggrData.Add(trail)
				}
			}
			aggrData.CalcAverages()

			if aggrData.Count > 0 {
				log.WithFields(log.Fields{
					"http_samples": aggrData.Count,
				}).Debug("Aggregated HTTP metrics")
				newSamples = append(newSamples, &Sample{
					Type:   DataTypeAggregatedHTTPReqs,
					Metric: "http_req_li_all",
					Data:   aggrData,
				})
			}
		}
		delete(c.aggrBuckets, bucketID)
	}

	if len(newSamples) > 0 {
		c.bufferMutex.Lock()
		c.bufferSamples = append(c.bufferSamples, newSamples...)
		c.bufferMutex.Unlock()
	}
}

func (c *Collector) flushHTTPTrails() {
	c.bufferMutex.Lock()
	defer c.bufferMutex.Unlock()

	newSamples := []*Sample{}
	for _, trail := range c.bufferHTTPTrails {
		newSamples = append(newSamples, NewSampleFromTrail(trail))
	}
	for _, bucket := range c.aggrBuckets {
		for _, trails := range bucket {
			for _, trail := range trails {
				newSamples = append(newSamples, NewSampleFromTrail(trail))
			}
		}
	}

	c.bufferHTTPTrails = nil
	c.aggrBuckets = map[int64]aggregationBucket{}
	c.bufferSamples = append(c.bufferSamples, newSamples...)
}
func (c *Collector) pushMetrics() {
	c.bufferMutex.Lock()
	if len(c.bufferSamples) == 0 {
		c.bufferMutex.Unlock()
		return
	}
	buffer := c.bufferSamples
	c.bufferSamples = nil
	c.bufferMutex.Unlock()

	log.WithFields(log.Fields{
		"samples": len(buffer),
	}).Debug("Pushing metrics to cloud")

	err := c.client.PushMetric(c.referenceID, c.config.NoCompress.Bool, buffer)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Warn("Failed to send metrics to cloud")
	}
}

func (c *Collector) testFinished() {
	if c.referenceID == "" {
		return
	}

	testTainted := false
	thresholdResults := make(ThresholdResult)
	for name, thresholds := range c.thresholds {
		thresholdResults[name] = make(map[string]bool)
		for _, t := range thresholds {
			thresholdResults[name][t.Source] = t.Failed
			if t.Failed {
				testTainted = true
			}
		}
	}

	log.WithFields(log.Fields{
		"ref":     c.referenceID,
		"tainted": testTainted,
	}).Debug("Sending test finished")

	err := c.client.TestFinished(c.referenceID, thresholdResults, testTainted)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Warn("Failed to send test finished to cloud")
	}
}

func sumStages(stages []lib.Stage) int64 {
	var total time.Duration
	for _, stage := range stages {
		total += time.Duration(stage.Duration.Duration)
	}

	return int64(total.Seconds())
}

// GetRequiredSystemTags returns which sample tags are needed by this collector
func (c *Collector) GetRequiredSystemTags() lib.TagSet {
	return lib.GetTagSet("name", "method", "status", "error", "check", "group")
}
<|endoftext|>