<commit_before>package miner

import (
	"bytes"
	"math/rand" // We should probably switch to crypto/rand, but we should use benchmarks first.

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/types"
)

// Creates a block that is ready for nonce grinding.
func (m *Miner) blockForWork() (b types.Block) {
	// Fill out the block with potentially ready values.
	b = types.Block{
		ParentID:     m.parent,
		Timestamp:    types.CurrentTimestamp(),
		Nonce:        uint64(rand.Int()),
		Transactions: m.transactions,
	}

	// Calculate the subsidy and create the miner payout.
	height, exists := m.state.HeightOfBlock(m.parent)
	if !exists {
		if build.DEBUG {
			panic("parent is not in state?")
		}
		return
	}
	subsidy := types.CalculateCoinbase(height + 1)
	for _, txn := range m.transactions {
		for _, fee := range txn.MinerFees {
			subsidy = subsidy.Add(fee)
		}
	}
	output := types.SiacoinOutput{Value: subsidy, UnlockHash: m.address}
	b.MinerPayouts = []types.SiacoinOutput{output}

	// If we've got a time earlier than the earliest legal timestamp, set the
	// timestamp equal to the earliest legal timestamp.
	if b.Timestamp < m.earliestTimestamp {
		b.Timestamp = m.earliestTimestamp
	}

	return
}

// mine attempts to generate blocks, and will run until desiredThreads is
// changd to be lower than `myThread`, which is set at the beginning of the
// function.
//
// The threading is fragile. Edit with caution!
func (m *Miner) threadedMine() {
	// Increment the number of threads running, because this thread is spinning
	// up. Also grab a number that will tell us when to shut down.
	m.mu.Lock()
	m.runningThreads++
	myThread := m.runningThreads
	m.mu.Unlock()

	// Try to solve a block repeatedly.
	for {
		// Grab the number of threads that are supposed to be running.
		m.mu.RLock()
		desiredThreads := m.desiredThreads
		m.mu.RUnlock()

		// If we are allowed to be running, mine a block, otherwise shut down.
		if desiredThreads >= myThread {
			// Grab the necessary variables for mining, and then attempt to
			// mine a block.
			m.mu.RLock()
			bfw := m.blockForWork()
			target := m.target
			iterations := m.iterationsPerAttempt
			m.mu.RUnlock()
			m.solveBlock(bfw, target, iterations)
		} else {
			m.mu.Lock()
			// Need to check the mining status again, something might have
			// changed while waiting for the lock.
			if desiredThreads < myThread {
				m.runningThreads--
				m.mu.Unlock()
				return
			}
			m.mu.Unlock()
		}
	}
}

func fastCheckTarget(target types.Target, b types.Block, bRoot crypto.Hash) bool {
	id := crypto.HashAll(
		b.ParentID,
		b.Nonce,
		bRoot,
	)
	return bytes.Compare(target[:], id[:]) >= 0
}

// solveBlock takes a block, target, and number of iterations as input and
// tries to find a block that meets the target. This function can take a long
// time to complete, and should not be called with a lock.
func (m *Miner) solveBlock(blockForWork types.Block, target types.Target, iterations uint64) (b types.Block, solved bool, err error) {
	// solveBlock could operate on a pointer, but it's not strictly necessary
	// and it makes calling weirder/more opaque.
	b = blockForWork
	bRoot := b.MerkleRoot()

	// Iterate through a bunch of nonces (from a random starting point) and try
	// to find a winnning solution.
	for maxNonce := b.Nonce + iterations; b.Nonce != maxNonce; b.Nonce++ {
		if fastCheckTarget(target, b, bRoot) {
			err = m.state.AcceptBlock(b)
			if err != nil {
				print("mined a block, but with an error: ")
				println(err.Error())
				m.tpool.PurgeTransactionPool()
			}
			solved = true
			if build.Release != "testing" {
				println("Found a block! If the block is not orphaned, you will receive the reward after 50 more blocks have been mined. Blocks are only orphaned when two miners find a block at the same time, as only 1 block can be accepted.")
			}

			// Grab a new address for the miner.
			m.mu.Lock()
			var addr types.UnlockHash
			addr, _, err = m.wallet.CoinAddress()
			if err == nil { // Special case: only update the address if there was no error.
				m.address = addr
			}
			m.mu.Unlock()
			return
		}
	}

	return
}

// FindBlock will attempt to solve a block and add it to the state. While less
// efficient than StartMining, it is guaranteed to find at most one block.
func (m *Miner) FindBlock() (types.Block, bool, error) {
	m.mu.Lock()
	bfw := m.blockForWork()
	target := m.target
	iterations := m.iterationsPerAttempt
	m.mu.Unlock()

	return m.solveBlock(bfw, target, iterations)
}

// SolveBlock attempts to solve a block, returning the solved block without
// submitting it to the state.
func (m *Miner) SolveBlock(blockForWork types.Block, target types.Target) (b types.Block, solved bool) {
	m.mu.RLock()
	iterations := m.iterationsPerAttempt
	m.mu.RUnlock()

	// Iterate through a bunch of nonces (from a random starting point) and try
	// to find a winnning solution.
	b = blockForWork
	for maxNonce := b.Nonce + iterations; b.Nonce != maxNonce; b.Nonce++ {
		if b.CheckTarget(target) {
			solved = true
			return
		}
	}
	return
}

// StartMining spawns a bunch of mining threads which will mine until stop is
// called.
func (m *Miner) StartMining() error {
	m.mu.Lock()
	defer m.mu.Unlock()

	// Increase the number of threads to m.desiredThreads.
	m.desiredThreads = m.threads
	for i := m.runningThreads; i < m.desiredThreads; i++ {
		go m.threadedMine()
	}

	return nil
}

// StopMining sets desiredThreads to 0, a value which is polled by mining
// threads. When set to 0, the mining threads will all cease mining.
func (m *Miner) StopMining() error {
	m.mu.Lock()
	defer m.mu.Unlock()

	// Set desiredThreads to 0. The miners will shut down automatically.
	m.desiredThreads = 0
	return nil
}
<commit_msg>nicer miner messages<commit_after>package miner

import (
	"bytes"
	"math/rand" // We should probably switch to crypto/rand, but we should use benchmarks first.

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/types"
)

// Creates a block that is ready for nonce grinding.
func (m *Miner) blockForWork() (b types.Block) {
	// Fill out the block with potentially ready values.
	b = types.Block{
		ParentID:     m.parent,
		Timestamp:    types.CurrentTimestamp(),
		Nonce:        uint64(rand.Int()),
		Transactions: m.transactions,
	}

	// Calculate the subsidy and create the miner payout.
	height, exists := m.state.HeightOfBlock(m.parent)
	if !exists {
		if build.DEBUG {
			panic("parent is not in state?")
		}
		return
	}
	subsidy := types.CalculateCoinbase(height + 1)
	for _, txn := range m.transactions {
		for _, fee := range txn.MinerFees {
			subsidy = subsidy.Add(fee)
		}
	}
	output := types.SiacoinOutput{Value: subsidy, UnlockHash: m.address}
	b.MinerPayouts = []types.SiacoinOutput{output}

	// If we've got a time earlier than the earliest legal timestamp, set the
	// timestamp equal to the earliest legal timestamp.
	if b.Timestamp < m.earliestTimestamp {
		b.Timestamp = m.earliestTimestamp
	}

	return
}

// mine attempts to generate blocks, and will run until desiredThreads is
// changd to be lower than `myThread`, which is set at the beginning of the
// function.
//
// The threading is fragile. Edit with caution!
func (m *Miner) threadedMine() {
	// Increment the number of threads running, because this thread is spinning
	// up. Also grab a number that will tell us when to shut down.
	m.mu.Lock()
	m.runningThreads++
	myThread := m.runningThreads
	m.mu.Unlock()

	// Try to solve a block repeatedly.
	for {
		// Grab the number of threads that are supposed to be running.
		m.mu.RLock()
		desiredThreads := m.desiredThreads
		m.mu.RUnlock()

		// If we are allowed to be running, mine a block, otherwise shut down.
		if desiredThreads >= myThread {
			// Grab the necessary variables for mining, and then attempt to
			// mine a block.
			m.mu.RLock()
			bfw := m.blockForWork()
			target := m.target
			iterations := m.iterationsPerAttempt
			m.mu.RUnlock()
			m.solveBlock(bfw, target, iterations)
		} else {
			m.mu.Lock()
			// Need to check the mining status again, something might have
			// changed while waiting for the lock.
			if desiredThreads < myThread {
				m.runningThreads--
				m.mu.Unlock()
				return
			}
			m.mu.Unlock()
		}
	}
}

func fastCheckTarget(target types.Target, b types.Block, bRoot crypto.Hash) bool {
	id := crypto.HashAll(
		b.ParentID,
		b.Nonce,
		bRoot,
	)
	return bytes.Compare(target[:], id[:]) >= 0
}

// solveBlock takes a block, target, and number of iterations as input and
// tries to find a block that meets the target. This function can take a long
// time to complete, and should not be called with a lock.
func (m *Miner) solveBlock(blockForWork types.Block, target types.Target, iterations uint64) (b types.Block, solved bool, err error) {
	// solveBlock could operate on a pointer, but it's not strictly necessary
	// and it makes calling weirder/more opaque.
	b = blockForWork
	bRoot := b.MerkleRoot()

	// Iterate through a bunch of nonces (from a random starting point) and try
	// to find a winnning solution.
	for maxNonce := b.Nonce + iterations; b.Nonce != maxNonce; b.Nonce++ {
		if fastCheckTarget(target, b, bRoot) {
			err = m.state.AcceptBlock(b)
			if err != nil {
				println("Mined a bad block " + err.Error())
				m.tpool.PurgeTransactionPool()
			}
			solved = true
			if build.Release != "testing" {
				println("Found a block! Reward will be received in 50 blocks.")
			}

			// Grab a new address for the miner.
			m.mu.Lock()
			var addr types.UnlockHash
			addr, _, err = m.wallet.CoinAddress()
			if err == nil { // Special case: only update the address if there was no error.
				m.address = addr
			}
			m.mu.Unlock()
			return
		}
	}

	return
}

// FindBlock will attempt to solve a block and add it to the state. While less
// efficient than StartMining, it is guaranteed to find at most one block.
func (m *Miner) FindBlock() (types.Block, bool, error) {
	m.mu.Lock()
	bfw := m.blockForWork()
	target := m.target
	iterations := m.iterationsPerAttempt
	m.mu.Unlock()

	return m.solveBlock(bfw, target, iterations)
}

// SolveBlock attempts to solve a block, returning the solved block without
// submitting it to the state.
func (m *Miner) SolveBlock(blockForWork types.Block, target types.Target) (b types.Block, solved bool) {
	m.mu.RLock()
	iterations := m.iterationsPerAttempt
	m.mu.RUnlock()

	// Iterate through a bunch of nonces (from a random starting point) and try
	// to find a winnning solution.
	b = blockForWork
	for maxNonce := b.Nonce + iterations; b.Nonce != maxNonce; b.Nonce++ {
		if b.CheckTarget(target) {
			solved = true
			return
		}
	}
	return
}

// StartMining spawns a bunch of mining threads which will mine until stop is
// called.
func (m *Miner) StartMining() error {
	m.mu.Lock()
	defer m.mu.Unlock()

	// Increase the number of threads to m.desiredThreads.
	m.desiredThreads = m.threads
	for i := m.runningThreads; i < m.desiredThreads; i++ {
		go m.threadedMine()
	}

	return nil
}

// StopMining sets desiredThreads to 0, a value which is polled by mining
// threads. When set to 0, the mining threads will all cease mining.
func (m *Miner) StopMining() error {
	m.mu.Lock()
	defer m.mu.Unlock()

	// Set desiredThreads to 0. The miners will shut down automatically.
	m.desiredThreads = 0
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Alex Browne.  All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

// File convert_test.go tests the conversion
// to and from go data structures of a variety of types.

package zoom

import (
	"reflect"
	"testing"
	"time"
)

func TestConvertPrimatives(t *testing.T) {
	testingSetUp()
	defer testingTearDown()
	testConvertType(t, indexedPrimativesModels, createIndexedPrimativesModel())
}

func TestConvertPointers(t *testing.T) {
	testingSetUp()
	defer testingTearDown()
	testConvertType(t, indexedPointersModels, createIndexedPointersModel())
}

func TestTimeDuration(t *testing.T) {
	testingSetUp()
	defer testingTearDown()

	type durationModel struct {
		Duration time.Duration
		RandomId
	}
	durationModels, err := testPool.NewCollection(&durationModel{})
	if err != nil {
		t.Errorf("Unexpected error in testPool.NewCollection: %s", err.Error())
	}
	model := &durationModel{
		Duration: 43 * time.Second,
	}
	testConvertType(t, durationModels, model)
}

func TestGobFallback(t *testing.T) {
	testingSetUp()
	defer testingTearDown()

	type gobModel struct {
		Complex     complex128
		IntSlice    []int
		StringSlice []string
		IntArray    [3]int
		StringArray [3]string
		StringMap   map[string]string
		IntMap      map[int]int
		RandomId
	}
	options := DefaultCollectionOptions.WithFallbackMarshalerUnmarshaler(GobMarshalerUnmarshaler)
	gobModels, err := testPool.NewCollectionWithOptions(&gobModel{}, options)
	if err != nil {
		t.Errorf("Unexpected error in testPool.NewCollection: %s", err.Error())
	}
	model := &gobModel{
		Complex:     randomComplex(),
		IntSlice:    []int{randomInt(), randomInt(), randomInt()},
		StringSlice: []string{randomString(), randomString(), randomString()},
		IntArray:    [3]int{randomInt(), randomInt(), randomInt()},
		StringArray: [3]string{randomString(), randomString(), randomString()},
		StringMap:   map[string]string{randomString(): randomString(), randomString(): randomString()},
		IntMap:      map[int]int{randomInt(): randomInt(), randomInt(): randomInt()},
	}
	testConvertType(t, gobModels, model)
}

func TestJSONFallback(t *testing.T) {
	testingSetUp()
	defer testingTearDown()

	type jsonModel struct {
		IntSlice       []int
		StringSlice    []string
		IntArray       [3]int
		StringArray    [3]string
		StringMap      map[string]string
		EmptyInterface interface{}
		RandomId
	}
	options := DefaultCollectionOptions.WithFallbackMarshalerUnmarshaler(JSONMarshalerUnmarshaler)
	jsonModels, err := testPool.NewCollectionWithOptions(&jsonModel{}, options)
	if err != nil {
		t.Errorf("Unexpected error in testPool.NewCollection: %s", err.Error())
	}
	model := &jsonModel{
		IntSlice:       []int{randomInt(), randomInt(), randomInt()},
		StringSlice:    []string{randomString(), randomString(), randomString()},
		IntArray:       [3]int{randomInt(), randomInt(), randomInt()},
		StringArray:    [3]string{randomString(), randomString(), randomString()},
		StringMap:      map[string]string{randomString(): randomString(), randomString(): randomString()},
		EmptyInterface: "This satisfies empty interface",
	}
	testConvertType(t, jsonModels, model)
}

type Embeddable struct {
	Int    int
	String string
	Bool   bool
}

func TestConvertEmbeddedStruct(t *testing.T) {
	testingSetUp()
	defer testingTearDown()

	type embeddedStructModel struct {
		Embeddable
		RandomId
	}
	embededStructModels, err := testPool.NewCollection(&embeddedStructModel{})
	if err != nil {
		t.Errorf("Unexpected error in testPool.NewCollection: %s", err.Error())
	}
	model := &embeddedStructModel{
		Embeddable: Embeddable{
			Int:    randomInt(),
			String: randomString(),
			Bool:   randomBool(),
		},
	}
	testConvertType(t, embededStructModels, model)
}

func TestEmbeddedPointerToStruct(t *testing.T) {
	testingSetUp()
	defer testingTearDown()

	type embeddedPointerToStructModel struct {
		*Embeddable
		RandomId
	}
	embededPointerToStructModels, err := testPool.NewCollection(&embeddedPointerToStructModel{})
	if err != nil {
		t.Errorf("Unexpected error in testPool.NewCollection: %s", err.Error())
	}
	model := &embeddedPointerToStructModel{
		Embeddable: &Embeddable{
			Int:    randomInt(),
			String: randomString(),
			Bool:   randomBool(),
		},
	}
	testConvertType(t, embededPointerToStructModels, model)
}

// testConvertType is a general test that uses reflection. It saves model to the databse then finds it. If
// the found copy does not exactly match the original, it reports an error via t.Error or t.Errorf
func testConvertType(t *testing.T, collection *Collection, model Model) {
	// Make sure we can save the model without errors
	if err := collection.Save(model); err != nil {
		t.Errorf("Unexpected error in Save: %s", err.Error())
	}
	// Find the model from the database and scan it into a new copy
	modelCopy, ok := reflect.New(collection.spec.typ.Elem()).Interface().(Model)
	if !ok {
		t.Fatalf("Unexpected error: Could not convert type %s to Model", collection.spec.typ.String())
	}
	if err := collection.Find(model.ModelId(), modelCopy); err != nil {
		t.Errorf("Unexpected error in Find: %s", err.Error())
	}
	// Make sure the copy equals the original
	if !reflect.DeepEqual(model, modelCopy) {
		t.Errorf("Model of type %T was not saved/retrieved correctly.\nExpected: %+v\nGot:      %+v", model, model, modelCopy)
	}
	// Make sure we can save a model with all nil fields. This should
	// not cause an error.
	emptyModel, ok := reflect.New(collection.spec.typ.Elem()).Interface().(Model)
	if !ok {
		t.Fatalf("Unexpected error: Could not convert type %s to Model", collection.spec.typ.String())
	}
	if err := collection.Save(emptyModel); err != nil {
		t.Errorf("Unexpected error saving an empty model: %s", err.Error())
	}
	emptyModelCopy, ok := reflect.New(collection.spec.typ.Elem()).Interface().(Model)
	if err := collection.Find(emptyModel.ModelId(), emptyModelCopy); err != nil {
		t.Errorf("Unexpected error in Find: %s", err.Error())
	}
	// Make sure the copy equals the original
	if !reflect.DeepEqual(emptyModel, emptyModelCopy) {
		t.Errorf("Model of type %T was not saved/retrieved correctly.\nExpected: %+v\nGot:      %+v", emptyModel, emptyModel, emptyModelCopy)
	}
}
<commit_msg>Improve robustness of convert_test empty interface example<commit_after>// Copyright 2015 Alex Browne.  All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

// File convert_test.go tests the conversion
// to and from go data structures of a variety of types.

package zoom

import (
	"reflect"
	"testing"
	"time"
)

func TestConvertPrimatives(t *testing.T) {
	testingSetUp()
	defer testingTearDown()
	testConvertType(t, indexedPrimativesModels, createIndexedPrimativesModel())
}

func TestConvertPointers(t *testing.T) {
	testingSetUp()
	defer testingTearDown()
	testConvertType(t, indexedPointersModels, createIndexedPointersModel())
}

func TestTimeDuration(t *testing.T) {
	testingSetUp()
	defer testingTearDown()

	type durationModel struct {
		Duration time.Duration
		RandomId
	}
	durationModels, err := testPool.NewCollection(&durationModel{})
	if err != nil {
		t.Errorf("Unexpected error in testPool.NewCollection: %s", err.Error())
	}
	model := &durationModel{
		Duration: 43 * time.Second,
	}
	testConvertType(t, durationModels, model)
}

func TestGobFallback(t *testing.T) {
	testingSetUp()
	defer testingTearDown()

	type gobModel struct {
		Complex     complex128
		IntSlice    []int
		StringSlice []string
		IntArray    [3]int
		StringArray [3]string
		StringMap   map[string]string
		IntMap      map[int]int
		RandomId
	}
	options := DefaultCollectionOptions.WithFallbackMarshalerUnmarshaler(GobMarshalerUnmarshaler)
	gobModels, err := testPool.NewCollectionWithOptions(&gobModel{}, options)
	if err != nil {
		t.Errorf("Unexpected error in testPool.NewCollection: %s", err.Error())
	}
	model := &gobModel{
		Complex:     randomComplex(),
		IntSlice:    []int{randomInt(), randomInt(), randomInt()},
		StringSlice: []string{randomString(), randomString(), randomString()},
		IntArray:    [3]int{randomInt(), randomInt(), randomInt()},
		StringArray: [3]string{randomString(), randomString(), randomString()},
		StringMap:   map[string]string{randomString(): randomString(), randomString(): randomString()},
		IntMap:      map[int]int{randomInt(): randomInt(), randomInt(): randomInt()},
	}
	testConvertType(t, gobModels, model)
}

func TestJSONFallback(t *testing.T) {
	testingSetUp()
	defer testingTearDown()

	type jsonModel struct {
		IntSlice       []int
		StringSlice    []string
		IntArray       [3]int
		StringArray    [3]string
		StringMap      map[string]string
		EmptyInterface interface{}
		RandomId
	}
	options := DefaultCollectionOptions.WithFallbackMarshalerUnmarshaler(JSONMarshalerUnmarshaler)
	jsonModels, err := testPool.NewCollectionWithOptions(&jsonModel{}, options)
	if err != nil {
		t.Errorf("Unexpected error in testPool.NewCollection: %s", err.Error())
	}
	model := &jsonModel{
		IntSlice:       []int{randomInt(), randomInt(), randomInt()},
		StringSlice:    []string{randomString(), randomString(), randomString()},
		IntArray:       [3]int{randomInt(), randomInt(), randomInt()},
		StringArray:    [3]string{randomString(), randomString(), randomString()},
		StringMap:      map[string]string{randomString(): randomString(), randomString(): randomString()},
		EmptyInterface: map[string]interface{}{"key": []interface{}{"This satisfies empty interface"}},
	}
	testConvertType(t, jsonModels, model)
}

type Embeddable struct {
	Int    int
	String string
	Bool   bool
}

func TestConvertEmbeddedStruct(t *testing.T) {
	testingSetUp()
	defer testingTearDown()

	type embeddedStructModel struct {
		Embeddable
		RandomId
	}
	embededStructModels, err := testPool.NewCollection(&embeddedStructModel{})
	if err != nil {
		t.Errorf("Unexpected error in testPool.NewCollection: %s", err.Error())
	}
	model := &embeddedStructModel{
		Embeddable: Embeddable{
			Int:    randomInt(),
			String: randomString(),
			Bool:   randomBool(),
		},
	}
	testConvertType(t, embededStructModels, model)
}

func TestEmbeddedPointerToStruct(t *testing.T) {
	testingSetUp()
	defer testingTearDown()

	type embeddedPointerToStructModel struct {
		*Embeddable
		RandomId
	}
	embededPointerToStructModels, err := testPool.NewCollection(&embeddedPointerToStructModel{})
	if err != nil {
		t.Errorf("Unexpected error in testPool.NewCollection: %s", err.Error())
	}
	model := &embeddedPointerToStructModel{
		Embeddable: &Embeddable{
			Int:    randomInt(),
			String: randomString(),
			Bool:   randomBool(),
		},
	}
	testConvertType(t, embededPointerToStructModels, model)
}

// testConvertType is a general test that uses reflection. It saves model to the databse then finds it. If
// the found copy does not exactly match the original, it reports an error via t.Error or t.Errorf
func testConvertType(t *testing.T, collection *Collection, model Model) {
	// Make sure we can save the model without errors
	if err := collection.Save(model); err != nil {
		t.Errorf("Unexpected error in Save: %s", err.Error())
	}
	// Find the model from the database and scan it into a new copy
	modelCopy, ok := reflect.New(collection.spec.typ.Elem()).Interface().(Model)
	if !ok {
		t.Fatalf("Unexpected error: Could not convert type %s to Model", collection.spec.typ.String())
	}
	if err := collection.Find(model.ModelId(), modelCopy); err != nil {
		t.Errorf("Unexpected error in Find: %s", err.Error())
	}
	// Make sure the copy equals the original
	if !reflect.DeepEqual(model, modelCopy) {
		t.Errorf("Model of type %T was not saved/retrieved correctly.\nExpected: %+v\nGot:      %+v", model, model, modelCopy)
	}
	// Make sure we can save a model with all nil fields. This should
	// not cause an error.
	emptyModel, ok := reflect.New(collection.spec.typ.Elem()).Interface().(Model)
	if !ok {
		t.Fatalf("Unexpected error: Could not convert type %s to Model", collection.spec.typ.String())
	}
	if err := collection.Save(emptyModel); err != nil {
		t.Errorf("Unexpected error saving an empty model: %s", err.Error())
	}
	emptyModelCopy, ok := reflect.New(collection.spec.typ.Elem()).Interface().(Model)
	if err := collection.Find(emptyModel.ModelId(), emptyModelCopy); err != nil {
		t.Errorf("Unexpected error in Find: %s", err.Error())
	}
	// Make sure the copy equals the original
	if !reflect.DeepEqual(emptyModel, emptyModelCopy) {
		t.Errorf("Model of type %T was not saved/retrieved correctly.\nExpected: %+v\nGot:      %+v", emptyModel, emptyModel, emptyModelCopy)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"mime/multipart"
	"os"
	"strings"

	"github.com/rogpeppe/rog-go/reverse"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
)

const (
	defaultLines = 1e5
	maxLines     = 1e6
)

func NewCmdLogSend(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:  "send",
		Usage: "Send recent debug logs to keybase",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdLogSend{Contextified: libkb.NewContextified(g)}, "send", c)
		},
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:  "n",
				Usage: "Number of lines in each log file",
			},
		},
	}
}

type CmdLogSend struct {
	libkb.Contextified
	numLines int
}

func (c *CmdLogSend) Run() error {
	if err := c.confirm(); err != nil {
		return err
	}

	c.G().Log.Debug("getting keybase status")
	statusCmd := &CmdStatus{Contextified: libkb.NewContextified(c.G())}
	status, err := statusCmd.load()
	if err != nil {
		return err
	}
	statusJSON, err := json.Marshal(status)
	if err != nil {
		return err
	}

	c.G().Log.Debug("tailing kbfs log %q", status.KBFS.Log)
	kbfsLog := c.tail(status.KBFS.Log, c.numLines)

	c.G().Log.Debug("tailing service log %q", status.Service.Log)
	svcLog := c.tail(status.Service.Log, c.numLines)

	c.G().Log.Debug("tailing desktop log %q", status.Desktop.Log)
	desktopLog := c.tail(status.Desktop.Log, c.numLines)

	return c.post(string(statusJSON), kbfsLog, svcLog, desktopLog)
}

func (c *CmdLogSend) confirm() error {
	ui := c.G().UI.GetTerminalUI()
	ui.Printf("This command will send recent keybase log entries to keybase.io\n")
	ui.Printf("for debugging purposes only.\n\n")
	ui.Printf("These logs don’t include your private keys or encrypted data,\n")
	ui.Printf("but they will include filenames and other metadata keybase normally\n")
	ui.Printf("can’t read, for debugging purposes.\n")
	return ui.PromptForConfirmation("Continue sending logs to keybase.io?")
}

func (c *CmdLogSend) post(status, kbfsLog, svcLog, desktopLog string) error {
	c.G().Log.Debug("sending status + logs to keybase")

	var body bytes.Buffer
	mpart := multipart.NewWriter(&body)

	if err := addFile(mpart, "status_gz", "status.gz", status); err != nil {
		return err
	}
	if err := addFile(mpart, "kbfs_log_gz", "kbfs_log.gz", kbfsLog); err != nil {
		return err
	}
	if err := addFile(mpart, "keybase_log_gz", "keybase_log.gz", svcLog); err != nil {
		return err
	}
	if err := addFile(mpart, "gui_log_gz", "gui_log.gz", desktopLog); err != nil {
		return err
	}

	if err := mpart.Close(); err != nil {
		return err
	}

	c.G().Log.Debug("body size: %d\n", body.Len())

	arg := libkb.APIArg{
		Contextified: libkb.NewContextified(c.G()),
		Endpoint:     "logdump/send",
	}

	resp, err := c.G().API.PostRaw(arg, mpart.FormDataContentType(), &body)
	if err != nil {
		c.G().Log.Debug("post error: %s", err)
		return err
	}

	id, err := resp.Body.AtKey("logdump_id").GetString()
	if err != nil {
		return err
	}

	c.outputInstructions(id)
	return nil
}

func (c *CmdLogSend) outputInstructions(id string) {
	ui := c.G().UI.GetTerminalUI()
	ui.Printf("keybase.io received your logs successfully! Your log dump ID is:\n\n")
	ui.Printf("\t%s\n\n", id)
	ui.Printf("Here's a URL to submit a bug report containing this ID:\n\n")
	ui.Printf("\thttps://github.com/keybase/keybase-issues/issues/new?body=log+dump+id+%s\n\n", id)
	ui.Printf("Thanks!\n")
}

func addFile(mpart *multipart.Writer, param, filename, data string) error {
	if len(data) == 0 {
		return nil
	}

	part, err := mpart.CreateFormFile(param, filename)
	if err != nil {
		return err
	}
	gz := gzip.NewWriter(part)
	if _, err := gz.Write([]byte(data)); err != nil {
		return err
	}
	if err := gz.Close(); err != nil {
		return err
	}

	return nil
}

func (c *CmdLogSend) tail(filename string, numLines int) string {
	f, err := os.Open(filename)
	if err != nil {
		c.G().Log.Warning("error opening log %q: %s", filename, err)
		return ""
	}
	b := reverse.NewScanner(f)
	b.Split(bufio.ScanLines)

	var lines []string
	for b.Scan() {
		lines = append(lines, b.Text())
		if len(lines) == numLines {
			break
		}
	}

	for left, right := 0, len(lines)-1; left < right; left, right = left+1, right-1 {
		lines[left], lines[right] = lines[right], lines[left]
	}

	return strings.Join(lines, "\n")
}

func (c *CmdLogSend) ParseArgv(ctx *cli.Context) error {
	if len(ctx.Args()) > 0 {
		return UnexpectedArgsError("log send")
	}
	c.numLines = ctx.Int("n")
	if c.numLines < 1 {
		c.numLines = defaultLines
	} else if c.numLines > maxLines {
		c.numLines = maxLines
	}
	return nil
}

func (c *CmdLogSend) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config: true,
		API:    true,
	}
}
<commit_msg>report to keybase/client and not keybase/keybase-issues<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"mime/multipart"
	"os"
	"strings"

	"github.com/rogpeppe/rog-go/reverse"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
)

const (
	defaultLines = 1e5
	maxLines     = 1e6
)

func NewCmdLogSend(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:  "send",
		Usage: "Send recent debug logs to keybase",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdLogSend{Contextified: libkb.NewContextified(g)}, "send", c)
		},
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:  "n",
				Usage: "Number of lines in each log file",
			},
		},
	}
}

type CmdLogSend struct {
	libkb.Contextified
	numLines int
}

func (c *CmdLogSend) Run() error {
	if err := c.confirm(); err != nil {
		return err
	}

	c.G().Log.Debug("getting keybase status")
	statusCmd := &CmdStatus{Contextified: libkb.NewContextified(c.G())}
	status, err := statusCmd.load()
	if err != nil {
		return err
	}
	statusJSON, err := json.Marshal(status)
	if err != nil {
		return err
	}

	c.G().Log.Debug("tailing kbfs log %q", status.KBFS.Log)
	kbfsLog := c.tail(status.KBFS.Log, c.numLines)

	c.G().Log.Debug("tailing service log %q", status.Service.Log)
	svcLog := c.tail(status.Service.Log, c.numLines)

	c.G().Log.Debug("tailing desktop log %q", status.Desktop.Log)
	desktopLog := c.tail(status.Desktop.Log, c.numLines)

	return c.post(string(statusJSON), kbfsLog, svcLog, desktopLog)
}

func (c *CmdLogSend) confirm() error {
	ui := c.G().UI.GetTerminalUI()
	ui.Printf("This command will send recent keybase log entries to keybase.io\n")
	ui.Printf("for debugging purposes only.\n\n")
	ui.Printf("These logs don’t include your private keys or encrypted data,\n")
	ui.Printf("but they will include filenames and other metadata keybase normally\n")
	ui.Printf("can’t read, for debugging purposes.\n")
	return ui.PromptForConfirmation("Continue sending logs to keybase.io?")
}

func (c *CmdLogSend) post(status, kbfsLog, svcLog, desktopLog string) error {
	c.G().Log.Debug("sending status + logs to keybase")

	var body bytes.Buffer
	mpart := multipart.NewWriter(&body)

	if err := addFile(mpart, "status_gz", "status.gz", status); err != nil {
		return err
	}
	if err := addFile(mpart, "kbfs_log_gz", "kbfs_log.gz", kbfsLog); err != nil {
		return err
	}
	if err := addFile(mpart, "keybase_log_gz", "keybase_log.gz", svcLog); err != nil {
		return err
	}
	if err := addFile(mpart, "gui_log_gz", "gui_log.gz", desktopLog); err != nil {
		return err
	}

	if err := mpart.Close(); err != nil {
		return err
	}

	c.G().Log.Debug("body size: %d\n", body.Len())

	arg := libkb.APIArg{
		Contextified: libkb.NewContextified(c.G()),
		Endpoint:     "logdump/send",
	}

	resp, err := c.G().API.PostRaw(arg, mpart.FormDataContentType(), &body)
	if err != nil {
		c.G().Log.Debug("post error: %s", err)
		return err
	}

	id, err := resp.Body.AtKey("logdump_id").GetString()
	if err != nil {
		return err
	}

	c.outputInstructions(id)
	return nil
}

func (c *CmdLogSend) outputInstructions(id string) {
	ui := c.G().UI.GetTerminalUI()
	ui.Printf("keybase.io received your logs successfully! Your log dump ID is:\n\n")
	ui.Printf("\t%s\n\n", id)
	ui.Printf("Here's a URL to submit a bug report containing this ID:\n\n")
	ui.Printf("\thttps://github.com/keybase/client/issues/new?body=log+dump+id+%s\n\n", id)
	ui.Printf("Thanks!\n")
}

func addFile(mpart *multipart.Writer, param, filename, data string) error {
	if len(data) == 0 {
		return nil
	}

	part, err := mpart.CreateFormFile(param, filename)
	if err != nil {
		return err
	}
	gz := gzip.NewWriter(part)
	if _, err := gz.Write([]byte(data)); err != nil {
		return err
	}
	if err := gz.Close(); err != nil {
		return err
	}

	return nil
}

func (c *CmdLogSend) tail(filename string, numLines int) string {
	f, err := os.Open(filename)
	if err != nil {
		c.G().Log.Warning("error opening log %q: %s", filename, err)
		return ""
	}
	b := reverse.NewScanner(f)
	b.Split(bufio.ScanLines)

	var lines []string
	for b.Scan() {
		lines = append(lines, b.Text())
		if len(lines) == numLines {
			break
		}
	}

	for left, right := 0, len(lines)-1; left < right; left, right = left+1, right-1 {
		lines[left], lines[right] = lines[right], lines[left]
	}

	return strings.Join(lines, "\n")
}

func (c *CmdLogSend) ParseArgv(ctx *cli.Context) error {
	if len(ctx.Args()) > 0 {
		return UnexpectedArgsError("log send")
	}
	c.numLines = ctx.Int("n")
	if c.numLines < 1 {
		c.numLines = defaultLines
	} else if c.numLines > maxLines {
		c.numLines = maxLines
	}
	return nil
}

func (c *CmdLogSend) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config: true,
		API:    true,
	}
}
<|endoftext|>


<commit_before>package ioutil2

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"testing"
	"time"
)

func TestWrite(t *testing.T) {
	data := []byte("test string\n")
	fname := fmt.Sprintf("/tmp/atomic-file-test-%v.txt", time.Now().UnixNano())
	err := WriteFileAtomic(fname, data, 0664)
	if err != nil {
		t.Fatal(err)
	}
	rData, err := ioutil.ReadFile(fname)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(data, rData) {
		t.Fatalf("data mismatch: %v != %v")
	}
	if err := os.Remove(fname); err != nil {
		t.Fatal(err)
	}
}
<commit_msg>add missing args to Fatalf call<commit_after>package ioutil2

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"testing"
	"time"
)

func TestWrite(t *testing.T) {
	data := []byte("test string\n")
	fname := fmt.Sprintf("/tmp/atomic-file-test-%v.txt", time.Now().UnixNano())
	err := WriteFileAtomic(fname, data, 0664)
	if err != nil {
		t.Fatal(err)
	}
	rData, err := ioutil.ReadFile(fname)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(data, rData) {
		t.Fatalf("data mismatch: %v != %v", data, rData)
	}
	if err := os.Remove(fname); err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package composition

import (
	"github.com/golang/mock/gomock"
	mockhttp "github.com/tarent/lib-compose/composition/mocks/net/http"
	"net/http"
	"testing"
)

func Test_CacheInvalidationHandler_Invalidation(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	//given
	cacheMocK := NewMockCache(ctrl)
	cih := &CacheInvalidationHandler{cache: cacheMocK}
	request, _ := http.NewRequest(http.MethodDelete, "internal/cache", nil)

	//when
	cacheMocK.EXPECT().Invalidate().Times(1)
	cih.ServeHTTP(nil, request)
}

func Test_CacheInvalidationHandler_Delegate_Is_Called(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	//given
	handlerMock := mockhttp.NewMockHandler(ctrl)
	cacheMocK := NewMockCache(ctrl)
	cih := &CacheInvalidationHandler{cache: cacheMocK, next: handlerMock}
	request, _ := http.NewRequest(http.MethodDelete, "internal/cache", nil)

	//when
	cacheMocK.EXPECT().Invalidate().AnyTimes()
	handlerMock.EXPECT().ServeHTTP(gomock.Any(), gomock.Any()).Times(1)
	cih.ServeHTTP(nil, request)
}
<commit_msg>Change unit tests for cache invalidation handler to include the constructor<commit_after>package composition

import (
	"github.com/golang/mock/gomock"
	mockhttp "github.com/tarent/lib-compose/composition/mocks/net/http"
	"net/http"
	"testing"
)

func Test_CacheInvalidationHandler_Invalidation(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	//given
	cacheMocK := NewMockCache(ctrl)
	cih := NewCacheInvalidationHandler(cacheMocK, nil)
	request, _ := http.NewRequest(http.MethodDelete, "internal/cache", nil)

	//when
	cacheMocK.EXPECT().Invalidate().Times(1)
	cih.ServeHTTP(nil, request)
}

func Test_CacheInvalidationHandler_Delegate_Is_Called(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	//given
	handlerMock := mockhttp.NewMockHandler(ctrl)
	cacheMocK := NewMockCache(ctrl)
	cih := NewCacheInvalidationHandler(cacheMocK, handlerMock)
	request, _ := http.NewRequest(http.MethodDelete, "internal/cache", nil)

	//when
	cacheMocK.EXPECT().Invalidate().AnyTimes()
	handlerMock.EXPECT().ServeHTTP(gomock.Any(), gomock.Any()).Times(1)
	cih.ServeHTTP(nil, request)
}
<|endoftext|>


<commit_before>package core

import (
	"bytes"
	"fmt"
	"html/template"
	"io/ioutil"
	"os"
	"runtime"
	"strings"
	"time"
)

type Instance struct {
	Config      map[string]interface{} `json:"config"`
	DataPathDir string                 `json:"data_path_dir" ` // "dir1,dir2"
	LogPathDir  string                 `json:"log_path_dir"`
}

type Host struct {
	Instances []Instance `json:"instances"`
	HostName  string     `json:"host_name"`
}

type Plugin struct {
	Name    string `json:"name"`
	Version string `json:"version"`
	url     string
}

func (this *Plugin) SetUrl() string {
	hostName, _ := os.Hostname()
	port := os.Getenv("S_PORT")
	this.url = fmt.Sprintf("http://%s:%s/%s-%s.zip", hostName, port, this.Name, this.Version)
	return this.url
}

type Cluster struct {
	Hosts       []Host                 `json:"hosts"`
	ClusterName string                 `json:"cluster_name"`
	Vars        map[string]interface{} `json:"vars"`       // for some common config in a cluster
	JVMConfig   map[string]interface{} `json:"jvm_config"` // for config jvm
	Mode        string                 `json:"mode"`
	DataPathDir []string               `json:"data_path_dir"`
	LogPathDir  string                 `json:"log_path_dir"`
	Plugins     []Plugin               `json:"plugins"`
}

func (this *Cluster) Init() *Cluster {
	initInstanceConfig(this)
	updateInstanceConfig(this)
	return this
}

func Create(name string, hosts []Host, dataPathDir []string, logPathDir string) *Cluster {
	c := Cluster{
		Hosts:       hosts,
		ClusterName: name,
		DataPathDir: dataPathDir,
		LogPathDir:  logPathDir,
		JVMConfig:   map[string]interface{}{"es_heap_size": "20g"},
	}
	return c.Init()
}

// select master node from Hosts
// require: initInstanceConfig(hosts)
func zenPingList(c Cluster) []string {
	hosts := c.Hosts
	zenPingList := []string{}
	for _, h := range hosts {
		for _, i := range h.Instances {
			isMaster, ok := i.Config["node.master"]
			if ok && isMaster == true {
				connUrl := fmt.Sprintf("%s:%d", h.HostName, i.Config["transport.tcp.port"])
				zenPingList = append(zenPingList, connUrl)
			}
		}
	}
	return zenPingList
}

func updateInstanceConfig(c *Cluster) {
	list := strings.Join(zenPingList(*c), ",")
	for sh, h := range c.Hosts {
		for serial, i := range h.Instances {
			i.Config["discovery.zen.ping.unicast.hosts"] = list
			h.Instances[serial] = i
		}
		c.Hosts[sh] = h
	}
}

// init for some special config
// node.maser
// http.port
// transport.tcp.port
func initInstanceConfig(c *Cluster) {
	// standalone mode: just use runtime.
	// multi-node cluster:  use metadata from agent
	var cpu int = 2
	if c.Mode == StandaloneMode {
		cpu = runtime.NumCPU()
	}
	c.JVMConfig = map[string]interface{}{"es_heap_size": "5g"}
	for sh, h := range c.Hosts {
		processorMax := cpu / len(h.Instances)
		if processorMax < 1 {
			processorMax = 1
		}
		for serial, i := range h.Instances {
			config := make(map[string]interface{})
			for k, v := range commonConfig {
				config[k] = v
			}
			for k, v := range i.Config {
				config[k] = v
			}
			var httpPort = 9200 + serial
			var transPort = 9300 + serial
			config["http.port"] = httpPort
			config["transport.tcp.port"] = transPort

			if _, ok := config["processors"]; !ok {
				config["processors"] = processorMax
			}
			if _, ok := config["node.master"]; !ok {
				config["node.master"] = true
			}

			if i.DataPathDir == "" {
				i.DataPathDir = strings.Join(dataPathAllocation(c.DataPathDir, serial, len(h.Instances)), ",")
			}
			if i.LogPathDir == "" {
				i.LogPathDir = c.LogPathDir
			}
			i.Config = config
			h.Instances[serial] = i
			c.Hosts[sh] = h
		}
	}
}

func dataPathAllocation(dataPathDir []string, index int, total int) []string {
	avg := len(dataPathDir) / total
	start := index * avg
	return dataPathDir[start : start+avg]
}

func (this *Cluster) generateAnsibleYml(cacheDir string, templateFile string) (string, error) {

	var err error
	var path string
	var parsedTemplate *template.Template
	parsedTemplate, err = template.ParseFiles(templateFile)
	if err != nil {
		return path, err
	}

	templateBuff := new(bytes.Buffer)
	err = parsedTemplate.Execute(templateBuff, this)
	if err != nil {
		return path, err
	}

	uid := fmt.Sprintf("%s-%d", this.ClusterName, time.Now().Unix())
	path = fmt.Sprintf("%s/%s", cacheDir, uid)
	err = os.MkdirAll(path, 0755)
	if err != nil {
		return path, err
	}

	fileName := fmt.Sprintf("%s/%s", path, DefaultYmlFile)
	return uid, ioutil.WriteFile(fileName, templateBuff.Bytes(), 0755)
}

func (this *Cluster) CreateConfigFile() (string, error) {
	err := this.updateHosts()
	if err != nil {
		return "", fmt.Errorf("err:%v, msg:%s", err, "fail to update ansible hosts file. stop create ansible yml.")
	}
	return this.generateAnsibleYml(DefaultCacheDir, DefaultYmlFile)
}

func (this *Cluster) updateHosts() error {
	hostNames := map[string]bool{}

	for _, h := range this.Hosts {
		hostNames[h.HostName] = true
	}

	// TODO before write add a lock && check lock
	f, err := os.OpenFile(DefaultHostFile, os.O_APPEND|os.O_WRONLY, 0600)
	if err != nil {
		return err
	}
	defer f.Close()
	for k := range hostNames {
		var erri error
		_, erri = f.WriteString(k)
		if erri != nil {
			return erri
		}
	}
	return nil

}
<commit_msg>add lock for hosts update<commit_after>package core

import (
	"bufio"
	"bytes"
	"fmt"
	"html/template"
	"io/ioutil"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"
)

type Instance struct {
	Config      map[string]interface{} `json:"config"`
	DataPathDir string                 `json:"data_path_dir" ` // "dir1,dir2"
	LogPathDir  string                 `json:"log_path_dir"`
}

type Host struct {
	Instances []Instance `json:"instances"`
	HostName  string     `json:"host_name"`
}

type Plugin struct {
	Name    string `json:"name"`
	Version string `json:"version"`
	url     string
}

func (this *Plugin) SetUrl() string {
	hostName, _ := os.Hostname()
	port := os.Getenv("S_PORT")
	this.url = fmt.Sprintf("http://%s:%s/%s-%s.zip", hostName, port, this.Name, this.Version)
	return this.url
}

type Cluster struct {
	Hosts       []Host                 `json:"hosts"`
	ClusterName string                 `json:"cluster_name"`
	Vars        map[string]interface{} `json:"vars"`       // for some common config in a cluster
	JVMConfig   map[string]interface{} `json:"jvm_config"` // for config jvm
	Mode        string                 `json:"mode"`
	DataPathDir []string               `json:"data_path_dir"`
	LogPathDir  string                 `json:"log_path_dir"`
	Plugins     []Plugin               `json:"plugins"`
}

func (this *Cluster) Init() *Cluster {
	initInstanceConfig(this)
	updateInstanceConfig(this)
	return this
}

func Create(name string, hosts []Host, dataPathDir []string, logPathDir string) *Cluster {
	c := Cluster{
		Hosts:       hosts,
		ClusterName: name,
		DataPathDir: dataPathDir,
		LogPathDir:  logPathDir,
		JVMConfig:   map[string]interface{}{"es_heap_size": "20g"},
	}
	return c.Init()
}

// select master node from Hosts
// require: initInstanceConfig(hosts)
func zenPingList(c Cluster) []string {
	hosts := c.Hosts
	zenPingList := []string{}
	for _, h := range hosts {
		for _, i := range h.Instances {
			isMaster, ok := i.Config["node.master"]
			if ok && isMaster == true {
				connUrl := fmt.Sprintf("%s:%d", h.HostName, i.Config["transport.tcp.port"])
				zenPingList = append(zenPingList, connUrl)
			}
		}
	}
	return zenPingList
}

func updateInstanceConfig(c *Cluster) {
	list := strings.Join(zenPingList(*c), ",")
	for sh, h := range c.Hosts {
		for serial, i := range h.Instances {
			i.Config["discovery.zen.ping.unicast.hosts"] = list
			h.Instances[serial] = i
		}
		c.Hosts[sh] = h
	}
}

// init for some special config
// node.maser
// http.port
// transport.tcp.port
func initInstanceConfig(c *Cluster) {
	// standalone mode: just use runtime.
	// multi-node cluster:  use metadata from agent
	var cpu int = 2
	if c.Mode == StandaloneMode {
		cpu = runtime.NumCPU()
	}
	c.JVMConfig = map[string]interface{}{"es_heap_size": "5g"}
	for sh, h := range c.Hosts {
		processorMax := cpu / len(h.Instances)
		if processorMax < 1 {
			processorMax = 1
		}
		for serial, i := range h.Instances {
			config := make(map[string]interface{})
			for k, v := range commonConfig {
				config[k] = v
			}
			for k, v := range i.Config {
				config[k] = v
			}
			var httpPort = 9200 + serial
			var transPort = 9300 + serial
			config["http.port"] = httpPort
			config["transport.tcp.port"] = transPort

			if _, ok := config["processors"]; !ok {
				config["processors"] = processorMax
			}
			if _, ok := config["node.master"]; !ok {
				config["node.master"] = true
			}

			if i.DataPathDir == "" {
				i.DataPathDir = strings.Join(dataPathAllocation(c.DataPathDir, serial, len(h.Instances)), ",")
			}
			if i.LogPathDir == "" {
				i.LogPathDir = c.LogPathDir
			}
			i.Config = config
			h.Instances[serial] = i
			c.Hosts[sh] = h
		}
	}
}

func dataPathAllocation(dataPathDir []string, index int, total int) []string {
	avg := len(dataPathDir) / total
	start := index * avg
	return dataPathDir[start : start+avg]
}

func (this *Cluster) generateAnsibleYml(cacheDir string, templateFile string) (string, error) {

	var err error
	var path string
	var parsedTemplate *template.Template
	parsedTemplate, err = template.ParseFiles(templateFile)
	if err != nil {
		return path, err
	}

	templateBuff := new(bytes.Buffer)
	err = parsedTemplate.Execute(templateBuff, this)
	if err != nil {
		return path, err
	}

	uid := fmt.Sprintf("%s-%d", this.ClusterName, time.Now().Unix())
	path = fmt.Sprintf("%s/%s", cacheDir, uid)
	err = os.MkdirAll(path, 0755)
	if err != nil {
		return path, err
	}

	fileName := fmt.Sprintf("%s/%s", path, DefaultYmlFile)
	return uid, ioutil.WriteFile(fileName, templateBuff.Bytes(), 0755)
}

func (this *Cluster) CreateConfigFile() (string, error) {
	err := this.updateHosts()
	if err != nil {
		return "", fmt.Errorf("err:%v, msg:%s", err, "fail to update ansible hosts file. stop create ansible yml.")
	}
	return this.generateAnsibleYml(DefaultCacheDir, DefaultYmlFile)
}

var lock sync.Mutex = sync.Mutex{}

func (this *Cluster) updateHosts() error {
	hostNames := map[string]bool{}

	savedHostName := map[string]bool{}

	lock.Lock()
	defer lock.Unlock()
	// TODO before write add a lock && check lock
	f, err := os.OpenFile(DefaultHostFile, os.O_APPEND|os.O_WRONLY, 0600)
	if err != nil {
		return err
	}

	r := bufio.NewReader(f)
	var (
		isPrefix bool = true
		line     []byte
	)
	for isPrefix && err == nil {
		line, isPrefix, err = r.ReadLine()
		savedHostName[string(line)] = true
	}

	for _, h := range this.Hosts {
		if _, found := savedHostName[h.HostName]; found {
			continue
		}
		hostNames[h.HostName] = true
	}

	defer f.Close()
	for k := range hostNames {
		var erri error
		_, erri = f.WriteString(k)
		if erri != nil {
			return erri
		}
	}
	return nil

}
<|endoftext|>


<commit_before>package core

import (
	"github.com/Sirupsen/logrus"
	"path/filepath"
	"pfi/sensorbee/sensorbee/data"
	"runtime"
	"sync"
	"sync/atomic"
)

var (
	temporaryIDCounter int64
)

// NewTemporaryID returns the new temporary 63bit ID. This can be used for
// any purpose.
func NewTemporaryID() int64 {
	return atomic.AddInt64(&temporaryIDCounter, 1)
}

// Context holds a set of functionality that is made available to each Topology
// at runtime. A context is created by the user before creating a Topology.
// Each Context is tied to one Topology and it must not be used by multiple
// topologies.
type Context struct {
	logger       *logrus.Logger
	topologyName string
	Flags        ContextFlags
	SharedStates SharedStateRegistry

	dtMutex   sync.RWMutex
	dtSources map[int64]*droppedTupleCollectorSource
}

// ContextConfig has configuration parameters of a Context.
type ContextConfig struct {
	// Logger provides a logrus's logger used by the Context.
	Logger *logrus.Logger
	Flags  ContextFlags
}

// NewContext creates a new Context based on the config. If config is nil,
// the default config will be used.
func NewContext(config *ContextConfig) *Context {
	if config == nil {
		config = &ContextConfig{}
	}
	logger := config.Logger
	if logger == nil {
		logger = logrus.StandardLogger()
	}
	c := &Context{
		logger:    logger,
		Flags:     config.Flags,
		dtSources: map[int64]*droppedTupleCollectorSource{},
	}
	c.SharedStates = NewDefaultSharedStateRegistry(c)
	return c
}

// Log returns the logger tied to the Context.
func (c *Context) Log() *logrus.Entry {
	return c.log(1)
}

// ErrLog returns the logger tied to the Context having an error information.
func (c *Context) ErrLog(err error) *logrus.Entry {
	return c.log(1).WithField("err", err)
}

func (c *Context) log(depth int) *logrus.Entry {
	// TODO: This is a temporary solution until logrus support filename and line number
	_, file, line, ok := runtime.Caller(depth + 1)
	if !ok {
		return c.logger.WithField("topology", c.topologyName)
	}
	file = filepath.Base(file) // only the filename at the moment
	return c.logger.WithFields(logrus.Fields{
		"file":     file,
		"line":     line,
		"topology": c.topologyName,
	})
}

// droppedTuple records tuples dropped by errors.
func (c *Context) droppedTuple(t *Tuple, nodeType NodeType, nodeName string, et EventType, err error) {
	if t.Flags.IsSet(TFDropped) {
		return // avoid infinite reporting
	}

	if c.Flags.DroppedTupleLog.Enabled() {
		var js string
		if c.Flags.DroppedTupleSummarization.Enabled() {
			js = data.Summarize(t.Data)
		} else {
			js = t.Data.String()
		}

		l := c.Log().WithFields(nodeLogFields(nodeType, nodeName)).WithFields(logrus.Fields{
			"event_type": et.String(),
			"tuple": logrus.Fields{
				"timestamp": data.Timestamp(t.Timestamp),
				"data":      js,
				// TODO: Add trace
			},
		})
		if err != nil {
			l = l.WithField("err", err)
		}
		l.Info("A tuple was dropped from the topology") // TODO: debug should be better?
	}

	c.dtMutex.RLock()
	defer c.dtMutex.RUnlock()
	if len(c.dtSources) == 0 {
		return
	}
	// TODO: reduce copies
	dt := t.Copy()
	dt.Data = data.Map{
		"node_type":  data.String(nodeType.String()),
		"node_name":  data.String(nodeName),
		"event_type": data.String(et.String()),
		"data":       dt.Data,
	}
	if err != nil {
		dt.Data["error"] = data.String(err.Error())
	}
	dt.Flags.Set(TFDropped)
	shouldCopy := len(c.dtSources) > 1
	for _, s := range c.dtSources {
		// TODO: reduce copies
		copied := dt
		if shouldCopy {
			copied = dt.Copy()
		}
		s.w.Write(c, copied) // There isn't much meaning to report errors here.
	}
}

// addDroppedTupleSource adds a listener which receives dropped tuples. The
// return value is the ID of the listener and it'll be required for
// removeDroppedTupleListener.
func (c *Context) addDroppedTupleSource(s *droppedTupleCollectorSource) int64 {
	c.dtMutex.Lock()
	defer c.dtMutex.Unlock()
	id := NewTemporaryID()
	c.dtSources[id] = s
	return id
}

func (c *Context) removeDroppedTupleSource(id int64) {
	c.dtMutex.Lock()
	defer c.dtMutex.Unlock()
	delete(c.dtSources, id)
}

// AtomicFlag is a boolean flag which can be read/written atomically.
type AtomicFlag int32

// Set sets a boolean value to the flag.
func (a *AtomicFlag) Set(b bool) {
	var i int32
	if b {
		i = 1
	}
	atomic.StoreInt32((*int32)(a), i)
}

// Enabled returns true if the flag is enabled.
func (a *AtomicFlag) Enabled() bool {
	return atomic.LoadInt32((*int32)(a)) != 0
}

// ContextFlags is an arrangement of SensorBee processing settings.
type ContextFlags struct {
	// TupleTrace is a Tuple's tracing on/off flag. If the flag is 0
	// (means false), a topology does not trace Tuple's events.
	// The flag can be set when creating a Context, or when the topology
	// is running. In the latter case, Context.TupleTrace.Set() should
	// be used for thread safety.
	// There is a delay between setting the flag and start/stop to trace Tuples.
	TupleTrace AtomicFlag

	// DroppedTupleLog is a flag which turns on/off logging of dropped tuple
	// events.
	DroppedTupleLog AtomicFlag

	// DroppedTupleSummarization is a flag to trun on/off summarization of
	// dropped tuple logging. If this flag is enabled, tuples being logged will
	// be a little smaller than the originals. However, they might not be parsed
	// as JSONs. If the flag is disabled, output JSONs can be parsed.
	DroppedTupleSummarization AtomicFlag
}

type droppedTupleCollectorSource struct {
	w        Writer
	id       int64
	m        sync.Mutex
	gsCond   *sync.Cond
	stopCond *sync.Cond
}

// NewDroppedTupleCollectorSource returns a source which generates a stream
// containing tuples dropped by other nodes. Tuples generated from this source
// won't be reported again even if they're dropped later on. This is done by
// setting TFDropped flag to the dropped tuple. Therefore, if a Box forgets to
// copy the flag when it emits a tuple derived from the dropped one, the tuple
// can infinitely be reported again and again.
//
// Tuples generated from this source has the following fields in Data:
//
//	- node_type: the type of the node which dropped the tuple
//	- node_name: the name of the node which dropped the tuple
//	- event_type: the type of the event indicating when the tuple was dropped
//	- error(optional): the error information if any
//	- data: the original content in which the dropped tuple had
func NewDroppedTupleCollectorSource() Source {
	src := &droppedTupleCollectorSource{}
	src.gsCond = sync.NewCond(&src.m)
	src.stopCond = sync.NewCond(&src.m)
	return src
}

func (s *droppedTupleCollectorSource) GenerateStream(ctx *Context, w Writer) error {
	s.m.Lock()
	defer s.m.Unlock()
	s.w = w
	s.id = ctx.addDroppedTupleSource(s)
	s.stopCond.Broadcast()
	s.gsCond.Wait()
	return nil
}

func (s *droppedTupleCollectorSource) Stop(ctx *Context) error {
	s.m.Lock()
	defer s.m.Unlock()
	// wait until GenerateStream() is called
	for s.w == nil {
		s.stopCond.Wait()
	}
	ctx.removeDroppedTupleSource(s.id)
	s.gsCond.Signal()
	return nil
}
<commit_msg>Improve droppedTupleCollectorSource by using topologyStateHolder.<commit_after>package core

import (
	"errors"
	"github.com/Sirupsen/logrus"
	"path/filepath"
	"pfi/sensorbee/sensorbee/data"
	"runtime"
	"sync"
	"sync/atomic"
)

var (
	temporaryIDCounter int64
)

// NewTemporaryID returns the new temporary 63bit ID. This can be used for
// any purpose.
func NewTemporaryID() int64 {
	return atomic.AddInt64(&temporaryIDCounter, 1)
}

// Context holds a set of functionality that is made available to each Topology
// at runtime. A context is created by the user before creating a Topology.
// Each Context is tied to one Topology and it must not be used by multiple
// topologies.
type Context struct {
	logger       *logrus.Logger
	topologyName string
	Flags        ContextFlags
	SharedStates SharedStateRegistry

	dtMutex   sync.RWMutex
	dtSources map[int64]*droppedTupleCollectorSource
}

// ContextConfig has configuration parameters of a Context.
type ContextConfig struct {
	// Logger provides a logrus's logger used by the Context.
	Logger *logrus.Logger
	Flags  ContextFlags
}

// NewContext creates a new Context based on the config. If config is nil,
// the default config will be used.
func NewContext(config *ContextConfig) *Context {
	if config == nil {
		config = &ContextConfig{}
	}
	logger := config.Logger
	if logger == nil {
		logger = logrus.StandardLogger()
	}
	c := &Context{
		logger:    logger,
		Flags:     config.Flags,
		dtSources: map[int64]*droppedTupleCollectorSource{},
	}
	c.SharedStates = NewDefaultSharedStateRegistry(c)
	return c
}

// Log returns the logger tied to the Context.
func (c *Context) Log() *logrus.Entry {
	return c.log(1)
}

// ErrLog returns the logger tied to the Context having an error information.
func (c *Context) ErrLog(err error) *logrus.Entry {
	return c.log(1).WithField("err", err)
}

func (c *Context) log(depth int) *logrus.Entry {
	// TODO: This is a temporary solution until logrus support filename and line number
	_, file, line, ok := runtime.Caller(depth + 1)
	if !ok {
		return c.logger.WithField("topology", c.topologyName)
	}
	file = filepath.Base(file) // only the filename at the moment
	return c.logger.WithFields(logrus.Fields{
		"file":     file,
		"line":     line,
		"topology": c.topologyName,
	})
}

// droppedTuple records tuples dropped by errors.
func (c *Context) droppedTuple(t *Tuple, nodeType NodeType, nodeName string, et EventType, err error) {
	if t.Flags.IsSet(TFDropped) {
		return // avoid infinite reporting
	}

	if c.Flags.DroppedTupleLog.Enabled() {
		var js string
		if c.Flags.DroppedTupleSummarization.Enabled() {
			js = data.Summarize(t.Data)
		} else {
			js = t.Data.String()
		}

		l := c.Log().WithFields(nodeLogFields(nodeType, nodeName)).WithFields(logrus.Fields{
			"event_type": et.String(),
			"tuple": logrus.Fields{
				"timestamp": data.Timestamp(t.Timestamp),
				"data":      js,
				// TODO: Add trace
			},
		})
		if err != nil {
			l = l.WithField("err", err)
		}
		l.Info("A tuple was dropped from the topology") // TODO: debug should be better?
	}

	c.dtMutex.RLock()
	defer c.dtMutex.RUnlock()
	if len(c.dtSources) == 0 {
		return
	}
	// TODO: reduce copies
	dt := t.Copy()
	dt.Data = data.Map{
		"node_type":  data.String(nodeType.String()),
		"node_name":  data.String(nodeName),
		"event_type": data.String(et.String()),
		"data":       dt.Data,
	}
	if err != nil {
		dt.Data["error"] = data.String(err.Error())
	}
	dt.Flags.Set(TFDropped)
	shouldCopy := len(c.dtSources) > 1
	for _, s := range c.dtSources {
		// TODO: reduce copies
		copied := dt
		if shouldCopy {
			copied = dt.Copy()
		}
		s.w.Write(c, copied) // There isn't much meaning to report errors here.
	}
}

// addDroppedTupleSource adds a listener which receives dropped tuples. The
// return value is the ID of the listener and it'll be required for
// removeDroppedTupleListener.
func (c *Context) addDroppedTupleSource(s *droppedTupleCollectorSource) int64 {
	c.dtMutex.Lock()
	defer c.dtMutex.Unlock()
	id := NewTemporaryID()
	c.dtSources[id] = s
	return id
}

func (c *Context) removeDroppedTupleSource(id int64) {
	c.dtMutex.Lock()
	defer c.dtMutex.Unlock()
	delete(c.dtSources, id)
}

// AtomicFlag is a boolean flag which can be read/written atomically.
type AtomicFlag int32

// Set sets a boolean value to the flag.
func (a *AtomicFlag) Set(b bool) {
	var i int32
	if b {
		i = 1
	}
	atomic.StoreInt32((*int32)(a), i)
}

// Enabled returns true if the flag is enabled.
func (a *AtomicFlag) Enabled() bool {
	return atomic.LoadInt32((*int32)(a)) != 0
}

// ContextFlags is an arrangement of SensorBee processing settings.
type ContextFlags struct {
	// TupleTrace is a Tuple's tracing on/off flag. If the flag is 0
	// (means false), a topology does not trace Tuple's events.
	// The flag can be set when creating a Context, or when the topology
	// is running. In the latter case, Context.TupleTrace.Set() should
	// be used for thread safety.
	// There is a delay between setting the flag and start/stop to trace Tuples.
	TupleTrace AtomicFlag

	// DroppedTupleLog is a flag which turns on/off logging of dropped tuple
	// events.
	DroppedTupleLog AtomicFlag

	// DroppedTupleSummarization is a flag to trun on/off summarization of
	// dropped tuple logging. If this flag is enabled, tuples being logged will
	// be a little smaller than the originals. However, they might not be parsed
	// as JSONs. If the flag is disabled, output JSONs can be parsed.
	DroppedTupleSummarization AtomicFlag
}

type droppedTupleCollectorSource struct {
	w     Writer
	id    int64
	m     sync.Mutex
	state *topologyStateHolder
}

// NewDroppedTupleCollectorSource returns a source which generates a stream
// containing tuples dropped by other nodes. Tuples generated from this source
// won't be reported again even if they're dropped later on. This is done by
// setting TFDropped flag to the dropped tuple. Therefore, if a Box forgets to
// copy the flag when it emits a tuple derived from the dropped one, the tuple
// can infinitely be reported again and again.
//
// Tuples generated from this source has the following fields in Data:
//
//	- node_type: the type of the node which dropped the tuple
//	- node_name: the name of the node which dropped the tuple
//	- event_type: the type of the event indicating when the tuple was dropped
//	- error(optional): the error information if any
//	- data: the original content in which the dropped tuple had
func NewDroppedTupleCollectorSource() Source {
	src := &droppedTupleCollectorSource{}
	src.state = newTopologyStateHolder(&src.m)
	return src
}

func (s *droppedTupleCollectorSource) GenerateStream(ctx *Context, w Writer) error {
	s.m.Lock()
	defer s.m.Unlock()
	if s.state.getWithoutLock() >= TSStopping {
		return errors.New("the source is already stopped")
	}
	s.w = w
	s.id = ctx.addDroppedTupleSource(s)
	s.state.setWithoutLock(TSRunning)
	defer s.state.setWithoutLock(TSStopped)
	s.state.waitWithoutLock(TSStopping)
	return nil
}

func (s *droppedTupleCollectorSource) Stop(ctx *Context) error {
	s.m.Lock()
	defer s.m.Unlock()
	switch s.state.getWithoutLock() {
	case TSStopping:
		s.state.waitWithoutLock(TSStopped)
		return nil
	case TSStopped:
		return nil
	}
	ctx.removeDroppedTupleSource(s.id)
	s.state.setWithoutLock(TSStopping)
	s.state.waitWithoutLock(TSStopped)
	return nil
}
<|endoftext|>


<commit_before>package srnd

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
)

type NullCache struct {
	handler *nullHandler
}

func (self *NullCache) InvertPagination() {
	self.handler.invertPagination = true
}

type nullHandler struct {
	database         Database
	attachments      bool
	requireCaptcha   bool
	name             string
	prefix           string
	translations     string
	i18n             map[string]*I18N
	access           sync.Mutex
	invertPagination bool
}

func (self *nullHandler) ForEachI18N(v func(string)) {
	self.access.Lock()
	for lang := range self.i18n {
		v(lang)
	}
	self.access.Unlock()
}

func (self *nullHandler) GetI18N(r *http.Request) *I18N {
	lang := r.URL.Query().Get("lang")
	if lang == "" {
		lang = I18nProvider.Name
	}
	self.access.Lock()
	i, ok := self.i18n[lang]
	if !ok {
		var err error
		i, err = NewI18n(lang, self.translations)
		if err != nil {
			log.Println(err)
		}
		if i != nil {
			self.i18n[lang] = i
		}
	}
	self.access.Unlock()
	return i
}

func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	i18n := self.GetI18N(r)

	path := r.URL.Path
	_, file := filepath.Split(path)

	isjson := strings.HasSuffix(path, "/json") || strings.HasSuffix(path, "/json/")

	if strings.HasPrefix(path, "/t/") {
		// thread handler
		parts := strings.Split(path[3:], "/")
		hash := parts[0]
		msg, err := self.database.GetMessageIDByHash(hash)
		if err == nil {

			if !self.database.HasArticleLocal(msg.MessageID()) {
				goto notfound
			}

			template.genThread(self.attachments, self.requireCaptcha, msg, self.prefix, self.name, w, self.database, isjson, i18n)
			return
		} else {
			goto notfound
		}
	}
	if strings.Trim(path, "/") == "overboard" {
		// generate ukko aka overboard
		template.genUkko(self.prefix, self.name, w, self.database, isjson, i18n, self.invertPagination)
		return
	}

	if strings.HasPrefix(path, "/b/") {
		// board handler
		parts := strings.Split(path[3:], "/")
		page := 0
		group := parts[0]
		if len(parts) > 1 && parts[1] != "" && parts[1] != "json" {
			var err error
			page, err = strconv.Atoi(parts[1])
			if err != nil {
				goto notfound
			}
		}
		if group == "" {
			goto notfound
		}
		hasgroup := self.database.HasNewsgroup(group)
		if !hasgroup {
			goto notfound
		}

		banned, _ := self.database.NewsgroupBanned(group)
		if banned {
			goto notfound
		}

		pages := self.database.GetGroupPageCount(group)
		if page >= int(pages) {
			goto notfound
		}

		template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination)
		return
	}

	if strings.HasPrefix(path, "/o/") {
		page := 0
		parts := strings.Split(path[3:], "/")
		if parts[0] != "json" && parts[0] != "" {
			var err error
			page, err = strconv.Atoi(parts[0])
			if err != nil {
				goto notfound
			}
		}
		template.genUkkoPaginated(self.prefix, self.name, w, self.database, page, isjson, i18n, self.invertPagination)
		return
	}

	if len(file) == 0 || file == "index.html" {
		template.genFrontPage(10, self.prefix, self.name, w, ioutil.Discard, self.database, i18n)
		return
	}

	if file == "index.json" {
		// TODO: index.json
		goto notfound
	}
	if strings.HasPrefix(file, "history.html") {
		template.genGraphs(self.prefix, w, self.database, i18n)
		return
	}
	if strings.HasPrefix(file, "boards.html") {
		template.genBoardList(self.prefix, self.name, w, self.database, i18n)
		return
	}

	if strings.HasPrefix(file, "boards.json") {
		b := self.database.GetAllNewsgroups()
		json.NewEncoder(w).Encode(b)
		return
	}

	if strings.HasPrefix(file, "ukko.html") {
		template.genUkko(self.prefix, self.name, w, self.database, false, i18n, self.invertPagination)
		return
	}
	if strings.HasPrefix(file, "ukko.json") {
		template.genUkko(self.prefix, self.name, w, self.database, true, i18n, self.invertPagination)
		return
	}

	if strings.HasPrefix(file, "ukko-") {
		page := getUkkoPage(file)
		template.genUkkoPaginated(self.prefix, self.name, w, self.database, page, isjson, i18n, self.invertPagination)
		return
	}
	if strings.HasPrefix(file, "thread-") {
		hash := getThreadHash(file)
		if len(hash) == 0 {
			goto notfound
		}

		msg, err := self.database.GetMessageIDByHash(hash)
		if err != nil {
			goto notfound
		}

		if !self.database.HasArticleLocal(msg.MessageID()) {
			goto notfound
		}

		template.genThread(self.attachments, self.requireCaptcha, msg, self.prefix, self.name, w, self.database, isjson, i18n)
		return
	}
	if strings.HasPrefix(file, "catalog-") {
		group := getGroupForCatalog(file)
		if len(group) == 0 {
			goto notfound
		}
		hasgroup := self.database.HasNewsgroup(group)
		if !hasgroup {
			goto notfound
		}
		template.genCatalog(self.prefix, self.name, group, w, self.database, i18n)
		return
	} else {
		group, page := getGroupAndPage(file)
		if len(group) == 0 || page < 0 {
			goto notfound
		}
		hasgroup := self.database.HasNewsgroup(group)
		if !hasgroup {
			goto notfound
		}
		pages := self.database.GetGroupPageCount(group)
		if page >= int(pages) {
			goto notfound
		}
		template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination)
		return
	}

notfound:
	template.renderNotFound(w, r, self.prefix, self.name, i18n)
}

func (self *NullCache) DeleteBoardMarkup(group string) {
}

// try to delete root post's page
func (self *NullCache) DeleteThreadMarkup(root_post_id string) {
}

// regen every newsgroup
func (self *NullCache) RegenAll() {
}

func (self *NullCache) RegenFrontPage() {
}

func (self *NullCache) SetRequireCaptcha(required bool) {
	self.handler.requireCaptcha = required
}

// regen every page of the board
func (self *NullCache) RegenerateBoard(group string) {
}

// regenerate pages after a mod event
func (self *NullCache) RegenOnModEvent(newsgroup, msgid, root string, page int) {
}

func (self *NullCache) Start() {
}

func (self *NullCache) Regen(msg ArticleEntry) {
}

func (self *NullCache) GetHandler() CacheHandler {
	return self.handler
}

func (self *NullCache) Close() {
	//nothig to do
}

func NewNullCache(prefix, webroot, name, translations string, attachments bool, db Database, store ArticleStore) CacheInterface {
	cache := new(NullCache)
	cache.handler = &nullHandler{
		prefix:         prefix,
		name:           name,
		attachments:    attachments,
		requireCaptcha: true,
		database:       db,
		i18n:           make(map[string]*I18N),
		translations:   translations,
	}

	return cache
}
<commit_msg>fixes<commit_after>package srnd

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
)

type NullCache struct {
	handler *nullHandler
}

func (self *NullCache) InvertPagination() {
	self.handler.invertPagination = true
}

type nullHandler struct {
	database         Database
	attachments      bool
	requireCaptcha   bool
	name             string
	prefix           string
	translations     string
	i18n             map[string]*I18N
	access           sync.Mutex
	invertPagination bool
}

func (self *nullHandler) ForEachI18N(v func(string)) {
	self.access.Lock()
	for lang := range self.i18n {
		v(lang)
	}
	self.access.Unlock()
}

func (self *nullHandler) GetI18N(r *http.Request) *I18N {
	lang := r.URL.Query().Get("lang")
	if lang == "" {
		lang = I18nProvider.Name
	}
	self.access.Lock()
	i, ok := self.i18n[lang]
	if !ok {
		var err error
		i, err = NewI18n(lang, self.translations)
		if err != nil {
			log.Println(err)
		}
		if i != nil {
			self.i18n[lang] = i
		}
	}
	self.access.Unlock()
	return i
}

func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	i18n := self.GetI18N(r)

	path := r.URL.Path
	_, file := filepath.Split(path)

	isjson := strings.HasSuffix(path, "/json") || strings.HasSuffix(path, "/json/")

	if strings.HasPrefix(path, "/t/") {
		// thread handler
		parts := strings.Split(path[3:], "/")
		hash := parts[0]
		msg, err := self.database.GetMessageIDByHash(hash)
		if err == nil {

			if !self.database.HasArticleLocal(msg.MessageID()) {
				goto notfound
			}

			template.genThread(self.attachments, self.requireCaptcha, msg, self.prefix, self.name, w, self.database, isjson, i18n)
			return
		} else {
			goto notfound
		}
	}
	if strings.Trim(path, "/") == "overboard" {
		// generate ukko aka overboard
		template.genUkko(self.prefix, self.name, w, self.database, isjson, i18n, self.invertPagination)
		return
	}

	if strings.HasPrefix(path, "/b/") {
		// board handler
		parts := strings.Split(path[3:], "/")
		page := 0
		group := parts[0]
		if len(parts) > 1 && parts[1] != "" && parts[1] != "json" {
			var err error
			page, err = strconv.Atoi(parts[1])
			if err != nil {
				goto notfound
			}
		}
		if group == "" {
			goto notfound
		}
		hasgroup := self.database.HasNewsgroup(group)
		if !hasgroup {
			goto notfound
		}

		banned, _ := self.database.NewsgroupBanned(group)
		if banned {
			goto notfound
		}

		pages := self.database.GetGroupPageCount(group)
		if page >= int(pages) {
			goto notfound
		}

		template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination)
		return
	}

	if strings.HasPrefix(path, "/o/") {
		page := 0
		parts := strings.Split(path[3:], "/")
		if parts[0] != "json" && parts[0] != "" {
			var err error
			page, err = strconv.Atoi(parts[0])
			if err != nil {
				goto notfound
			}
		}
		if page == 0 {
			template.genUkko(self.prefix, self.name, w, self.database, isjson, i18n, self.invertPagination)
		} else {
			template.genUkkoPaginated(self.prefix, self.name, w, self.database, page, isjson, i18n, self.invertPagination)
		}
		return
	}

	if len(file) == 0 || file == "index.html" {
		template.genFrontPage(10, self.prefix, self.name, w, ioutil.Discard, self.database, i18n)
		return
	}

	if file == "index.json" {
		// TODO: index.json
		goto notfound
	}
	if strings.HasPrefix(file, "history.html") {
		template.genGraphs(self.prefix, w, self.database, i18n)
		return
	}
	if strings.HasPrefix(file, "boards.html") {
		template.genBoardList(self.prefix, self.name, w, self.database, i18n)
		return
	}

	if strings.HasPrefix(file, "boards.json") {
		b := self.database.GetAllNewsgroups()
		json.NewEncoder(w).Encode(b)
		return
	}

	if strings.HasPrefix(file, "ukko.html") {
		template.genUkko(self.prefix, self.name, w, self.database, false, i18n, self.invertPagination)
		return
	}
	if strings.HasPrefix(file, "ukko.json") {
		template.genUkko(self.prefix, self.name, w, self.database, true, i18n, self.invertPagination)
		return
	}

	if strings.HasPrefix(file, "ukko-") {
		page := getUkkoPage(file)
		template.genUkkoPaginated(self.prefix, self.name, w, self.database, page, isjson, i18n, self.invertPagination)
		return
	}
	if strings.HasPrefix(file, "thread-") {
		hash := getThreadHash(file)
		if len(hash) == 0 {
			goto notfound
		}

		msg, err := self.database.GetMessageIDByHash(hash)
		if err != nil {
			goto notfound
		}

		if !self.database.HasArticleLocal(msg.MessageID()) {
			goto notfound
		}

		template.genThread(self.attachments, self.requireCaptcha, msg, self.prefix, self.name, w, self.database, isjson, i18n)
		return
	}
	if strings.HasPrefix(file, "catalog-") {
		group := getGroupForCatalog(file)
		if len(group) == 0 {
			goto notfound
		}
		hasgroup := self.database.HasNewsgroup(group)
		if !hasgroup {
			goto notfound
		}
		template.genCatalog(self.prefix, self.name, group, w, self.database, i18n)
		return
	} else {
		group, page := getGroupAndPage(file)
		if len(group) == 0 || page < 0 {
			goto notfound
		}
		hasgroup := self.database.HasNewsgroup(group)
		if !hasgroup {
			goto notfound
		}
		pages := self.database.GetGroupPageCount(group)
		if page >= int(pages) {
			goto notfound
		}
		template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination)
		return
	}

notfound:
	template.renderNotFound(w, r, self.prefix, self.name, i18n)
}

func (self *NullCache) DeleteBoardMarkup(group string) {
}

// try to delete root post's page
func (self *NullCache) DeleteThreadMarkup(root_post_id string) {
}

// regen every newsgroup
func (self *NullCache) RegenAll() {
}

func (self *NullCache) RegenFrontPage() {
}

func (self *NullCache) SetRequireCaptcha(required bool) {
	self.handler.requireCaptcha = required
}

// regen every page of the board
func (self *NullCache) RegenerateBoard(group string) {
}

// regenerate pages after a mod event
func (self *NullCache) RegenOnModEvent(newsgroup, msgid, root string, page int) {
}

func (self *NullCache) Start() {
}

func (self *NullCache) Regen(msg ArticleEntry) {
}

func (self *NullCache) GetHandler() CacheHandler {
	return self.handler
}

func (self *NullCache) Close() {
	//nothig to do
}

func NewNullCache(prefix, webroot, name, translations string, attachments bool, db Database, store ArticleStore) CacheInterface {
	cache := new(NullCache)
	cache.handler = &nullHandler{
		prefix:         prefix,
		name:           name,
		attachments:    attachments,
		requireCaptcha: true,
		database:       db,
		i18n:           make(map[string]*I18N),
		translations:   translations,
	}

	return cache
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package spdy

import (
	"fmt"
	"net/http"
	"net/url"

	"k8s.io/apimachinery/pkg/util/httpstream"
	"k8s.io/apimachinery/pkg/util/httpstream/spdy"
	restclient "k8s.io/client-go/rest"
)

// Upgrader validates a response from the server after a SPDY upgrade.
type Upgrader interface {
	// NewConnection validates the response and creates a new Connection.
	NewConnection(resp *http.Response) (httpstream.Connection, error)
}

// RoundTripperFor returns a round tripper and upgrader to use with SPDY.
func RoundTripperFor(config *restclient.Config) (http.RoundTripper, Upgrader, error) {
	tlsConfig, err := restclient.TLSConfigFor(config)
	if err != nil {
		return nil, nil, err
	}
	proxy := http.ProxyFromEnvironment
	if config.Proxy != nil {
		proxy = config.Proxy
	}
	upgradeRoundTripper := spdy.NewRoundTripperWithProxy(tlsConfig, true, false, proxy)
	wrapper, err := restclient.HTTPWrappersForConfig(config, upgradeRoundTripper)
	if err != nil {
		return nil, nil, err
	}
	return wrapper, upgradeRoundTripper, nil
}

// dialer implements the httpstream.Dialer interface.
type dialer struct {
	client   *http.Client
	upgrader Upgrader
	method   string
	url      *url.URL
}

var _ httpstream.Dialer = &dialer{}

// NewDialer will create a dialer that connects to the provided URL and upgrades the connection to SPDY.
func NewDialer(upgrader Upgrader, client *http.Client, method string, url *url.URL) httpstream.Dialer {
	return &dialer{
		client:   client,
		upgrader: upgrader,
		method:   method,
		url:      url,
	}
}

func (d *dialer) Dial(protocols ...string) (httpstream.Connection, string, error) {
	req, err := http.NewRequest(d.method, d.url.String(), nil)
	if err != nil {
		return nil, "", fmt.Errorf("error creating request: %v", err)
	}
	return Negotiate(d.upgrader, d.client, req, protocols...)
}

// Negotiate opens a connection to a remote server and attempts to negotiate
// a SPDY connection. Upon success, it returns the connection and the protocol selected by
// the server. The client transport must use the upgradeRoundTripper - see RoundTripperFor.
func Negotiate(upgrader Upgrader, client *http.Client, req *http.Request, protocols ...string) (httpstream.Connection, string, error) {
	for i := range protocols {
		req.Header.Add(httpstream.HeaderProtocolVersion, protocols[i])
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, "", fmt.Errorf("error sending request: %v", err)
	}
	defer resp.Body.Close()
	conn, err := upgrader.NewConnection(resp)
	if err != nil {
		return nil, "", err
	}
	return conn, resp.Header.Get(httpstream.HeaderProtocolVersion), nil
}
<commit_msg>feat: enable SPDY pings on connections<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package spdy

import (
	"fmt"
	"net/http"
	"net/url"
	"time"

	"k8s.io/apimachinery/pkg/util/httpstream"
	"k8s.io/apimachinery/pkg/util/httpstream/spdy"
	restclient "k8s.io/client-go/rest"
)

// Upgrader validates a response from the server after a SPDY upgrade.
type Upgrader interface {
	// NewConnection validates the response and creates a new Connection.
	NewConnection(resp *http.Response) (httpstream.Connection, error)
}

// RoundTripperFor returns a round tripper and upgrader to use with SPDY.
func RoundTripperFor(config *restclient.Config) (http.RoundTripper, Upgrader, error) {
	tlsConfig, err := restclient.TLSConfigFor(config)
	if err != nil {
		return nil, nil, err
	}
	proxy := http.ProxyFromEnvironment
	if config.Proxy != nil {
		proxy = config.Proxy
	}
	upgradeRoundTripper := spdy.NewRoundTripperWithConfig(spdy.RoundTripperConfig{
		TLS:                      tlsConfig,
		FollowRedirects:          true,
		RequireSameHostRedirects: false,
		Proxier:                  proxy,
		PingPeriod:               time.Second * 5,
	})
	wrapper, err := restclient.HTTPWrappersForConfig(config, upgradeRoundTripper)
	if err != nil {
		return nil, nil, err
	}
	return wrapper, upgradeRoundTripper, nil
}

// dialer implements the httpstream.Dialer interface.
type dialer struct {
	client   *http.Client
	upgrader Upgrader
	method   string
	url      *url.URL
}

var _ httpstream.Dialer = &dialer{}

// NewDialer will create a dialer that connects to the provided URL and upgrades the connection to SPDY.
func NewDialer(upgrader Upgrader, client *http.Client, method string, url *url.URL) httpstream.Dialer {
	return &dialer{
		client:   client,
		upgrader: upgrader,
		method:   method,
		url:      url,
	}
}

func (d *dialer) Dial(protocols ...string) (httpstream.Connection, string, error) {
	req, err := http.NewRequest(d.method, d.url.String(), nil)
	if err != nil {
		return nil, "", fmt.Errorf("error creating request: %v", err)
	}
	return Negotiate(d.upgrader, d.client, req, protocols...)
}

// Negotiate opens a connection to a remote server and attempts to negotiate
// a SPDY connection. Upon success, it returns the connection and the protocol selected by
// the server. The client transport must use the upgradeRoundTripper - see RoundTripperFor.
func Negotiate(upgrader Upgrader, client *http.Client, req *http.Request, protocols ...string) (httpstream.Connection, string, error) {
	for i := range protocols {
		req.Header.Add(httpstream.HeaderProtocolVersion, protocols[i])
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, "", fmt.Errorf("error sending request: %v", err)
	}
	defer resp.Body.Close()
	conn, err := upgrader.NewConnection(resp)
	if err != nil {
		return nil, "", err
	}
	return conn, resp.Header.Get(httpstream.HeaderProtocolVersion), nil
}
<|endoftext|>