<commit_before>// RTLAMR - An rtl-sdr receiver for smart meters operating in the 900MHz ISM band.
// Copyright (C) 2014 Douglas Hall
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"errors"
	"fmt"
	"log"
	"math"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"time"

	"github.com/bemasher/rtlamr/bch"
	"github.com/bemasher/rtlamr/preamble"
	"github.com/bemasher/rtltcp"
)

const (
	CenterFreq = 920299072
	DataRate   = 32768

	PreambleSymbols = 42
	Preamble        = 0x1F2A60
	PreambleBits    = "111110010101001100000"

	PacketSymbols = 192

	GenPoly = 0x16F63

	RestrictLocal = false
	TimeFormat    = "2006-01-02T15:04:05.000"
)

var config Config

type Receiver struct {
	rtltcp.SDR

	pd  preamble.PreambleDetector
	bch bch.BCH
	lut MagLUT
}

func NewReceiver() (rcvr Receiver) {
	// Plan the preamble detector.
	rcvr.pd = preamble.NewPreambleDetector(uint(config.BlockSize<<1), config.SymbolLength, PreambleBits)

	// Create a new BCH for error detection.
	rcvr.bch = bch.NewBCH(GenPoly)
	if !config.Quiet {
		config.Log.Printf("BCH: %+v\n", rcvr.bch)
	}

	rcvr.lut = NewMagLUT()

	// Connect to rtl_tcp server.
	if err := rcvr.Connect(config.ServerAddr); err != nil {
		config.Log.Fatal(err)
	}

	// Tell the user how many gain settings were reported by rtl_tcp.
	if !config.Quiet {
		config.Log.Println("GainCount:", rcvr.SDR.Info.GainCount)
	}

	// Set some parameters for listening.
	rcvr.SetCenterFreq(uint32(config.CenterFreq))
	rcvr.SetSampleRate(uint32(config.SampleRate))
	rcvr.SetGainMode(true)

	return
}

// Clean up rtl_tcp connection and destroy preamble detection plan.
func (rcvr *Receiver) Close() {
	rcvr.SDR.Close()
	rcvr.pd.Close()
}

func (rcvr *Receiver) Run() {
	// Setup signal channel for interruption.
	sigint := make(chan os.Signal, 1)
	signal.Notify(sigint, os.Kill, os.Interrupt)

	// Allocate sample and demodulated signal buffers.
	raw := make([]byte, (config.PacketLength+config.BlockSize)<<1)
	amBuf := make([]float64, config.PacketLength+config.BlockSize)

	// Setup time limit channel
	tLimit := make(<-chan time.Time, 1)
	if config.TimeLimit != 0 {
		tLimit = time.After(config.TimeLimit)
	}

	start := time.Now()
	for {
		// Exit on interrupt or time limit, otherwise receive.
		select {
		case <-sigint:
			return
		case <-tLimit:
			fmt.Println("Time Limit Reached:", time.Since(start))
			return
		default:
			copy(raw, raw[config.BlockSize<<1:])
			copy(amBuf, amBuf[config.BlockSize:])

			// Read new sample block.
			_, err := rcvr.Read(raw[config.PacketLength<<1:])
			if err != nil {
				config.Log.Fatal("Error reading samples: ", err)
			}

			// AM Demodulate
			block := amBuf[config.PacketLength:]
			for idx := range block {
				block[idx] = math.Sqrt(rcvr.lut[raw[(idx)<<1]] + rcvr.lut[raw[((idx)<<1)+1]])
			}

			// Detect preamble in first half of demod buffer.
			rcvr.pd.Execute(amBuf)
			align := rcvr.pd.ArgMax()

			// Bad framing, catch message on next block.
			if uint(align) > config.BlockSize {
				continue
			}

			// Filter signal and bit slice enough data to catch the preamble.
			filtered := MatchedFilter(amBuf[align:], PreambleSymbols>>1)
			bits := BitSlice(filtered)

			// If the preamble matches.
			if bits == PreambleBits {
				// Filter, slice and parse the rest of the buffered samples.
				filtered := MatchedFilter(amBuf[align:], PacketSymbols>>1)
				bits := BitSlice(filtered)

				// If the checksum fails, bail.
				if rcvr.bch.Encode(bits[16:]) != 0 {
					continue
				}

				// Parse SCM
				scm, err := ParseSCM(bits)
				if err != nil {
					config.Log.Fatal("Error parsing SCM: ", err)
				}

				// If filtering by ID and ID doesn't match, bail.
				if config.MeterID != 0 && uint(scm.ID) != config.MeterID {
					continue
				}

				// Get current file offset.
				offset, err := config.SampleFile.Seek(0, os.SEEK_CUR)
				if err != nil {
					config.Log.Fatal("Error getting sample file offset: ", err)
				}

				// Dump message to file.
				_, err = config.SampleFile.Write(raw)
				if err != nil {
					config.Log.Fatal("Error dumping samples: ", err)
				}

				msg := Message{time.Now(), offset, len(raw), scm}

				// Write or encode message to log file.
				if config.Encoder == nil {
					// A nil encoder is just plain-text output.
					fmt.Fprintf(config.LogFile, "%+v", msg)
				} else {
					err = config.Encoder.Encode(msg)
					if err != nil {
						log.Fatal("Error encoding message: ", err)
					}

					// The XML encoder doesn't write new lines after each
					// element, add them.
					if strings.ToLower(config.format) == "xml" {
						fmt.Fprintln(config.LogFile)
					}
				}

				if config.Single {
					return
				}
			}
		}
	}
}

// A lookup table for calculating magnitude of an interleaved unsigned byte
// stream.
type MagLUT []float64

// Shifts sample by 127.4 (most common DC offset value of rtl-sdr dongles) and
// stores square.
func NewMagLUT() (lut MagLUT) {
	lut = make([]float64, 0x100)
	for idx := range lut {
		lut[idx] = 127.4 - float64(idx)
		lut[idx] *= lut[idx]
	}
	return
}

// Matched filter implemented as integrate and dump. Output array is equal to
// the number of manchester coded symbols per packet.
func MatchedFilter(input []float64, bits int) (output []float64) {
	output = make([]float64, bits)

	fIdx := 0
	for idx := 0; fIdx < bits; idx += config.SymbolLength * 2 {
		offset := idx + config.SymbolLength

		for i := 0; i < config.SymbolLength; i++ {
			output[fIdx] += input[idx+i] - input[offset+i]
		}
		fIdx++
	}
	return
}

func BitSlice(input []float64) (output string) {
	for _, v := range input {
		if v > 0.0 {
			output += "1"
		} else {
			output += "0"
		}
	}
	return
}

func ParseUint(raw string) uint64 {
	tmp, _ := strconv.ParseUint(raw, 2, 64)
	return tmp
}

// Message for logging.
type Message struct {
	Time   time.Time
	Offset int64
	Length int
	SCM    SCM
}

func (msg Message) String() string {
	// If we aren't dumping samples, omit offset and packet length.
	if config.sampleFilename == os.DevNull {
		return fmt.Sprintf("{Time:%s SCM:%+v}\n",
			msg.Time.Format(TimeFormat), msg.SCM,
		)
	}

	return fmt.Sprintf("{Time:%s Offset:%d Length:%d SCM:%+v}\n",
		msg.Time.Format(TimeFormat), msg.Offset, msg.Length, msg.SCM,
	)
}

func (msg Message) Record() (record []string) {
	record = append(record, msg.Time.Format(time.RFC3339Nano))
	record = append(record, strconv.FormatInt(int64(msg.Offset), 10))
	record = append(record, strconv.FormatInt(int64(msg.Length), 10))
	record = append(record, msg.SCM.Record()...)
	return record
}

// Standard Consumption Message
type SCM struct {
	ID          uint32
	Type        uint8
	Tamper      Tamper
	Consumption uint32
	Checksum    uint16
}

func (scm SCM) String() string {
	return fmt.Sprintf("{ID:%8d Type:%2d Tamper:%+v Consumption:%8d Checksum:0x%04X}",
		scm.ID, scm.Type, scm.Tamper, scm.Consumption, scm.Checksum,
	)
}

func (scm SCM) Record() (record []string) {
	record = append(record, strconv.FormatInt(int64(scm.ID), 10))
	record = append(record, strconv.FormatInt(int64(scm.Type), 10))
	record = append(record, scm.Tamper.Record()...)
	record = append(record, strconv.FormatInt(int64(scm.Consumption), 10))
	record = append(record, fmt.Sprintf("0x%04X", scm.Checksum))
	return
}

type Tamper struct {
	Phy uint8
	Enc uint8
}

func (t Tamper) String() string {
	return fmt.Sprintf("{Phy:%d Enc:%d}", t.Phy, t.Enc)
}

func (tamper Tamper) Record() (record []string) {
	record = append(record, strconv.FormatInt(int64(tamper.Phy), 10))
	record = append(record, strconv.FormatInt(int64(tamper.Enc), 10))
	return
}

// Given a string of bits, parse the message.
func ParseSCM(data string) (scm SCM, err error) {
	if len(data) != 96 {
		return scm, errors.New("invalid input length")
	}

	scm.ID = uint32(ParseUint(data[21:23] + data[56:80]))
	scm.Type = uint8(ParseUint(data[26:30]))
	scm.Tamper.Phy = uint8(ParseUint(data[24:26]))
	scm.Tamper.Enc = uint8(ParseUint(data[30:32]))
	scm.Consumption = uint32(ParseUint(data[32:56]))
	scm.Checksum = uint16(ParseUint(data[80:96]))

	return scm, nil
}

func init() {
	err := config.Parse()
	if err != nil {
		log.Fatal("Error parsing flags: ", err)
	}
}

func main() {
	if !config.Quiet {
		config.Log.Println("Server:", config.ServerAddr)
		config.Log.Println("BlockSize:", config.BlockSize)
		config.Log.Println("SampleRate:", config.SampleRate)
		config.Log.Println("DataRate:", DataRate)
		config.Log.Println("SymbolLength:", config.SymbolLength)
		config.Log.Println("PreambleSymbols:", PreambleSymbols)
		config.Log.Println("PreambleLength:", config.PreambleLength)
		config.Log.Println("PacketSymbols:", PacketSymbols)
		config.Log.Println("PacketLength:", config.PacketLength)
		config.Log.Println("CenterFreq:", config.CenterFreq)
		config.Log.Println("TimeLimit:", config.TimeLimit)

		config.Log.Println("Format:", config.format)
		config.Log.Println("LogFile:", config.logFilename)
		config.Log.Println("SampleFile:", config.sampleFilename)

		if config.MeterID != 0 {
			config.Log.Println("FilterID:", config.MeterID)
		}
	}

	rcvr := NewReceiver()
	defer rcvr.Close()
	defer config.Close()

	if !config.Quiet {
		config.Log.Println("Running...")
	}

	rcvr.Run()
}
<commit_msg>Fix demodulation indexing. Should fix samplefile behavior.<commit_after>// RTLAMR - An rtl-sdr receiver for smart meters operating in the 900MHz ISM band.
// Copyright (C) 2014 Douglas Hall
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"errors"
	"fmt"
	"log"
	"math"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"time"

	"github.com/bemasher/rtlamr/bch"
	"github.com/bemasher/rtlamr/preamble"
	"github.com/bemasher/rtltcp"
)

const (
	CenterFreq = 920299072
	DataRate   = 32768

	PreambleSymbols = 42
	Preamble        = 0x1F2A60
	PreambleBits    = "111110010101001100000"

	PacketSymbols = 192

	GenPoly = 0x16F63

	RestrictLocal = false
	TimeFormat    = "2006-01-02T15:04:05.000"
)

var config Config

type Receiver struct {
	rtltcp.SDR

	pd  preamble.PreambleDetector
	bch bch.BCH
	lut MagLUT
}

func NewReceiver() (rcvr Receiver) {
	// Plan the preamble detector.
	rcvr.pd = preamble.NewPreambleDetector(uint(config.BlockSize<<1), config.SymbolLength, PreambleBits)

	// Create a new BCH for error detection.
	rcvr.bch = bch.NewBCH(GenPoly)
	if !config.Quiet {
		config.Log.Printf("BCH: %+v\n", rcvr.bch)
	}

	rcvr.lut = NewMagLUT()

	// Connect to rtl_tcp server.
	if err := rcvr.Connect(config.ServerAddr); err != nil {
		config.Log.Fatal(err)
	}

	// Tell the user how many gain settings were reported by rtl_tcp.
	if !config.Quiet {
		config.Log.Println("GainCount:", rcvr.SDR.Info.GainCount)
	}

	// Set some parameters for listening.
	rcvr.SetCenterFreq(uint32(config.CenterFreq))
	rcvr.SetSampleRate(uint32(config.SampleRate))
	rcvr.SetGainMode(true)

	return
}

// Clean up rtl_tcp connection and destroy preamble detection plan.
func (rcvr *Receiver) Close() {
	rcvr.SDR.Close()
	rcvr.pd.Close()
}

func (rcvr *Receiver) Run() {
	// Setup signal channel for interruption.
	sigint := make(chan os.Signal, 1)
	signal.Notify(sigint, os.Kill, os.Interrupt)

	// Allocate sample and demodulated signal buffers.
	raw := make([]byte, (config.PacketLength+config.BlockSize)<<1)
	amBuf := make([]float64, config.PacketLength+config.BlockSize)

	// Setup time limit channel
	tLimit := make(<-chan time.Time, 1)
	if config.TimeLimit != 0 {
		tLimit = time.After(config.TimeLimit)
	}

	start := time.Now()
	for {
		// Exit on interrupt or time limit, otherwise receive.
		select {
		case <-sigint:
			return
		case <-tLimit:
			fmt.Println("Time Limit Reached:", time.Since(start))
			return
		default:
			copy(raw, raw[config.BlockSize<<1:])
			copy(amBuf, amBuf[config.BlockSize:])

			// Read new sample block.
			_, err := rcvr.Read(raw[config.PacketLength<<1:])
			if err != nil {
				config.Log.Fatal("Error reading samples: ", err)
			}

			// AM Demodulate
			block := amBuf[config.PacketLength:]
			rawBlock := raw[config.PacketLength<<1:]
			for idx := range block {
				block[idx] = math.Sqrt(rcvr.lut[rawBlock[idx<<1]] + rcvr.lut[rawBlock[(idx<<1)+1]])
			}

			// Detect preamble in first half of demod buffer.
			rcvr.pd.Execute(amBuf)
			align := rcvr.pd.ArgMax()

			// Bad framing, catch message on next block.
			if uint(align) > config.BlockSize {
				continue
			}

			// Filter signal and bit slice enough data to catch the preamble.
			filtered := MatchedFilter(amBuf[align:], PreambleSymbols>>1)
			bits := BitSlice(filtered)

			// If the preamble matches.
			if bits == PreambleBits {
				// Filter, slice and parse the rest of the buffered samples.
				filtered := MatchedFilter(amBuf[align:], PacketSymbols>>1)
				bits := BitSlice(filtered)

				// If the checksum fails, bail.
				if rcvr.bch.Encode(bits[16:]) != 0 {
					continue
				}

				// Parse SCM
				scm, err := ParseSCM(bits)
				if err != nil {
					config.Log.Fatal("Error parsing SCM: ", err)
				}

				// If filtering by ID and ID doesn't match, bail.
				if config.MeterID != 0 && uint(scm.ID) != config.MeterID {
					continue
				}

				// Get current file offset.
				offset, err := config.SampleFile.Seek(0, os.SEEK_CUR)
				if err != nil {
					config.Log.Fatal("Error getting sample file offset: ", err)
				}

				// Dump message to file.
				_, err = config.SampleFile.Write(raw)
				if err != nil {
					config.Log.Fatal("Error dumping samples: ", err)
				}

				msg := Message{time.Now(), offset, len(raw), scm}

				// Write or encode message to log file.
				if config.Encoder == nil {
					// A nil encoder is just plain-text output.
					fmt.Fprintf(config.LogFile, "%+v", msg)
				} else {
					err = config.Encoder.Encode(msg)
					if err != nil {
						log.Fatal("Error encoding message: ", err)
					}

					// The XML encoder doesn't write new lines after each
					// element, add them.
					if strings.ToLower(config.format) == "xml" {
						fmt.Fprintln(config.LogFile)
					}
				}

				if config.Single {
					return
				}
			}
		}
	}
}

// A lookup table for calculating magnitude of an interleaved unsigned byte
// stream.
type MagLUT []float64

// Shifts sample by 127.4 (most common DC offset value of rtl-sdr dongles) and
// stores square.
func NewMagLUT() (lut MagLUT) {
	lut = make([]float64, 0x100)
	for idx := range lut {
		lut[idx] = 127.4 - float64(idx)
		lut[idx] *= lut[idx]
	}
	return
}

// Matched filter implemented as integrate and dump. Output array is equal to
// the number of manchester coded symbols per packet.
func MatchedFilter(input []float64, bits int) (output []float64) {
	output = make([]float64, bits)

	fIdx := 0
	for idx := 0; fIdx < bits; idx += config.SymbolLength * 2 {
		offset := idx + config.SymbolLength

		for i := 0; i < config.SymbolLength; i++ {
			output[fIdx] += input[idx+i] - input[offset+i]
		}
		fIdx++
	}
	return
}

func BitSlice(input []float64) (output string) {
	for _, v := range input {
		if v > 0.0 {
			output += "1"
		} else {
			output += "0"
		}
	}
	return
}

func ParseUint(raw string) uint64 {
	tmp, _ := strconv.ParseUint(raw, 2, 64)
	return tmp
}

// Message for logging.
type Message struct {
	Time   time.Time
	Offset int64
	Length int
	SCM    SCM
}

func (msg Message) String() string {
	// If we aren't dumping samples, omit offset and packet length.
	if config.sampleFilename == os.DevNull {
		return fmt.Sprintf("{Time:%s SCM:%+v}\n",
			msg.Time.Format(TimeFormat), msg.SCM,
		)
	}

	return fmt.Sprintf("{Time:%s Offset:%d Length:%d SCM:%+v}\n",
		msg.Time.Format(TimeFormat), msg.Offset, msg.Length, msg.SCM,
	)
}

func (msg Message) Record() (record []string) {
	record = append(record, msg.Time.Format(time.RFC3339Nano))
	record = append(record, strconv.FormatInt(int64(msg.Offset), 10))
	record = append(record, strconv.FormatInt(int64(msg.Length), 10))
	record = append(record, msg.SCM.Record()...)
	return record
}

// Standard Consumption Message
type SCM struct {
	ID          uint32
	Type        uint8
	Tamper      Tamper
	Consumption uint32
	Checksum    uint16
}

func (scm SCM) String() string {
	return fmt.Sprintf("{ID:%8d Type:%2d Tamper:%+v Consumption:%8d Checksum:0x%04X}",
		scm.ID, scm.Type, scm.Tamper, scm.Consumption, scm.Checksum,
	)
}

func (scm SCM) Record() (record []string) {
	record = append(record, strconv.FormatInt(int64(scm.ID), 10))
	record = append(record, strconv.FormatInt(int64(scm.Type), 10))
	record = append(record, scm.Tamper.Record()...)
	record = append(record, strconv.FormatInt(int64(scm.Consumption), 10))
	record = append(record, fmt.Sprintf("0x%04X", scm.Checksum))
	return
}

type Tamper struct {
	Phy uint8
	Enc uint8
}

func (t Tamper) String() string {
	return fmt.Sprintf("{Phy:%d Enc:%d}", t.Phy, t.Enc)
}

func (tamper Tamper) Record() (record []string) {
	record = append(record, strconv.FormatInt(int64(tamper.Phy), 10))
	record = append(record, strconv.FormatInt(int64(tamper.Enc), 10))
	return
}

// Given a string of bits, parse the message.
func ParseSCM(data string) (scm SCM, err error) {
	if len(data) != 96 {
		return scm, errors.New("invalid input length")
	}

	scm.ID = uint32(ParseUint(data[21:23] + data[56:80]))
	scm.Type = uint8(ParseUint(data[26:30]))
	scm.Tamper.Phy = uint8(ParseUint(data[24:26]))
	scm.Tamper.Enc = uint8(ParseUint(data[30:32]))
	scm.Consumption = uint32(ParseUint(data[32:56]))
	scm.Checksum = uint16(ParseUint(data[80:96]))

	return scm, nil
}

func init() {
	err := config.Parse()
	if err != nil {
		log.Fatal("Error parsing flags: ", err)
	}
}

func main() {
	if !config.Quiet {
		config.Log.Println("Server:", config.ServerAddr)
		config.Log.Println("BlockSize:", config.BlockSize)
		config.Log.Println("SampleRate:", config.SampleRate)
		config.Log.Println("DataRate:", DataRate)
		config.Log.Println("SymbolLength:", config.SymbolLength)
		config.Log.Println("PreambleSymbols:", PreambleSymbols)
		config.Log.Println("PreambleLength:", config.PreambleLength)
		config.Log.Println("PacketSymbols:", PacketSymbols)
		config.Log.Println("PacketLength:", config.PacketLength)
		config.Log.Println("CenterFreq:", config.CenterFreq)
		config.Log.Println("TimeLimit:", config.TimeLimit)

		config.Log.Println("Format:", config.format)
		config.Log.Println("LogFile:", config.logFilename)
		config.Log.Println("SampleFile:", config.sampleFilename)

		if config.MeterID != 0 {
			config.Log.Println("FilterID:", config.MeterID)
		}
	}

	rcvr := NewReceiver()
	defer rcvr.Close()
	defer config.Close()

	if !config.Quiet {
		config.Log.Println("Running...")
	}

	rcvr.Run()
}
<|endoftext|>


<commit_before>// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package weakhash

import (
	"bytes"
	"context"
	"fmt"
	"hash"
	vadler32 "hash/adler32"
	"io"
	"math/rand"
	"os"
	"testing"

	"github.com/chmduquesne/rollinghash/adler32"
	"github.com/chmduquesne/rollinghash/bozo32"
	"github.com/chmduquesne/rollinghash/buzhash32"
	"github.com/chmduquesne/rollinghash/buzhash64"
)

const testFile = "../model/testdata/tmpfile"
const size = 128 << 10

func BenchmarkFind1MFile(b *testing.B) {
	b.ReportAllocs()
	b.SetBytes(1 << 20)
	for i := 0; i < b.N; i++ {
		fd, err := os.Open(testFile)
		if err != nil {
			b.Fatal(err)
		}
		_, err = Find(context.Background(), fd, []uint32{0, 1, 2}, size)
		if err != nil {
			b.Fatal(err)
		}
		fd.Close()
	}
}

type RollingHash interface {
	hash.Hash
	Roll(byte)
}

func BenchmarkBlock(b *testing.B) {
	tests := []struct {
		name string
		hash hash.Hash
	}{
		{
			"adler32", adler32.New(),
		},
		{
			"bozo32", bozo32.New(),
		},
		{
			"buzhash32", buzhash32.New(),
		},
		{
			"buzhash64", buzhash64.New(),
		},
		{
			"vanilla-adler32", vadler32.New(),
		},
	}

	sizes := []int64{128 << 10, 16 << 20}

	buf := make([]byte, 16<<20)
	rand.Read(buf)

	for _, testSize := range sizes {
		for _, test := range tests {
			b.Run(test.name+"-"+fmt.Sprint(testSize), func(bb *testing.B) {
				bb.Run("", func(bbb *testing.B) {
					bbb.ResetTimer()
					for i := 0; i < bbb.N; i++ {
						lr := io.LimitReader(bytes.NewReader(buf), testSize)
						n, err := io.Copy(test.hash, lr)
						if err != nil {
							bbb.Error(err)
						}
						if n != testSize {
							bbb.Errorf("%d != %d", n, testSize)
						}

						test.hash.Sum(nil)
						test.hash.Reset()
					}

					bbb.SetBytes(int64(len(buf)))
					bbb.ReportAllocs()
				})

			})
		}
	}
}

func BenchmarkRoll(b *testing.B) {
	tests := []struct {
		name string
		hash RollingHash
	}{
		{
			"adler32", adler32.New(),
		},
		{
			"bozo32", bozo32.New(),
		},
		{
			"buzhash32", buzhash32.New(),
		},
		{
			"buzhash64", buzhash64.New(),
		},
	}

	sizes := []int64{128 << 10, 16 << 20}

	for _, testSize := range sizes {
		for _, test := range tests {
			b.Run(test.name+"-"+fmt.Sprint(testSize), func(bb *testing.B) {
				bb.Run("", func(bbb *testing.B) {
					data := make([]byte, testSize)

					if _, err := test.hash.Write(data); err != nil {
						bbb.Error(err)
					}

					bbb.ResetTimer()

					for i := 0; i < bbb.N; i++ {
						for j := int64(0); j <= testSize; j++ {
							test.hash.Roll('a')
						}
					}

					bbb.SetBytes(testSize)
					bbb.ReportAllocs()
				})

			})
		}
	}
}
<commit_msg>lib/weakhash: Fix speed reporting in benchmark (#6470)<commit_after>// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package weakhash

import (
	"bytes"
	"context"
	"fmt"
	"hash"
	vadler32 "hash/adler32"
	"io"
	"math/rand"
	"os"
	"testing"

	"github.com/chmduquesne/rollinghash/adler32"
	"github.com/chmduquesne/rollinghash/bozo32"
	"github.com/chmduquesne/rollinghash/buzhash32"
	"github.com/chmduquesne/rollinghash/buzhash64"
)

const testFile = "../model/testdata/tmpfile"
const size = 128 << 10

func BenchmarkFind1MFile(b *testing.B) {
	b.ReportAllocs()
	b.SetBytes(1 << 20)
	for i := 0; i < b.N; i++ {
		fd, err := os.Open(testFile)
		if err != nil {
			b.Fatal(err)
		}
		_, err = Find(context.Background(), fd, []uint32{0, 1, 2}, size)
		if err != nil {
			b.Fatal(err)
		}
		fd.Close()
	}
}

type RollingHash interface {
	hash.Hash
	Roll(byte)
}

func BenchmarkBlock(b *testing.B) {
	tests := []struct {
		name string
		hash hash.Hash
	}{
		{
			"adler32", adler32.New(),
		},
		{
			"bozo32", bozo32.New(),
		},
		{
			"buzhash32", buzhash32.New(),
		},
		{
			"buzhash64", buzhash64.New(),
		},
		{
			"vanilla-adler32", vadler32.New(),
		},
	}

	sizes := []int64{128 << 10, 16 << 20}

	buf := make([]byte, 16<<20)
	rand.Read(buf)

	for _, testSize := range sizes {
		for _, test := range tests {
			b.Run(test.name+"-"+fmt.Sprint(testSize), func(bb *testing.B) {
				bb.Run("", func(bbb *testing.B) {
					bbb.ResetTimer()
					for i := 0; i < bbb.N; i++ {
						lr := io.LimitReader(bytes.NewReader(buf), testSize)
						n, err := io.Copy(test.hash, lr)
						if err != nil {
							bbb.Error(err)
						}
						if n != testSize {
							bbb.Errorf("%d != %d", n, testSize)
						}

						test.hash.Sum(nil)
						test.hash.Reset()
					}

					bbb.SetBytes(testSize)
					bbb.ReportAllocs()
				})

			})
		}
	}
}

func BenchmarkRoll(b *testing.B) {
	tests := []struct {
		name string
		hash RollingHash
	}{
		{
			"adler32", adler32.New(),
		},
		{
			"bozo32", bozo32.New(),
		},
		{
			"buzhash32", buzhash32.New(),
		},
		{
			"buzhash64", buzhash64.New(),
		},
	}

	sizes := []int64{128 << 10, 16 << 20}

	for _, testSize := range sizes {
		for _, test := range tests {
			b.Run(test.name+"-"+fmt.Sprint(testSize), func(bb *testing.B) {
				bb.Run("", func(bbb *testing.B) {
					data := make([]byte, testSize)

					if _, err := test.hash.Write(data); err != nil {
						bbb.Error(err)
					}

					bbb.ResetTimer()

					for i := 0; i < bbb.N; i++ {
						for j := int64(0); j <= testSize; j++ {
							test.hash.Roll('a')
						}
					}

					bbb.SetBytes(testSize)
					bbb.ReportAllocs()
				})

			})
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 Kevin Gillette. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package slug transforms strings into a normalized
// form well suited for use in URLs.
package slug

import (
	"code.google.com/p/go.text/unicode/norm"
	"unicode"
)

var nop = []*unicode.RangeTable{unicode.Mark, unicode.Sk, unicode.Lm}

// Slug replaces each run of characters which are not unicode letters or
// numbers with a single hyphen, except for leading or trailing runs. Letters
// will be stripped of diacritical marks and lowercased.
func Slug(s string) string {
	buf := make([]rune, 0, len(s))
	dash := false
	for _, r := range norm.NFKD.String(s) {
		switch {
		// unicode 'letters' like mandarin characters pass through
		case unicode.IsLetter(r):
			r = unicode.ToLower(r)
			fallthrough
		case unicode.IsNumber(r):
			buf = append(buf, unicode.ToLower(r))
			dash = true
		case unicode.IsOneOf(nop, r):
			// skip
		case dash:
			buf = append(buf, '-')
			dash = false
		}
	}
	if i := len(buf) - 1; i >= 0 && buf[i] == '-' {
		buf = buf[:i]
	}
	return string(buf)
}

func IsSlugAscii(s string) bool {
	dash := true
	for _, r := range s {
		switch {
		case r == '-':
			if dash {
				return false
			}
			dash = true
		case 'a' <= r && r <= 'z', '0' <= r && r <= '9':
			dash = false
		}
	}
	return !dash
}
<commit_msg>Removed redundant code.<commit_after>// Copyright 2012 Kevin Gillette. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package slug transforms strings into a normalized
// form well suited for use in URLs.
package slug

import (
	"code.google.com/p/go.text/unicode/norm"
	"unicode"
)

var lat = []*unicode.RangeTable{unicode.Letter, unicode.Number}
var nop = []*unicode.RangeTable{unicode.Mark, unicode.Sk, unicode.Lm}

// Slug replaces each run of characters which are not unicode letters or
// numbers with a single hyphen, except for leading or trailing runs. Letters
// will be stripped of diacritical marks and lowercased.
func Slug(s string) string {
	buf := make([]rune, 0, len(s))
	dash := false
	for _, r := range norm.NFKD.String(s) {
		switch {
		// unicode 'letters' like mandarin characters pass through
		case unicode.IsOneOf(lat, r):
			buf = append(buf, unicode.ToLower(r))
			dash = true
		case unicode.IsOneOf(nop, r):
			// skip
		case dash:
			buf = append(buf, '-')
			dash = false
		}
	}
	if i := len(buf) - 1; i >= 0 && buf[i] == '-' {
		buf = buf[:i]
	}
	return string(buf)
}

func IsSlugAscii(s string) bool {
	dash := true
	for _, r := range s {
		switch {
		case r == '-':
			if dash {
				return false
			}
			dash = true
		case 'a' <= r && r <= 'z', '0' <= r && r <= '9':
			dash = false
		}
	}
	return !dash
}
<|endoftext|>


<commit_before>package main

import (
	"collector"
	"encoding/json"
	"fmt"
	"helper"
	"log"
	"os"
	"sse"
	"time"
)

var (
	URL = "http://mothership.serverstatusmonitoring.com"
	URIRegister = "/register"
	URICollector = "/collector"
	URIStatus = "/status"

	Hostname = ""
	IPAddress = ""

	LogFile = "/var/log/sphire-sse.log"
	ConfigurationFile = "/etc/sse/sse.conf"
	Configuration = new(Configuration)

	CollectFrequencySeconds = 1 // Collect a snapshot and store in cache every X seconds
	ReportFrequencySeconds = 2 // Report all snapshots in cache every Y seconds

	CPU     collector.CPU     = collector.CPU{}
	Disks   collector.Disks   = collector.Disks{}
	Memory  collector.Memory  = collector.Memory{}
	Network collector.Network = collector.Network{}
	System  collector.System  = collector.System{}

	Version = "1.0.0"
)

/*
 Configuration struct is a direct map to the configuration located in the configuration JSON file.
*/
type Configuration struct {
	Identification struct {
		AccountID        string `json:"account_id"`
		OrganizationID   string `json:"organization_id"`
		OrganizationName string `json:"organization_name"`
		MachineNickname  string `json:"machine_nickname"`
	} `json:"identification"`
}

func main(){
	// Define the global logger
	logger, err := os.OpenFile(LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	HandleError(err)
	defer logger.Close()
	log.SetOutput(logger)

	// Load and parse configuration file
	file, err := os.Open(ConfigurationFile)
	HandleError(err)
	err := json.NewDecoder(file).Decode(Configuration)
	HandleError(err)

	var status helper.Status = helper.Status{}
	var status_result bool = status.CheckStatus(URL + URIStatus)
	if status_result == false {
		HandleError(error("Mothership unreachable. Check your internet connection."))
	}

	// Perform system initialization
	var server_obj sse.Server = sse.Server{}
	var server *sse.Server = &sse.Server{}
	server, ipAddress, hostname, version, err = server_obj.Initialize()
	HandleError(err)

	// Perform registration
	body, err := sse.Register(map[string]interface{}{
		"configuration":     Configuration,
		"mothership_url":    URL,
		"register_uri":      URIRegister,
		"version":           Version,
		"collect_frequency": CollectFrequencySeconds,
		"report_frequency":  ReportFrequencySeconds,
		"hostname":          Hostname,
		"ip_address":        IPAddress,
		"log_file":          LogFile,
		"config_file":       ConfigurationFile,
	}, URL + URIRegister + "/" + Version)
	if err != nil {
		HandleError(error("Unable to register this machine"+string(body)))
	}

	// Set up our collector
	var counter int = 0
	var snapshot sse.Snapshot = sse.Snapshot{}
	var cache sse.Cache = sse.Cache{
		AccountId:        Configuration.Identification.AccountID,
		OrganizationID:   Configuration.Identification.OrganizationID,
		OrganizationName: Configuration.Identification.OrganizationName,
		MachineNickname:  Configuration.Identification.MachineNickname,
		Version:          Version,
		Server:           server}

	ticker := time.NewTicker(time.Duration(CollectFrequencySeconds) * time.Second)

	for {
		<-ticker.C // send the updated time back via the channel

		// reset the snapshot to an empty struct
		snapshot = sse.Snapshot{}

		// fill in the Snapshot struct and add to the cache
		cache.Node = append(cache.Node, snapshot.Collector())
		counter++

		if counter > 0 && counter%ReportFrequencySeconds == 0 {
			cache.Sender(URL + URICollector)
			cache.Node = nil // Clear the Node Cache
			counter = 0
		}
	}

	return

}

func HandleError(err error){
	if err != nil {
		log.Println(helper.Trace(err, "ERROR"))
		fmt.Println(err, "ERROR")
		os.Exit(1)
	}
}
<commit_msg>fmt lean.go<commit_after>package main

import (
	"collector"
	"encoding/json"
	"fmt"
	"helper"
	"log"
	"os"
	"sse"
	"time"
)

var (
	URL          = "http://mothership.serverstatusmonitoring.com"
	URIRegister  = "/register"
	URICollector = "/collector"
	URIStatus    = "/status"

	Hostname  = ""
	IPAddress = ""

	LogFile           = "/var/log/sphire-sse.log"
	ConfigurationFile = "/etc/sse/sse.conf"
	Configuration     = new(Configuration)

	CollectFrequencySeconds = 1 // Collect a snapshot and store in cache every X seconds
	ReportFrequencySeconds  = 2 // Report all snapshots in cache every Y seconds

	CPU     collector.CPU     = collector.CPU{}
	Disks   collector.Disks   = collector.Disks{}
	Memory  collector.Memory  = collector.Memory{}
	Network collector.Network = collector.Network{}
	System  collector.System  = collector.System{}

	Version = "1.0.0"
)

/*
 Configuration struct is a direct map to the configuration located in the configuration JSON file.
*/
type Configuration struct {
	Identification struct {
		AccountID        string `json:"account_id"`
		OrganizationID   string `json:"organization_id"`
		OrganizationName string `json:"organization_name"`
		MachineNickname  string `json:"machine_nickname"`
	} `json:"identification"`
}

func main() {
	// Define the global logger
	logger, err := os.OpenFile(LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	HandleError(err)
	defer logger.Close()
	log.SetOutput(logger)

	// Load and parse configuration file
	file, err := os.Open(ConfigurationFile)
	HandleError(err)
	err := json.NewDecoder(file).Decode(Configuration)
	HandleError(err)

	var status helper.Status = helper.Status{}
	var status_result bool = status.CheckStatus(URL + URIStatus)
	if status_result == false {
		HandleError(error("Mothership unreachable. Check your internet connection."))
	}

	// Perform system initialization
	var server_obj sse.Server = sse.Server{}
	var server *sse.Server = &sse.Server{}
	server, ipAddress, hostname, version, err = server_obj.Initialize()
	HandleError(err)

	// Perform registration
	body, err := sse.Register(map[string]interface{}{
		"configuration":     Configuration,
		"mothership_url":    URL,
		"register_uri":      URIRegister,
		"version":           Version,
		"collect_frequency": CollectFrequencySeconds,
		"report_frequency":  ReportFrequencySeconds,
		"hostname":          Hostname,
		"ip_address":        IPAddress,
		"log_file":          LogFile,
		"config_file":       ConfigurationFile,
	}, URL+URIRegister+"/"+Version)
	if err != nil {
		HandleError(error("Unable to register this machine" + string(body)))
	}

	// Set up our collector
	var counter int = 0
	var snapshot sse.Snapshot = sse.Snapshot{}
	var cache sse.Cache = sse.Cache{
		AccountId:        Configuration.Identification.AccountID,
		OrganizationID:   Configuration.Identification.OrganizationID,
		OrganizationName: Configuration.Identification.OrganizationName,
		MachineNickname:  Configuration.Identification.MachineNickname,
		Version:          Version,
		Server:           server}

	ticker := time.NewTicker(time.Duration(CollectFrequencySeconds) * time.Second)

	for {
		<-ticker.C // send the updated time back via the channel

		// reset the snapshot to an empty struct
		snapshot = sse.Snapshot{}

		// fill in the Snapshot struct and add to the cache
		cache.Node = append(cache.Node, snapshot.Collector())
		counter++

		if counter > 0 && counter%ReportFrequencySeconds == 0 {
			cache.Sender(URL + URICollector)
			cache.Node = nil // Clear the Node Cache
			counter = 0
		}
	}

	return

}

func HandleError(err error) {
	if err != nil {
		log.Println(helper.Trace(err, "ERROR"))
		fmt.Println(err, "ERROR")
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package aero

import (
	"fmt"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"time"

	"crypto/sha256"

	"encoding/base64"
	"encoding/json"

	"io/ioutil"

	"github.com/aerogo/session"
	memstore "github.com/aerogo/session-store-memory"
	"github.com/fatih/color"
	"github.com/julienschmidt/httprouter"
	cache "github.com/patrickmn/go-cache"
)

const (
	gzipCacheDuration = 5 * time.Minute
	gzipCacheCleanup  = 1 * time.Minute
)

// Middleware ...
type Middleware func(*Context, func())

// Application represents a single web service.
type Application struct {
	root string

	Config   *Configuration
	Layout   func(*Context, string) string
	Sessions session.Manager
	Security ApplicationSecurity

	router *httprouter.Router
	routes struct {
		GET  []string
		POST []string
	}
	routeTests map[string][]string
	gzipCache  *cache.Cache
	start      time.Time
	rewrite    func(*RewriteContext)

	middleware []Middleware

	css            string
	cssHash        string
	cssReplacement string

	contentSecurityPolicy string
}

// New creates a new application.
func New() *Application {
	app := new(Application)
	app.start = time.Now()
	app.router = httprouter.New()
	app.routeTests = make(map[string][]string)
	app.gzipCache = cache.New(gzipCacheDuration, gzipCacheCleanup)
	app.Layout = func(ctx *Context, content string) string {
		return content
	}
	app.Config = new(Configuration)
	app.Config.Reset()
	app.Load()

	app.Sessions.Store = memstore.New()

	return app
}

// Get registers your function to be called when a certain GET path has been requested.
func (app *Application) Get(path string, handle Handle) {
	app.routes.GET = append(app.routes.GET, path)
	app.router.GET(path, app.createRouteHandler(path, handle))
}

// Post registers your function to be called when a certain POST path has been requested.
func (app *Application) Post(path string, handle Handle) {
	app.routes.POST = append(app.routes.POST, path)
	app.router.POST(path, app.createRouteHandler(path, handle))
}

// createRouteHandler creates a handler function for httprouter.
func (app *Application) createRouteHandler(path string, handle Handle) httprouter.Handle {
	return func(response http.ResponseWriter, request *http.Request, params httprouter.Params) {
		// Create context.
		ctx := Context{
			App:        app,
			StatusCode: http.StatusOK,
			request:    request,
			response:   response,
			params:     params,
			start:      time.Now(),
		}

		// The last part of the call chain will send the actual response.
		lastPartOfCallChain := func() {
			data := handle(&ctx)
			ctx.Respond(data)
		}

		// Declare the type of generateNext so that we can define it recursively in the next part.
		var generateNext func(index int) func()

		// Create a function that returns a bound function next()
		// which can be used as the 2nd parameter in the call chain.
		generateNext = func(index int) func() {
			if index == len(app.middleware) {
				return lastPartOfCallChain
			}

			return func() {
				app.middleware[index](&ctx, generateNext(index+1))
			}
		}

		generateNext(0)()
	}
}

// Ajax calls app.Get for both /route and /_/route
func (app *Application) Ajax(path string, handle Handle) {
	app.Get("/_"+path, handle)
	app.Get(path, func(ctx *Context) string {
		page := handle(ctx)
		html := app.Layout(ctx, page)
		html = strings.Replace(html, "<body", app.cssReplacement, 1)
		html = strings.Replace(html, "<html", "<!DOCTYPE html><html", 1)
		return html
	})
}

// Run starts your application.
func (app *Application) Run() {
	app.TestManifest()
	app.TestRoutes()
	app.Listen()
}

// Use adds middleware to your middleware chain.
func (app *Application) Use(middlewares ...Middleware) {
	app.middleware = append(app.middleware, middlewares...)
}

// Load loads the application data from the file system.
func (app *Application) Load() {
	config, readError := ioutil.ReadFile("config.json")

	if readError == nil {
		jsonDecodeError := json.Unmarshal(config, app.Config)

		if jsonDecodeError != nil {
			color.Red(jsonDecodeError.Error())
		}
	}

	if app.Config.Manifest.Name == "" {
		app.Config.Manifest.Name = app.Config.Title
	}

	if app.Config.Manifest.ShortName == "" {
		app.Config.Manifest.ShortName = app.Config.Title
	}

	if app.Config.Manifest.Lang == "" {
		app.Config.Manifest.Lang = "en"
	}

	if app.Config.Manifest.Display == "" {
		app.Config.Manifest.Display = "fullscreen"
	}

	if app.Config.Manifest.StartURL == "" {
		app.Config.Manifest.StartURL = "/"
	}
}

// Listen starts the server.
func (app *Application) Listen() {
	if app.Security.Key != "" && app.Security.Certificate != "" {
		go func() {
			app.serveHTTPS(":" + strconv.Itoa(app.Config.Ports.HTTPS))
		}()

		fmt.Println("Server running on:", color.GreenString("https://localhost:"+strconv.Itoa(app.Config.Ports.HTTPS)))
	} else {
		fmt.Println("Server running on:", color.GreenString("http://localhost:"+strconv.Itoa(app.Config.Ports.HTTP)))
	}

	app.serveHTTP(":" + strconv.Itoa(app.Config.Ports.HTTP))
}

// // listen listens on the specified host and port.
// func (app *Application) listen(port int) net.Listener {
// 	address := ":" + strconv.Itoa(port)

// 	listener, bindError := net.Listen("tcp", address)

// 	if bindError != nil {
// 		panic(bindError)
// 	}

// 	return listener
// }

// Rewrite sets the URL rewrite function.
func (app *Application) Rewrite(rewrite func(*RewriteContext)) {
	app.rewrite = rewrite
}

// SetStyle ...
func (app *Application) SetStyle(css string) {
	app.css = css

	hash := sha256.Sum256([]byte(css))
	app.cssHash = base64.StdEncoding.EncodeToString(hash[:])
	app.cssReplacement = "<body"
	app.contentSecurityPolicy = "default-src 'none'; img-src https:; script-src 'self'; style-src 'sha256-" + app.cssHash + "'; font-src https:; manifest-src 'self'; child-src https:; connect-src https: wss:"
}

// StartTime ...
func (app *Application) StartTime() time.Time {
	return app.start
}

// Handler returns the request handler.
func (app *Application) Handler() http.Handler {
	router := app.router
	rewrite := app.rewrite

	if rewrite != nil {
		return &rewriteHandler{
			rewrite: rewrite,
			router:  router,
		}
	}

	return router
}

// serveHTTP serves requests from the given listener.
func (app *Application) serveHTTP(address string) {
	serveError := http.ListenAndServe(address, app.Handler())

	if serveError != nil {
		panic(serveError)
	}
}

// serveHTTPS serves requests from the given listener.
func (app *Application) serveHTTPS(address string) {
	serveError := http.ListenAndServeTLS(address, app.Security.Certificate, app.Security.Key, app.Handler())

	if serveError != nil {
		panic(serveError)
	}
}

// Test tests the given URI paths when the application starts.
func (app *Application) Test(route string, paths []string) {
	app.routeTests[route] = paths
}

// TestManifest tests your application's manifest.
func (app *Application) TestManifest() {
	manifest := app.Config.Manifest

	if len(manifest.ShortName) >= 12 {
		color.Yellow("The short name of your application should have less than 12 characters")
	}
}

// TestRoutes tests your application's routes.
func (app *Application) TestRoutes() {
	fmt.Println(strings.Repeat("-", 80))

	go func() {
		sort.Strings(app.routes.GET)

		for _, route := range app.routes.GET {
			// Ajax routes
			if strings.HasPrefix(route, "/_") {
				continue
			}

			testRoutes, exists := app.routeTests[route]

			if exists {
				for _, testRoute := range testRoutes {
					app.TestRoute(testRoute)
				}

				continue
			}

			// Routes with parameters
			if strings.Contains(route, ":") {
				color.Yellow(route)
				continue
			}

			app.TestRoute(route)
		}

		// json, _ := Post("https://html5.validator.nu/?out=json").Header("Content-Type", "text/html; charset=utf-8").Header("Content-Encoding", "gzip").Body(body).Send()
		// fmt.Println(json)
	}()
}

// TestRoute tests the given route.
func (app *Application) TestRoute(route string) {
	start := time.Now()
	body, _ := Get("http://localhost:" + strconv.Itoa(app.Config.Ports.HTTP) + route).Send()
	responseTime := time.Since(start).Nanoseconds() / 1000000
	responseSize := float64(len(body)) / 1024

	faint := color.New(color.Faint).SprintFunc()

	// Response size color
	var responseSizeColor func(a ...interface{}) string

	switch {
	case responseSize < 15:
		responseSizeColor = color.New(color.FgGreen).SprintFunc()
	case responseSize < 100:
		responseSizeColor = color.New(color.FgYellow).SprintFunc()
	default:
		responseSizeColor = color.New(color.FgRed).SprintFunc()
	}

	// Response time color
	var responseTimeColor func(a ...interface{}) string

	switch {
	case responseTime < 10:
		responseTimeColor = color.New(color.FgGreen).SprintFunc()
	case responseTime < 100:
		responseTimeColor = color.New(color.FgYellow).SprintFunc()
	default:
		responseTimeColor = color.New(color.FgRed).SprintFunc()
	}

	fmt.Printf("%-67s %s %s %s %s\n", color.BlueString(route), responseSizeColor(fmt.Sprintf("%6.0f", responseSize)), faint("KB"), responseTimeColor(fmt.Sprintf("%7d", responseTime)), faint("ms"))
}
<commit_msg>Removed DOCTYPE (in pixy now)<commit_after>package aero

import (
	"fmt"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"time"

	"crypto/sha256"

	"encoding/base64"
	"encoding/json"

	"io/ioutil"

	"github.com/aerogo/session"
	memstore "github.com/aerogo/session-store-memory"
	"github.com/fatih/color"
	"github.com/julienschmidt/httprouter"
	cache "github.com/patrickmn/go-cache"
)

const (
	gzipCacheDuration = 5 * time.Minute
	gzipCacheCleanup  = 1 * time.Minute
)

// Middleware ...
type Middleware func(*Context, func())

// Application represents a single web service.
type Application struct {
	root string

	Config   *Configuration
	Layout   func(*Context, string) string
	Sessions session.Manager
	Security ApplicationSecurity

	router *httprouter.Router
	routes struct {
		GET  []string
		POST []string
	}
	routeTests map[string][]string
	gzipCache  *cache.Cache
	start      time.Time
	rewrite    func(*RewriteContext)

	middleware []Middleware

	css            string
	cssHash        string
	cssReplacement string

	contentSecurityPolicy string
}

// New creates a new application.
func New() *Application {
	app := new(Application)
	app.start = time.Now()
	app.router = httprouter.New()
	app.routeTests = make(map[string][]string)
	app.gzipCache = cache.New(gzipCacheDuration, gzipCacheCleanup)
	app.Layout = func(ctx *Context, content string) string {
		return content
	}
	app.Config = new(Configuration)
	app.Config.Reset()
	app.Load()

	app.Sessions.Store = memstore.New()

	return app
}

// Get registers your function to be called when a certain GET path has been requested.
func (app *Application) Get(path string, handle Handle) {
	app.routes.GET = append(app.routes.GET, path)
	app.router.GET(path, app.createRouteHandler(path, handle))
}

// Post registers your function to be called when a certain POST path has been requested.
func (app *Application) Post(path string, handle Handle) {
	app.routes.POST = append(app.routes.POST, path)
	app.router.POST(path, app.createRouteHandler(path, handle))
}

// createRouteHandler creates a handler function for httprouter.
func (app *Application) createRouteHandler(path string, handle Handle) httprouter.Handle {
	return func(response http.ResponseWriter, request *http.Request, params httprouter.Params) {
		// Create context.
		ctx := Context{
			App:        app,
			StatusCode: http.StatusOK,
			request:    request,
			response:   response,
			params:     params,
			start:      time.Now(),
		}

		// The last part of the call chain will send the actual response.
		lastPartOfCallChain := func() {
			data := handle(&ctx)
			ctx.Respond(data)
		}

		// Declare the type of generateNext so that we can define it recursively in the next part.
		var generateNext func(index int) func()

		// Create a function that returns a bound function next()
		// which can be used as the 2nd parameter in the call chain.
		generateNext = func(index int) func() {
			if index == len(app.middleware) {
				return lastPartOfCallChain
			}

			return func() {
				app.middleware[index](&ctx, generateNext(index+1))
			}
		}

		generateNext(0)()
	}
}

// Ajax calls app.Get for both /route and /_/route
func (app *Application) Ajax(path string, handle Handle) {
	app.Get("/_"+path, handle)
	app.Get(path, func(ctx *Context) string {
		page := handle(ctx)
		html := app.Layout(ctx, page)
		html = strings.Replace(html, "<body", app.cssReplacement, 1)
		return html
	})
}

// Run starts your application.
func (app *Application) Run() {
	app.TestManifest()
	app.TestRoutes()
	app.Listen()
}

// Use adds middleware to your middleware chain.
func (app *Application) Use(middlewares ...Middleware) {
	app.middleware = append(app.middleware, middlewares...)
}

// Load loads the application data from the file system.
func (app *Application) Load() {
	config, readError := ioutil.ReadFile("config.json")

	if readError == nil {
		jsonDecodeError := json.Unmarshal(config, app.Config)

		if jsonDecodeError != nil {
			color.Red(jsonDecodeError.Error())
		}
	}

	if app.Config.Manifest.Name == "" {
		app.Config.Manifest.Name = app.Config.Title
	}

	if app.Config.Manifest.ShortName == "" {
		app.Config.Manifest.ShortName = app.Config.Title
	}

	if app.Config.Manifest.Lang == "" {
		app.Config.Manifest.Lang = "en"
	}

	if app.Config.Manifest.Display == "" {
		app.Config.Manifest.Display = "fullscreen"
	}

	if app.Config.Manifest.StartURL == "" {
		app.Config.Manifest.StartURL = "/"
	}
}

// Listen starts the server.
func (app *Application) Listen() {
	if app.Security.Key != "" && app.Security.Certificate != "" {
		go func() {
			app.serveHTTPS(":" + strconv.Itoa(app.Config.Ports.HTTPS))
		}()

		fmt.Println("Server running on:", color.GreenString("https://localhost:"+strconv.Itoa(app.Config.Ports.HTTPS)))
	} else {
		fmt.Println("Server running on:", color.GreenString("http://localhost:"+strconv.Itoa(app.Config.Ports.HTTP)))
	}

	app.serveHTTP(":" + strconv.Itoa(app.Config.Ports.HTTP))
}

// // listen listens on the specified host and port.
// func (app *Application) listen(port int) net.Listener {
// 	address := ":" + strconv.Itoa(port)

// 	listener, bindError := net.Listen("tcp", address)

// 	if bindError != nil {
// 		panic(bindError)
// 	}

// 	return listener
// }

// Rewrite sets the URL rewrite function.
func (app *Application) Rewrite(rewrite func(*RewriteContext)) {
	app.rewrite = rewrite
}

// SetStyle ...
func (app *Application) SetStyle(css string) {
	app.css = css

	hash := sha256.Sum256([]byte(css))
	app.cssHash = base64.StdEncoding.EncodeToString(hash[:])
	app.cssReplacement = "<body"
	app.contentSecurityPolicy = "default-src 'none'; img-src https:; script-src 'self'; style-src 'sha256-" + app.cssHash + "'; font-src https:; manifest-src 'self'; child-src https:; connect-src https: wss:"
}

// StartTime ...
func (app *Application) StartTime() time.Time {
	return app.start
}

// Handler returns the request handler.
func (app *Application) Handler() http.Handler {
	router := app.router
	rewrite := app.rewrite

	if rewrite != nil {
		return &rewriteHandler{
			rewrite: rewrite,
			router:  router,
		}
	}

	return router
}

// serveHTTP serves requests from the given listener.
func (app *Application) serveHTTP(address string) {
	serveError := http.ListenAndServe(address, app.Handler())

	if serveError != nil {
		panic(serveError)
	}
}

// serveHTTPS serves requests from the given listener.
func (app *Application) serveHTTPS(address string) {
	serveError := http.ListenAndServeTLS(address, app.Security.Certificate, app.Security.Key, app.Handler())

	if serveError != nil {
		panic(serveError)
	}
}

// Test tests the given URI paths when the application starts.
func (app *Application) Test(route string, paths []string) {
	app.routeTests[route] = paths
}

// TestManifest tests your application's manifest.
func (app *Application) TestManifest() {
	manifest := app.Config.Manifest

	if len(manifest.ShortName) >= 12 {
		color.Yellow("The short name of your application should have less than 12 characters")
	}
}

// TestRoutes tests your application's routes.
func (app *Application) TestRoutes() {
	fmt.Println(strings.Repeat("-", 80))

	go func() {
		sort.Strings(app.routes.GET)

		for _, route := range app.routes.GET {
			// Ajax routes
			if strings.HasPrefix(route, "/_") {
				continue
			}

			testRoutes, exists := app.routeTests[route]

			if exists {
				for _, testRoute := range testRoutes {
					app.TestRoute(testRoute)
				}

				continue
			}

			// Routes with parameters
			if strings.Contains(route, ":") {
				color.Yellow(route)
				continue
			}

			app.TestRoute(route)
		}

		// json, _ := Post("https://html5.validator.nu/?out=json").Header("Content-Type", "text/html; charset=utf-8").Header("Content-Encoding", "gzip").Body(body).Send()
		// fmt.Println(json)
	}()
}

// TestRoute tests the given route.
func (app *Application) TestRoute(route string) {
	start := time.Now()
	body, _ := Get("http://localhost:" + strconv.Itoa(app.Config.Ports.HTTP) + route).Send()
	responseTime := time.Since(start).Nanoseconds() / 1000000
	responseSize := float64(len(body)) / 1024

	faint := color.New(color.Faint).SprintFunc()

	// Response size color
	var responseSizeColor func(a ...interface{}) string

	switch {
	case responseSize < 15:
		responseSizeColor = color.New(color.FgGreen).SprintFunc()
	case responseSize < 100:
		responseSizeColor = color.New(color.FgYellow).SprintFunc()
	default:
		responseSizeColor = color.New(color.FgRed).SprintFunc()
	}

	// Response time color
	var responseTimeColor func(a ...interface{}) string

	switch {
	case responseTime < 10:
		responseTimeColor = color.New(color.FgGreen).SprintFunc()
	case responseTime < 100:
		responseTimeColor = color.New(color.FgYellow).SprintFunc()
	default:
		responseTimeColor = color.New(color.FgRed).SprintFunc()
	}

	fmt.Printf("%-67s %s %s %s %s\n", color.BlueString(route), responseSizeColor(fmt.Sprintf("%6.0f", responseSize)), faint("KB"), responseTimeColor(fmt.Sprintf("%7d", responseTime)), faint("ms"))
}
<|endoftext|>


<commit_before>package txtdirect

import (
	"bytes"
	"compress/gzip"
	"context"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/cretz/bine/tor"
	"github.com/mholt/caddy"
	cproxy "github.com/mholt/caddy/caddyhttp/proxy"
	"golang.org/x/net/proxy"
	lumberjack "gopkg.in/natefinch/lumberjack.v2"
)

// DefaultOnionServicePort is the port used to serve the onion service on
const DefaultOnionServicePort = 4242

// Tor type config struct
type Tor struct {
	Enable bool
	// Socks5 proxy port
	Port      int
	DataDir   string
	Torrc     string
	DebugMode bool
	LogFile   string

	instance        *tor.Tor
	contextCanceler context.CancelFunc
	onion           *tor.OnionService
}

type TorResponse struct {
	headers    http.Header
	body       []byte
	bodyReader bytes.Buffer
	bodyWriter bytes.Buffer
	status     int
}

// TODO: Discuss these values
const (
	torProxyKeepalive = 30000000
	torFallbackDelay  = 30000000 * time.Millisecond
	torProxyTimeout   = 30000000 * time.Second
)

var bufferPool = sync.Pool{New: createBuffer}

func createBuffer() interface{} {
	return make([]byte, 0, 32*1024)
}

func (t *Tor) Start(c *caddy.Controller) {
	var debugger io.Writer
	if t.DebugMode {
		if t.LogFile != "" {
			debugger = &lumberjack.Logger{
				Filename:   t.LogFile,
				MaxSize:    100,
				MaxAge:     14,
				MaxBackups: 10,
			}
		}
		debugger = os.Stdout
	}

	torInstance, err := tor.Start(nil, &tor.StartConf{
		NoAutoSocksPort: true,
		ExtraArgs:       []string{"--SocksPort", strconv.Itoa(t.Port)},
		TempDataDirBase: t.DataDir,
		TorrcFile:       t.Torrc,
		DebugWriter:     debugger,
	})
	if err != nil {
		log.Panicf("Unable to start Tor: %v", err)
	}

	listenCtx := context.Background()

	onion, err := torInstance.Listen(listenCtx, &tor.ListenConf{LocalPort: 8868, RemotePorts: []int{80}})
	if err != nil {
		log.Panicf("Unable to start onion service: %v", err)
	}

	t.onion = onion
	t.instance = torInstance
}

// Stop stops the tor instance, context listener and the onion service
func (t *Tor) Stop() error {
	if err := t.instance.Close(); err != nil {
		return fmt.Errorf("[txtdirect]: Couldn't close the tor instance. %s", err.Error())
	}
	t.onion.Close()
	return nil
}

// Proxy redirects the request to the local onion serivce and the actual proxying
// happens inside onion service's http handler
func (t *Tor) Proxy(w http.ResponseWriter, r *http.Request, rec record, c Config) error {
	u, err := url.Parse(rec.To)
	if err != nil {
		return err
	}

	// Use Proxied Host
	r.Host = u.Host

	// Create a socks5 dialer
	dialer, err := proxy.SOCKS5("tcp", fmt.Sprintf("127.0.0.1:%d", t.Port), nil, proxy.Direct)
	if err != nil {
		log.Fatal(err)
	}

	reverseProxy := cproxy.NewSingleHostReverseProxy(u, "", torProxyKeepalive, torProxyTimeout, torFallbackDelay)
	reverseProxy.Transport = &http.Transport{
		Dial: dialer.Dial,
	}

	tmpResposne := TorResponse{headers: make(http.Header)}
	if err := reverseProxy.ServeHTTP(&tmpResposne, r, nil); err != nil {
		return fmt.Errorf("[txtdirect]: Coudln't proxy the request to the background onion service. %s", err.Error())
	}

	// Decompress the body based on "Content-Encoding" header and write to a writer buffer
	if err := tmpResposne.WriteBody(); err != nil {
		return fmt.Errorf("[txtdirect]: Couldn't write the response body: %s", err.Error())
	}

	// Replace the URL hosts with the request's host
	if err := tmpResposne.ReplaceBody(u.Scheme, u.Host, r.Host); err != nil {
		return fmt.Errorf("[txtdirect]: Couldn't replace urls inside the response body: %s", err.Error())
	}

	copyHeader(w.Header(), tmpResposne.Header())

	// Write the final response from the temporary ResponseWriter to the main ResponseWriter
	if _, err := w.Write(tmpResposne.Body()); err != nil {
		return fmt.Errorf("[txtdirect]: Couldn't write the temporary response to main response body: %s", err.Error())
	}

	return nil
}

// ParseTor parses the txtdirect config for Tor proxy
func (t *Tor) ParseTor(c *caddy.Controller) error {
	switch c.Val() {
	case "port":
		value, err := strconv.Atoi(c.RemainingArgs()[0])
		if err != nil {
			return fmt.Errorf("The given value for port field is not standard. It should be an integer")
		}
		t.Port = value

	case "datadir":
		t.DataDir = c.RemainingArgs()[0]

	case "torrc":
		t.Torrc = c.RemainingArgs()[0]

	case "debug_mode":
		value, err := strconv.ParseBool(c.RemainingArgs()[0])
		if err != nil {
			return fmt.Errorf("The given value for debug_mode field is not standard. It should be a boolean")
		}
		t.DebugMode = value

	case "logfile":
		t.LogFile = c.RemainingArgs()[0]

	default:
		return c.ArgErr() // unhandled option for tor
	}
	return nil
}

// SetDefaults sets the default values for prometheus config
// if the fields are empty
func (t *Tor) SetDefaults() {
	if t.Port == 0 {
		t.Port = DefaultOnionServicePort
	}
}

// Header returns response headers
func (t *TorResponse) Header() http.Header {
	return t.headers
}

func (t *TorResponse) Write(body []byte) (int, error) {
	reader := bytes.NewReader(body)
	pooledIoCopy(&t.bodyReader, reader)
	t.body = body
	return len(body), nil
}

// Body returns response's body. This method should only get called after WriteBody()
func (t *TorResponse) Body() []byte {
	return t.bodyWriter.Bytes()
}

// WriteHeader Writes the given status code to response
func (t *TorResponse) WriteHeader(status int) {
	t.status = status
}

func (t *TorResponse) ReplaceBody(scheme, to, host string) error {
	replacedBody := bytes.Replace(t.bodyWriter.Bytes(), []byte(scheme+"://"+to), []byte(scheme+"://"+host), -1)
	t.bodyWriter.Reset()
	if _, err := t.bodyWriter.Write(replacedBody); err != nil {
		return err
	}
	return nil
}

func (t *TorResponse) WriteBody() error {
	switch t.Header().Get("Content-Encoding") {
	case "gzip":
		reader, err := gzip.NewReader(&t.bodyReader)
		if err != nil {
			return err
		}
		defer reader.Close()
		_, err = io.Copy(&t.bodyWriter, reader)
		if err != nil {
			return err
		}
		t.Header().Del("Content-Encoding")
	default:
		_, err := io.Copy(&t.bodyWriter, &t.bodyReader)
		if err != nil {
			return err
		}
	}
	return nil
}

var skipHeaders = map[string]struct{}{
	"Content-Type":        {},
	"Content-Disposition": {},
	"Accept-Ranges":       {},
	"Set-Cookie":          {},
	"Cache-Control":       {},
	"Expires":             {},
}

func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		if _, ok := dst[k]; ok {
			if _, shouldSkip := skipHeaders[k]; shouldSkip {
				continue
			}
			if k != "Server" {
				dst.Del(k)
			}
		}
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

func pooledIoCopy(dst io.Writer, src io.Reader) {
	buf := bufferPool.Get().([]byte)
	defer bufferPool.Put(buf)

	bufCap := cap(buf)
	io.CopyBuffer(dst, src, buf[0:bufCap:bufCap])
}
<commit_msg>Fix typo<commit_after>package txtdirect

import (
	"bytes"
	"compress/gzip"
	"context"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/cretz/bine/tor"
	"github.com/mholt/caddy"
	cproxy "github.com/mholt/caddy/caddyhttp/proxy"
	"golang.org/x/net/proxy"
	lumberjack "gopkg.in/natefinch/lumberjack.v2"
)

// DefaultOnionServicePort is the port used to serve the onion service on
const DefaultOnionServicePort = 4242

// Tor type config struct
type Tor struct {
	Enable bool
	// Socks5 proxy port
	Port      int
	DataDir   string
	Torrc     string
	DebugMode bool
	LogFile   string

	instance        *tor.Tor
	contextCanceler context.CancelFunc
	onion           *tor.OnionService
}

type TorResponse struct {
	headers    http.Header
	body       []byte
	bodyReader bytes.Buffer
	bodyWriter bytes.Buffer
	status     int
}

// TODO: Discuss these values
const (
	torProxyKeepalive = 30000000
	torFallbackDelay  = 30000000 * time.Millisecond
	torProxyTimeout   = 30000000 * time.Second
)

var bufferPool = sync.Pool{New: createBuffer}

func createBuffer() interface{} {
	return make([]byte, 0, 32*1024)
}

func (t *Tor) Start(c *caddy.Controller) {
	var debugger io.Writer
	if t.DebugMode {
		if t.LogFile != "" {
			debugger = &lumberjack.Logger{
				Filename:   t.LogFile,
				MaxSize:    100,
				MaxAge:     14,
				MaxBackups: 10,
			}
		}
		debugger = os.Stdout
	}

	torInstance, err := tor.Start(nil, &tor.StartConf{
		NoAutoSocksPort: true,
		ExtraArgs:       []string{"--SocksPort", strconv.Itoa(t.Port)},
		TempDataDirBase: t.DataDir,
		TorrcFile:       t.Torrc,
		DebugWriter:     debugger,
	})
	if err != nil {
		log.Panicf("Unable to start Tor: %v", err)
	}

	listenCtx := context.Background()

	onion, err := torInstance.Listen(listenCtx, &tor.ListenConf{LocalPort: 8868, RemotePorts: []int{80}})
	if err != nil {
		log.Panicf("Unable to start onion service: %v", err)
	}

	t.onion = onion
	t.instance = torInstance
}

// Stop stops the tor instance, context listener and the onion service
func (t *Tor) Stop() error {
	if err := t.instance.Close(); err != nil {
		return fmt.Errorf("[txtdirect]: Couldn't close the tor instance. %s", err.Error())
	}
	t.onion.Close()
	return nil
}

// Proxy redirects the request to the local onion serivce and the actual proxying
// happens inside onion service's http handler
func (t *Tor) Proxy(w http.ResponseWriter, r *http.Request, rec record, c Config) error {
	u, err := url.Parse(rec.To)
	if err != nil {
		return err
	}

	// Use Proxied Host
	r.Host = u.Host

	// Create a socks5 dialer
	dialer, err := proxy.SOCKS5("tcp", fmt.Sprintf("127.0.0.1:%d", t.Port), nil, proxy.Direct)
	if err != nil {
		log.Fatal(err)
	}

	reverseProxy := cproxy.NewSingleHostReverseProxy(u, "", torProxyKeepalive, torProxyTimeout, torFallbackDelay)
	reverseProxy.Transport = &http.Transport{
		Dial: dialer.Dial,
	}

	tmpResponse := TorResponse{headers: make(http.Header)}
	if err := reverseProxy.ServeHTTP(&tmpResponse, r, nil); err != nil {
		return fmt.Errorf("[txtdirect]: Coudln't proxy the request to the background onion service. %s", err.Error())
	}

	// Decompress the body based on "Content-Encoding" header and write to a writer buffer
	if err := tmpResponse.WriteBody(); err != nil {
		return fmt.Errorf("[txtdirect]: Couldn't write the response body: %s", err.Error())
	}

	// Replace the URL hosts with the request's host
	if err := tmpResponse.ReplaceBody(u.Scheme, u.Host, r.Host); err != nil {
		return fmt.Errorf("[txtdirect]: Couldn't replace urls inside the response body: %s", err.Error())
	}

	copyHeader(w.Header(), tmpResponse.Header())

	// Write the final response from the temporary ResponseWriter to the main ResponseWriter
	if _, err := w.Write(tmpResponse.Body()); err != nil {
		return fmt.Errorf("[txtdirect]: Couldn't write the temporary response to main response body: %s", err.Error())
	}

	return nil
}

// ParseTor parses the txtdirect config for Tor proxy
func (t *Tor) ParseTor(c *caddy.Controller) error {
	switch c.Val() {
	case "port":
		value, err := strconv.Atoi(c.RemainingArgs()[0])
		if err != nil {
			return fmt.Errorf("The given value for port field is not standard. It should be an integer")
		}
		t.Port = value

	case "datadir":
		t.DataDir = c.RemainingArgs()[0]

	case "torrc":
		t.Torrc = c.RemainingArgs()[0]

	case "debug_mode":
		value, err := strconv.ParseBool(c.RemainingArgs()[0])
		if err != nil {
			return fmt.Errorf("The given value for debug_mode field is not standard. It should be a boolean")
		}
		t.DebugMode = value

	case "logfile":
		t.LogFile = c.RemainingArgs()[0]

	default:
		return c.ArgErr() // unhandled option for tor
	}
	return nil
}

// SetDefaults sets the default values for prometheus config
// if the fields are empty
func (t *Tor) SetDefaults() {
	if t.Port == 0 {
		t.Port = DefaultOnionServicePort
	}
}

// Header returns response headers
func (t *TorResponse) Header() http.Header {
	return t.headers
}

func (t *TorResponse) Write(body []byte) (int, error) {
	reader := bytes.NewReader(body)
	pooledIoCopy(&t.bodyReader, reader)
	t.body = body
	return len(body), nil
}

// Body returns response's body. This method should only get called after WriteBody()
func (t *TorResponse) Body() []byte {
	return t.bodyWriter.Bytes()
}

// WriteHeader Writes the given status code to response
func (t *TorResponse) WriteHeader(status int) {
	t.status = status
}

func (t *TorResponse) ReplaceBody(scheme, to, host string) error {
	replacedBody := bytes.Replace(t.bodyWriter.Bytes(), []byte(scheme+"://"+to), []byte(scheme+"://"+host), -1)
	t.bodyWriter.Reset()
	if _, err := t.bodyWriter.Write(replacedBody); err != nil {
		return err
	}
	return nil
}

func (t *TorResponse) WriteBody() error {
	switch t.Header().Get("Content-Encoding") {
	case "gzip":
		reader, err := gzip.NewReader(&t.bodyReader)
		if err != nil {
			return err
		}
		defer reader.Close()
		_, err = io.Copy(&t.bodyWriter, reader)
		if err != nil {
			return err
		}
		t.Header().Del("Content-Encoding")
	default:
		_, err := io.Copy(&t.bodyWriter, &t.bodyReader)
		if err != nil {
			return err
		}
	}
	return nil
}

var skipHeaders = map[string]struct{}{
	"Content-Type":        {},
	"Content-Disposition": {},
	"Accept-Ranges":       {},
	"Set-Cookie":          {},
	"Cache-Control":       {},
	"Expires":             {},
}

func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		if _, ok := dst[k]; ok {
			if _, shouldSkip := skipHeaders[k]; shouldSkip {
				continue
			}
			if k != "Server" {
				dst.Del(k)
			}
		}
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

func pooledIoCopy(dst io.Writer, src io.Reader) {
	buf := bufferPool.Get().([]byte)
	defer bufferPool.Put(buf)

	bufCap := cap(buf)
	io.CopyBuffer(dst, src, buf[0:bufCap:bufCap])
}
<|endoftext|>


<commit_before>package main
import (
	"fmt"
	"strings"
	"strconv"
	"bufio"
	"os"
	"os/exec"
	"github.com/codegangsta/cli"
)

var CONFIG = make(map[string]string)
var REMOTE_REFS = make([]string, 0)

func list_pr(c *cli.Context) {
	if err := validate_repo(c); err != nil {
		fmt.Println("Could not list Pull Reqests")
		os.Exit(1)
	}

	fmt.Printf("%s\n", REMOTE_REFS)
	fmt.Printf("%s", CONFIG)
}

func apply_pr(c *cli.Context) {
	if err := validate_repo(c); err != nil {
		fmt.Println("Could not apply the Pull Request")
		os.Exit(1)
	}
	args := c.Args()
	fmt.Printf("%s\n", args)
}

func revert_master(c *cli.Context) {
	if err := validate_repo(c); err != nil {
		fmt.Println("Could not revert to master branch")
		os.Exit(1)
	}
}

func switchRef(c *cli.Context) {
	// switch ref

}

func validate_repo(c *cli.Context) (err error){
	_, gitErr := exec.Command("git", "rev-parse").Output()
	if gitErr != nil {
		fmt.Println("Current directory not under git version control")
		return gitErr
	} else {
		initialize_config()
		return nil
	}
}

func initialize_config() {
	output, err := exec.Command("git", "remote", "show").Output()

	if err != nil {
		fmt.Println("Error running 'git remote show'")
		os.Exit(1)
	}
	outputString := fmt.Sprintf("%s", string(output[:]))
	refs := strings.Split(outputString, "\n")
	for index := range refs {
		if len(refs[index]) != 0 {
			REMOTE_REFS = append(REMOTE_REFS, refs[index])
		}
	}
	if len(REMOTE_REFS) == 0 {
		fmt.Println("No remote refs defined")
		refName, refUrl := get_ref()
		_, err := exec.Command("git", "remote", "add", refName, refUrl).Output()
		if err != nil {
			fmt.Println("Error while inserting new git ref")
			os.Exit(1)
		}
		CONFIG["DEFAULT_REMOTE_REF"] = refName
		REMOTE_REFS[0] = refName
	} else if len(REMOTE_REFS) == 1 {
		CONFIG["DEFAULT_REMOTE_REF"] = REMOTE_REFS[0]
	} else {
		CONFIG["DEFAULT_REMOTE_REF"] = REMOTE_REFS[0]
		CONFIG["DEFAULT_REMOTE_REF"] = get_default_ref()
	}

	CONFIG["DEFAULT_BRANCH"] = "master"
}

func get_default_ref() (string) {
	fmt.Println("Choose ref to set as remote")
	for index := range REMOTE_REFS {
		fmt.Println("\t","(", (index+1), ") ", REMOTE_REFS[index])
	}
	reader := bufio.NewReader(os.Stdin)
	selected, _ := reader.ReadString('\n')
	selected = strings.TrimSpace(selected)
	index, _ := strconv.Atoi(selected)
	return REMOTE_REFS[index - 1]
}

func get_ref() (string, string) {
	reader := bufio.NewReader(os.Stdin)
	fmt.Print("Enter ref name (e.g. parent): ")
	name, _ := reader.ReadString('\n')
	name = strings.TrimSpace(name)

	fmt.Print("Enter the url (e.g. git@github.com:ric03uec/tpr.git: ")
	refUrl, _ := reader.ReadString('\n')
	refUrl = strings.TrimSpace(refUrl)

	return name, refUrl
}

func main() {
	app := cli.NewApp()
	app.Name = "tpr"
	app.Usage = "Test github pull requests locally"
	app.Version = "0.1.0"
	app.Commands = []cli.Command{
		{
			Name: "list",
			ShortName: "l",
			Usage: "List of all the Pull Requests",
			Action: list_pr,

		},
		{
			Name: "apply",
			ShortName: "a",
			Usage: "Apply the specified Pull Request",
			Action: apply_pr,
		},
		{
			Name: "revert",
			ShortName: "r",
			Usage: "Revert back to master branch",
			Action: revert_master,

		},
		{
			Name: "switch",
			ShortName: "s",
			Usage: "Switch default remote ref",
			Action: switchRef,
		},
	}

	app.Run(os.Args)
}
<commit_msg>listing all the branches<commit_after>package main
import (
	"fmt"
	"strings"
	"strconv"
	"bufio"
	"os"
	"os/exec"
	"github.com/codegangsta/cli"
)

var CONFIG = make(map[string]string)
var REMOTE_REFS = make([]string, 0)

func list_pr(c *cli.Context) {
	if err := validate_repo(c); err != nil {
		fmt.Println("Could not list Pull Reqests")
		os.Exit(1)
	}

	refSpec := fmt.Sprintf("refs/pull/*/head:refs/remotes/%s/pr/*", CONFIG["DEFAULT_REMOTE_REF"])
	pulls, err := exec.Command("git", "fetch", CONFIG["DEFAULT_REMOTE_REF"], refSpec).Output()
	if err != nil {
		fmt.Println("Could not fetch remote Pull Requests")
		os.Exit(1)
	}

	branches, _ := exec.Command("git", "branch", "-a").Output()
	fmt.Printf("%s\n", branches)
	fmt.Printf("%s\n", pulls)
}

func apply_pr(c *cli.Context) {
	if err := validate_repo(c); err != nil {
		fmt.Println("Could not apply the Pull Request")
		os.Exit(1)
	}
	args := c.Args()
	fmt.Printf("%s\n", args)
}

func revert_master(c *cli.Context) {
	if err := validate_repo(c); err != nil {
		fmt.Println("Could not revert to master branch")
		os.Exit(1)
	}
}

func switchRef(c *cli.Context) {
	// switch ref

}

func validate_repo(c *cli.Context) (err error){
	_, gitErr := exec.Command("git", "rev-parse").Output()
	if gitErr != nil {
		fmt.Println("Current directory not under git version control")
		return gitErr
	} else {
		initialize_config()
		return nil
	}
}

func initialize_config() {
	output, err := exec.Command("git", "remote", "show").Output()

	if err != nil {
		fmt.Println("Error running 'git remote show'")
		os.Exit(1)
	}
	outputString := fmt.Sprintf("%s", string(output[:]))
	refs := strings.Split(outputString, "\n")
	for index := range refs {
		if len(refs[index]) != 0 {
			REMOTE_REFS = append(REMOTE_REFS, refs[index])
		}
	}
	if len(REMOTE_REFS) == 0 {
		fmt.Println("No remote refs defined")
		refName, refUrl := get_ref()
		_, err := exec.Command("git", "remote", "add", refName, refUrl).Output()
		if err != nil {
			fmt.Println("Error while inserting new git ref")
			os.Exit(1)
		}
		CONFIG["DEFAULT_REMOTE_REF"] = refName
		REMOTE_REFS[0] = refName
	} else if len(REMOTE_REFS) == 1 {
		CONFIG["DEFAULT_REMOTE_REF"] = REMOTE_REFS[0]
	} else {
		CONFIG["DEFAULT_REMOTE_REF"] = REMOTE_REFS[0]
		CONFIG["DEFAULT_REMOTE_REF"] = get_default_ref()
	}

	CONFIG["DEFAULT_BRANCH"] = "master"
}

func get_default_ref() (string) {
	fmt.Println("Choose ref to set as remote")
	for index := range REMOTE_REFS {
		fmt.Println("\t","(", (index+1), ") ", REMOTE_REFS[index])
	}
	reader := bufio.NewReader(os.Stdin)
	selected, _ := reader.ReadString('\n')
	selected = strings.TrimSpace(selected)
	index, _ := strconv.Atoi(selected)
	return REMOTE_REFS[index - 1]
}

func get_ref() (string, string) {
	reader := bufio.NewReader(os.Stdin)
	fmt.Print("Enter ref name (e.g. parent): ")
	name, _ := reader.ReadString('\n')
	name = strings.TrimSpace(name)

	fmt.Print("Enter the url (e.g. git@github.com:ric03uec/tpr.git: ")
	refUrl, _ := reader.ReadString('\n')
	refUrl = strings.TrimSpace(refUrl)

	return name, refUrl
}

func main() {
	app := cli.NewApp()
	app.Name = "tpr"
	app.Usage = "Test github pull requests locally"
	app.Version = "0.1.0"
	app.Commands = []cli.Command{
		{
			Name: "list",
			ShortName: "l",
			Usage: "List of all the Pull Requests",
			Action: list_pr,

		},
		{
			Name: "apply",
			ShortName: "a",
			Usage: "Apply the specified Pull Request",
			Action: apply_pr,
		},
		{
			Name: "revert",
			ShortName: "r",
			Usage: "Revert back to master branch",
			Action: revert_master,

		},
		{
			Name: "switch",
			ShortName: "s",
			Usage: "Switch default remote ref",
			Action: switchRef,
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"log"
)

func loadNameWithSynset(synset string) string {
	db, err := sql.Open("sqlite3", "wnjpn.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	sql := "select name from synset where synset=?;"
	stmt, err := db.Prepare(sql)
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()
	var name string
	err = stmt.QueryRow(synset).Scan(&name)

	if err != nil {
		log.Fatal(err, ": DBにwordidが見つかりません")
	}
	return name
}

func loadSensesWithWordid(wordid int) []string {
	db, err := sql.Open("sqlite3", "wnjpn.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	sql := "select synset from sense where wordid=?;"
	stmt, err := db.Prepare(sql)
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()
	rows, err := stmt.Query(wordid)
	defer rows.Close()
	synsets := []string{}
	for i := 0; rows.Next(); i++ {
		var synset string
		rows.Scan(&synset)
		synsets = append(synsets, synset)
	}

	if err != nil {
		log.Fatal(err, ": DBにwordidが見つかりません")
	}
	return synsets
}

func loadWordsWithLemmma(lemma string) int {
	db, err := sql.Open("sqlite3", "wnjpn.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	sql := "select wordid from word where lemma = ?;"
	stmt, err := db.Prepare(sql)
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()
	var wordid int
	err = stmt.QueryRow("夢").Scan(&wordid)

	if err != nil {
		log.Fatal(err, ": DBに単語が見つかりません")
	}
	return wordid
}

func main() {
	lemma := "夢"
	wordid := loadWordsWithLemmma(lemma)
	synsets := loadSensesWithWordid(wordid)
	for i := 0; i < len(synsets); i++ {
		fmt.Println(loadNameWithSynset(synsets[i]))
	}
	fmt.Println(synsets)
}
<commit_msg>infinity recursion<commit_after>package main

import (
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"log"
)

func loadNameWithSynsetFromSynset(synset string) string {
	db, err := sql.Open("sqlite3", "wnjpn.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	sql := "select name from synset where synset=?;"
	stmt, err := db.Prepare(sql)
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()
	var name string
	err = stmt.QueryRow(synset).Scan(&name)

	if err != nil {
		log.Fatal(err, ": DBにwordidが見つかりません")
	}
	return name
}

func loadSynsetsWithWordidFromSense(wordid string) []string {
	db, err := sql.Open("sqlite3", "wnjpn.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	sql := "select synset from sense where wordid=?;"
	stmt, err := db.Prepare(sql)
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()
	rows, err := stmt.Query(wordid)
	defer rows.Close()
	synsets := []string{}
	for i := 0; rows.Next(); i++ {
		var synset string
		rows.Scan(&synset)
		synsets = append(synsets, synset)
	}

	if err != nil {
		log.Fatal(err, ": senseにwordidが見つかりません")
	}
	return synsets
}

func loadWordsWithLemmmaFromWord(lemma string) []string {
	db, err := sql.Open("sqlite3", "wnjpn.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	sql := "select wordid from word where lemma = ?;"
	stmt, err := db.Prepare(sql)
	if err != nil {
		log.Fatal(err)
	}
	rows, err := stmt.Query(lemma)
	defer rows.Close()
	var wordids []string
	for i := 0; rows.Next(); i++ {
		var wordid string
		rows.Scan(&wordid)
		wordids = append(wordids, wordid)
	}

	if err != nil {
		log.Fatal(err, ": wordにwordidが見つかりません")
	}
	return wordids
}

func loadSynsetsWithSynsetFromSynlink(synset string) []string {
	db, err := sql.Open("sqlite3", "wnjpn.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	sql := "select synset2 from synlink where synset1=? and link='hype';"
	stmt, err := db.Prepare(sql)
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()
	rows, err := stmt.Query(synset)
	defer rows.Close()
	synsets := []string{}
	for i := 0; rows.Next(); i++ {
		var synset2 string
		rows.Scan(&synset2)
		synsets = append(synsets, synset2)
	}

	if err != nil {
		log.Fatal(err, ": synlinkにwordidが見つかりません")
	}
	return synsets
}

func loadLemmaWithWordidFromWord(wordid string) string {
	db, err := sql.Open("sqlite3", "wnjpn.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	sql := "select lemma from word where wordid = ?;"
	stmt, err := db.Prepare(sql)
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()
	var lemma string
	err = stmt.QueryRow(wordid).Scan(&lemma)

	if err != nil {
		log.Fatal(err, ": wordに単語が見つかりません")
	}
	return lemma
}

func loadWordidWithSynsetFromSense(synset string) string {
	db, err := sql.Open("sqlite3", "wnjpn.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	sql := "select wordid from sense where synset = ?;"
	stmt, err := db.Prepare(sql)
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()
	var wordid string
	err = stmt.QueryRow(synset).Scan(&wordid)

	if err != nil {
		log.Fatal(err, ": senseに単語が見つかりません", synset)
	}
	return wordid
}
func printSynlinksRecursively(wordids []string, depth int) {
	for j := 0; j < len(wordids); j++ {
		synsets := loadSynsetsWithWordidFromSense(wordids[j])
		synlinksSynsets := []string{}
		for i := 0; i < len(synsets); i++ {
			synlinksSynsets = append(synlinksSynsets, loadSynsetsWithSynsetFromSynlink(synsets[i])...)
		}
		deeperWordids := []string{}
		if len(synlinksSynsets) != 0 {
			for i := 0; i < depth+1; i++ {
				fmt.Print("  ")
			}
			lemma := loadLemmaWithWordidFromWord(wordids[j])
			fmt.Println("  ", lemma)
			for i := 0; i < len(synlinksSynsets); i++ {
				newWordid := loadWordidWithSynsetFromSense(synlinksSynsets[i])
				deeperWordids = append(deeperWordids, newWordid)
			}
			printSynlinksRecursively(deeperWordids, depth+1)
		}
	}
}

func main() {
	lemma := "夢"
	wordids := loadWordsWithLemmmaFromWord(lemma)
	printSynlinksRecursively(wordids, 0)
}
<|endoftext|>


<commit_before>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"encoding/json"
	"github.com/globocom/tsuru/api"
	"github.com/globocom/tsuru/auth"
	"io"
	"io/ioutil"
	"net/http"
)

func init() {
	api.RegisterHandler("/node", "GET", api.AdminRequiredHandler(listNodeHandler))
	api.RegisterHandler("/node/:address/containers", "GET", api.AdminRequiredHandler(listContainersByHostHandler))
	api.RegisterHandler("/node/:appname/containers", "GET", api.AdminRequiredHandler(listContainersByAppHandler))
	api.RegisterAdminHandler("/node/add", "POST", api.Handler(addNodeHandler))
	api.RegisterAdminHandler("/node/remove", "DELETE", api.Handler(removeNodeHandler))
}

// addNodeHandler calls scheduler.Register to registering a node into it.
func addNodeHandler(w http.ResponseWriter, r *http.Request) error {
	params, err := unmarshal(r.Body)
	if err != nil {
		return err
	}
	return dockerCluster().Register(params)
}

// removeNodeHandler calls scheduler.Unregister to unregistering a node into it.
func removeNodeHandler(w http.ResponseWriter, r *http.Request) error {
	params, err := unmarshal(r.Body)
	if err != nil {
		return err
	}
	return dockerCluster().Unregister(params)
}

//listNodeHandler call scheduler.Nodes to list all nodes into it.
func listNodeHandler(w http.ResponseWriter, r *http.Request, t *auth.Token) error {
	nodeList, err := dockerCluster().Nodes()
	if err != nil {
		return err
	}
	return json.NewEncoder(w).Encode(nodeList)
}

//listContainersHandler call scheduler.Containers to list all containers into it.
func listContainersByHostHandler(w http.ResponseWriter, r *http.Request, t *auth.Token) error {
	n := r.URL.Query().Get(":address")
	containerList, err := listContainersByHost(n)
	if err != nil {
		return err
	}
	return json.NewEncoder(w).Encode(containerList)
}

func listContainersByAppHandler(w http.ResponseWriter, r *http.Request, t *auth.Token) error {
	n := r.URL.Query().Get(":appname")
	containerList, err := listContainersByApp(n)
	if err != nil {
		return err
	}
	return json.NewEncoder(w).Encode(containerList)
}

func unmarshal(body io.ReadCloser) (map[string]string, error) {
	b, err := ioutil.ReadAll(body)
	if err != nil {
		return nil, err
	}
	params := map[string]string{}
	err = json.Unmarshal(b, &params)
	if err != nil {
		return nil, err
	}
	return params, nil
}
<commit_msg>provision: modified listContainersByApp url<commit_after>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"encoding/json"
	"github.com/globocom/tsuru/api"
	"github.com/globocom/tsuru/auth"
	"io"
	"io/ioutil"
	"net/http"
)

func init() {
	api.RegisterHandler("/node", "GET", api.AdminRequiredHandler(listNodeHandler))
	api.RegisterHandler("/node/apps/:appname/containers", "GET", api.AdminRequiredHandler(listContainersByAppHandler))
	api.RegisterHandler("/node/:address/containers", "GET", api.AdminRequiredHandler(listContainersByHostHandler))
	api.RegisterAdminHandler("/node/add", "POST", api.Handler(addNodeHandler))
	api.RegisterAdminHandler("/node/remove", "DELETE", api.Handler(removeNodeHandler))
}

// addNodeHandler calls scheduler.Register to registering a node into it.
func addNodeHandler(w http.ResponseWriter, r *http.Request) error {
	params, err := unmarshal(r.Body)
	if err != nil {
		return err
	}
	return dockerCluster().Register(params)
}

// removeNodeHandler calls scheduler.Unregister to unregistering a node into it.
func removeNodeHandler(w http.ResponseWriter, r *http.Request) error {
	params, err := unmarshal(r.Body)
	if err != nil {
		return err
	}
	return dockerCluster().Unregister(params)
}

//listNodeHandler call scheduler.Nodes to list all nodes into it.
func listNodeHandler(w http.ResponseWriter, r *http.Request, t *auth.Token) error {
	nodeList, err := dockerCluster().Nodes()
	if err != nil {
		return err
	}
	return json.NewEncoder(w).Encode(nodeList)
}

//listContainersHandler call scheduler.Containers to list all containers into it.
func listContainersByHostHandler(w http.ResponseWriter, r *http.Request, t *auth.Token) error {
	n := r.URL.Query().Get(":address")
	containerList, err := listContainersByHost(n)
	if err != nil {
		return err
	}
	return json.NewEncoder(w).Encode(containerList)
}

func listContainersByAppHandler(w http.ResponseWriter, r *http.Request, t *auth.Token) error {
	n := r.URL.Query().Get(":appname")
	containerList, err := listContainersByApp(n)
	if err != nil {
		return err
	}
	return json.NewEncoder(w).Encode(containerList)
}

func unmarshal(body io.ReadCloser) (map[string]string, error) {
	b, err := ioutil.ReadAll(body)
	if err != nil {
		return nil, err
	}
	params := map[string]string{}
	err = json.Unmarshal(b, &params)
	if err != nil {
		return nil, err
	}
	return params, nil
}
<|endoftext|>