<commit_before>package main

import (
	"log"
	"strconv"
	"strings"
	"time"
	"sync"
	"fmt"

	"github.com/kidoman/embd"
	_ "github.com/kidoman/embd/host/all"
	"github.com/kidoman/embd/sensor/bmp180"
	"github.com/tarm/serial"

	"./mpu6050"
)

type SituationData struct {
	mu_GPS					*sync.Mutex

	// From GPS.
	lastFixSinceMidnightUTC uint32
	lat                     float32
	lng                     float32
	quality                 uint8
	satellites              uint16
	accuracy                float32 // Meters.
	alt                     float32 // Feet.
	alt_accuracy            float32
	lastFixLocalTime        time.Time
	trueCourse              uint16
	groundSpeed             uint16
	lastGroundTrackTime     time.Time

	mu_Attitude				*sync.Mutex

	// From BMP180 pressure sensor.
	temp					float64
	pressure_alt			float64
	lastTempPressTime		time.Time

	// From MPU6050 accel/gyro.
	pitch					float64
	roll					float64
	lastAttitudeTime		time.Time
}

var serialConfig *serial.Config
var serialPort *serial.Port

func initGPSSerial() bool {
	serialConfig = &serial.Config{Name: "/dev/ttyACM0", Baud: 9600}
	p, err := serial.OpenPort(serialConfig)
	if err != nil {
		log.Printf("serial port err: %s\n", err.Error())
		return false
	}
	serialPort = p
	return true
}

func processNMEALine(l string) bool {
	x := strings.Split(l, ",")
	if x[0] == "$GNVTG" { // Ground track information.
		mySituation.mu_GPS.Lock()
		defer mySituation.mu_GPS.Unlock()
		if len(x) < 10 {
			return false
		}
		trueCourse := uint16(0)
		if len(x[1]) > 0 {
			tc, err := strconv.ParseFloat(x[1], 32)
			if err != nil {
				return false
			}
			trueCourse = uint16(tc)
		} else {
			// No movement.
			mySituation.trueCourse = 0
			mySituation.groundSpeed = 0
			mySituation.lastGroundTrackTime = time.Time{}
			return true
		}
		groundSpeed, err := strconv.ParseFloat(x[5], 32) // Knots.
		if err != nil {
			return false
		}

		mySituation.trueCourse = uint16(trueCourse)
		mySituation.groundSpeed = uint16(groundSpeed)
		mySituation.lastGroundTrackTime = time.Now()

	} else if x[0] == "$GNGGA" { // GPS fix.
		if len(x) < 15 {
			return false
		}
		mySituation.mu_GPS.Lock()
		defer mySituation.mu_GPS.Unlock()
		// Timestamp.
		if len(x[1]) < 9 {
			return false
		}
		hr, err1 := strconv.Atoi(x[1][0:2])
		min, err2 := strconv.Atoi(x[1][2:4])
		sec, err3 := strconv.Atoi(x[1][4:6])
		if err1 != nil || err2 != nil || err3 != nil {
			return false
		}

		mySituation.lastFixSinceMidnightUTC = uint32((hr * 60 * 60) + (min * 60) + sec)

		// Latitude.
		if len(x[2]) < 10 {
			return false
		}
		hr, err1 = strconv.Atoi(x[2][0:2])
		minf, err2 := strconv.ParseFloat(x[2][2:10], 32)
		if err1 != nil || err2 != nil {
			return false
		}

		mySituation.lat = float32(hr) + float32(minf/60.0)
		if x[3] == "S" { // South = negative.
			mySituation.lat = -mySituation.lat
		}

		// Longitude.
		if len(x[4]) < 11 {
			return false
		}
		hr, err1 = strconv.Atoi(x[4][0:3])
		minf, err2 = strconv.ParseFloat(x[4][3:11], 32)
		if err1 != nil || err2 != nil {
			return false
		}

		mySituation.lng = float32(hr) + float32(minf/60.0)
		if x[5] == "W" { // West = negative.
			mySituation.lng = -mySituation.lng
		}

		// Quality indicator.
		q, err1 := strconv.Atoi(x[6])
		if err1 != nil {
			return false
		}
		mySituation.quality = uint8(q)

		// Satellites.
		sat, err1 := strconv.Atoi(x[7])
		if err1 != nil {
			return false
		}
		mySituation.satellites = uint16(sat)

		// Accuracy.
		hdop, err1 := strconv.ParseFloat(x[8], 32)
		if err1 != nil {
			return false
		}
		mySituation.accuracy = float32(hdop * 5.0) //FIXME: 5 meters ~ 1.0 HDOP?

		// Altitude.
		alt, err1 := strconv.ParseFloat(x[9], 32)
		if err1 != nil {
			return false
		}
		mySituation.alt = float32(alt * 3.28084) // Covnert to feet.

		//TODO: Altitude accuracy.
		mySituation.alt_accuracy = 0

		// Timestamp.
		mySituation.lastFixLocalTime = time.Now()

	}
	return true
}

func gpsSerialReader() {
	defer serialPort.Close()
	for globalSettings.GPS_Enabled && globalStatus.GPS_connected {
		buf := make([]byte, 1024)
		n, err := serialPort.Read(buf)
		if err != nil {
			log.Printf("gps serial read error: %s\n", err.Error())
			globalStatus.GPS_connected = false
			break
		}
		s := string(buf[:n])
		x := strings.Split(s, "\n")
		for _, l := range x {
			processNMEALine(l)
		}
	}
	globalStatus.GPS_connected = false
}

var i2cbus embd.I2CBus
var myBMP180 *bmp180.BMP180
var myMPU6050 *mpu6050.MPU6050

func readBMP180() (float64, float64, error) { // ºCelsius, Meters
	temp, err := myBMP180.Temperature()
	if err != nil {
		return temp, 0.0, err
	}
	altitude, err := myBMP180.Altitude()
	if err != nil {
		return temp, altitude, err
	}
	return temp, altitude, nil
}

func readMPU6050() (float64, float64, error) {//TODO: error checking.
	pitch, roll := myMPU6050.PitchAndRoll()
	return pitch, roll, nil
}

func initBMP180() error {
	myBMP180 = bmp180.New(i2cbus) //TODO: error checking.
	return nil
}

func initMPU6050() error {
	myMPU6050 = mpu6050.New(i2cbus) //TODO: error checking.
	return nil
}

func initI2C() error {
	i2cbus = embd.NewI2CBus(1) //TODO: error checking.
	return nil
}

// Unused at the moment. 5 second update, since read functions in bmp180 are slow.
func tempAndPressureReader() {
	timer := time.NewTicker(5 * time.Second)
	for globalStatus.RY835AI_connected && globalSettings.AHRS_Enabled {
		<-timer.C
		// Read temperature and pressure altitude.
		temp, alt, err_bmp180 := readBMP180()
		// Process.
		if err_bmp180 != nil {
			log.Printf("readBMP180(): %s\n", err_bmp180.Error())
			globalStatus.RY835AI_connected = false
		} else {
			mySituation.temp = temp
			mySituation.pressure_alt = alt
			mySituation.lastTempPressTime = time.Now()
		}
	}
	globalStatus.RY835AI_connected = false
}

func attitudeReaderSender() {
	timer := time.NewTicker(100 * time.Millisecond) // ~10Hz update.
	for globalStatus.RY835AI_connected && globalSettings.AHRS_Enabled {
		<-timer.C
		// Read pitch and roll.
		pitch, roll, err_mpu6050 := readMPU6050()

		mySituation.mu_Attitude.Lock()

		if err_mpu6050 != nil {
			log.Printf("readMPU6050(): %s\n", err_mpu6050.Error())
			globalStatus.RY835AI_connected = false
			break
		} else {
			mySituation.pitch = pitch
			mySituation.roll = roll
			mySituation.lastAttitudeTime = time.Now()
		}

		// Send, if valid.
//		if isGPSGroundTrackValid()
		s := fmt.Sprintf("XATTStratux,%d,%f,%f", mySituation.trueCourse, mySituation.pitch, mySituation.roll)

		sendMsg([]byte(s), NETWORK_AHRS)

		mySituation.mu_Attitude.Unlock()
	}
	globalStatus.RY835AI_connected = false
}

func isGPSValid() bool {
	return time.Since(mySituation.lastFixLocalTime).Seconds() < 15
}

func isGPSGroundTrackValid() bool {
	return time.Since(mySituation.lastGroundTrackTime).Seconds() < 15
}

func isAHRSValid() bool {
	return time.Since(mySituation.lastAttitudeTime).Seconds() < 1 // If attitude information gets to be over 1 second old, declare invalid.
}

func initAHRS() error {
	if err := initI2C(); err != nil { // I2C bus.
		return err
	}
	if err := initBMP180(); err != nil { // I2C temperature and pressure altitude.
		i2cbus.Close()
		return err
	}
	if err := initMPU6050(); err != nil { // I2C accel/gyro.
		i2cbus.Close()
		myBMP180.Close()
		return err
	}
	globalStatus.RY835AI_connected = true
	go attitudeReaderSender()
	go tempAndPressureReader()

	return nil
}

func pollRY835AI() {
	timer := time.NewTicker(10 * time.Second)
	for {
		<-timer.C
		// GPS enabled, was not connected previously?
		if globalSettings.GPS_Enabled && !globalStatus.GPS_connected {
			globalStatus.GPS_connected = initGPSSerial() // via USB for now.
			if globalStatus.GPS_connected {
				go gpsSerialReader()
			}
		}
		// RY835AI I2C enabled, was not connected previously?
		if globalSettings.AHRS_Enabled && !globalStatus.RY835AI_connected {
			err := initAHRS()
			if err != nil {
				log.Printf("initAHRS(): %s\ndisabling AHRS sensors.\n", err.Error())
				globalStatus.RY835AI_connected = false
			}
		}
	}
}

func initRY835AI() {
	mySituation.mu_GPS = &sync.Mutex{}
	mySituation.mu_Attitude = &sync.Mutex{}

	go pollRY835AI()
}
<commit_msg>Use BMP180 for pressure altitude in Ownship report.<commit_after>package main

import (
	"fmt"
	"log"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/kidoman/embd"
	_ "github.com/kidoman/embd/host/all"
	"github.com/kidoman/embd/sensor/bmp180"
	"github.com/tarm/serial"

	"./mpu6050"
)

type SituationData struct {
	mu_GPS *sync.Mutex

	// From GPS.
	lastFixSinceMidnightUTC uint32
	lat                     float32
	lng                     float32
	quality                 uint8
	satellites              uint16
	accuracy                float32 // Meters.
	alt                     float32 // Feet.
	alt_accuracy            float32
	lastFixLocalTime        time.Time
	trueCourse              uint16
	groundSpeed             uint16
	lastGroundTrackTime     time.Time

	mu_Attitude *sync.Mutex

	// From BMP180 pressure sensor.
	temp              float64
	pressure_alt      float64
	lastTempPressTime time.Time

	// From MPU6050 accel/gyro.
	pitch            float64
	roll             float64
	lastAttitudeTime time.Time
}

var serialConfig *serial.Config
var serialPort *serial.Port

func initGPSSerial() bool {
	serialConfig = &serial.Config{Name: "/dev/ttyACM0", Baud: 9600}
	p, err := serial.OpenPort(serialConfig)
	if err != nil {
		log.Printf("serial port err: %s\n", err.Error())
		return false
	}
	serialPort = p
	return true
}

func processNMEALine(l string) bool {
	x := strings.Split(l, ",")
	if x[0] == "$GNVTG" { // Ground track information.
		mySituation.mu_GPS.Lock()
		defer mySituation.mu_GPS.Unlock()
		if len(x) < 10 {
			return false
		}
		trueCourse := uint16(0)
		if len(x[1]) > 0 {
			tc, err := strconv.ParseFloat(x[1], 32)
			if err != nil {
				return false
			}
			trueCourse = uint16(tc)
		} else {
			// No movement.
			mySituation.trueCourse = 0
			mySituation.groundSpeed = 0
			mySituation.lastGroundTrackTime = time.Time{}
			return true
		}
		groundSpeed, err := strconv.ParseFloat(x[5], 32) // Knots.
		if err != nil {
			return false
		}

		mySituation.trueCourse = uint16(trueCourse)
		mySituation.groundSpeed = uint16(groundSpeed)
		mySituation.lastGroundTrackTime = time.Now()

	} else if x[0] == "$GNGGA" { // GPS fix.
		if len(x) < 15 {
			return false
		}
		mySituation.mu_GPS.Lock()
		defer mySituation.mu_GPS.Unlock()
		// Timestamp.
		if len(x[1]) < 9 {
			return false
		}
		hr, err1 := strconv.Atoi(x[1][0:2])
		min, err2 := strconv.Atoi(x[1][2:4])
		sec, err3 := strconv.Atoi(x[1][4:6])
		if err1 != nil || err2 != nil || err3 != nil {
			return false
		}

		mySituation.lastFixSinceMidnightUTC = uint32((hr * 60 * 60) + (min * 60) + sec)

		// Latitude.
		if len(x[2]) < 10 {
			return false
		}
		hr, err1 = strconv.Atoi(x[2][0:2])
		minf, err2 := strconv.ParseFloat(x[2][2:10], 32)
		if err1 != nil || err2 != nil {
			return false
		}

		mySituation.lat = float32(hr) + float32(minf/60.0)
		if x[3] == "S" { // South = negative.
			mySituation.lat = -mySituation.lat
		}

		// Longitude.
		if len(x[4]) < 11 {
			return false
		}
		hr, err1 = strconv.Atoi(x[4][0:3])
		minf, err2 = strconv.ParseFloat(x[4][3:11], 32)
		if err1 != nil || err2 != nil {
			return false
		}

		mySituation.lng = float32(hr) + float32(minf/60.0)
		if x[5] == "W" { // West = negative.
			mySituation.lng = -mySituation.lng
		}

		// Quality indicator.
		q, err1 := strconv.Atoi(x[6])
		if err1 != nil {
			return false
		}
		mySituation.quality = uint8(q)

		// Satellites.
		sat, err1 := strconv.Atoi(x[7])
		if err1 != nil {
			return false
		}
		mySituation.satellites = uint16(sat)

		// Accuracy.
		hdop, err1 := strconv.ParseFloat(x[8], 32)
		if err1 != nil {
			return false
		}
		mySituation.accuracy = float32(hdop * 5.0) //FIXME: 5 meters ~ 1.0 HDOP?

		// Altitude.
		alt, err1 := strconv.ParseFloat(x[9], 32)
		if err1 != nil {
			return false
		}
		mySituation.alt = float32(alt * 3.28084) // Covnert to feet.

		//TODO: Altitude accuracy.
		mySituation.alt_accuracy = 0

		// Timestamp.
		mySituation.lastFixLocalTime = time.Now()

	}
	return true
}

func gpsSerialReader() {
	defer serialPort.Close()
	for globalSettings.GPS_Enabled && globalStatus.GPS_connected {
		buf := make([]byte, 1024)
		n, err := serialPort.Read(buf)
		if err != nil {
			log.Printf("gps serial read error: %s\n", err.Error())
			globalStatus.GPS_connected = false
			break
		}
		s := string(buf[:n])
		x := strings.Split(s, "\n")
		for _, l := range x {
			processNMEALine(l)
		}
	}
	globalStatus.GPS_connected = false
}

var i2cbus embd.I2CBus
var myBMP180 *bmp180.BMP180
var myMPU6050 *mpu6050.MPU6050

func readBMP180() (float64, float64, error) { // ºCelsius, Meters
	temp, err := myBMP180.Temperature()
	if err != nil {
		return temp, 0.0, err
	}
	altitude, err := myBMP180.Altitude()
	if err != nil {
		return temp, altitude, err
	}
	return temp, altitude, nil
}

func readMPU6050() (float64, float64, error) { //TODO: error checking.
	pitch, roll := myMPU6050.PitchAndRoll()
	return pitch, roll, nil
}

func initBMP180() error {
	myBMP180 = bmp180.New(i2cbus) //TODO: error checking.
	return nil
}

func initMPU6050() error {
	myMPU6050 = mpu6050.New(i2cbus) //TODO: error checking.
	return nil
}

func initI2C() error {
	i2cbus = embd.NewI2CBus(1) //TODO: error checking.
	return nil
}

// Unused at the moment. 5 second update, since read functions in bmp180 are slow.
func tempAndPressureReader() {
	timer := time.NewTicker(5 * time.Second)
	for globalStatus.RY835AI_connected && globalSettings.AHRS_Enabled {
		<-timer.C
		// Read temperature and pressure altitude.
		temp, alt, err_bmp180 := readBMP180()
		// Process.
		if err_bmp180 != nil {
			log.Printf("readBMP180(): %s\n", err_bmp180.Error())
			globalStatus.RY835AI_connected = false
		} else {
			mySituation.temp = temp
			mySituation.pressure_alt = alt
			mySituation.lastTempPressTime = time.Now()
		}
	}
	globalStatus.RY835AI_connected = false
}

func attitudeReaderSender() {
	timer := time.NewTicker(100 * time.Millisecond) // ~10Hz update.
	for globalStatus.RY835AI_connected && globalSettings.AHRS_Enabled {
		<-timer.C
		// Read pitch and roll.
		pitch, roll, err_mpu6050 := readMPU6050()

		mySituation.mu_Attitude.Lock()

		if err_mpu6050 != nil {
			log.Printf("readMPU6050(): %s\n", err_mpu6050.Error())
			globalStatus.RY835AI_connected = false
			break
		} else {
			mySituation.pitch = pitch
			mySituation.roll = roll
			mySituation.lastAttitudeTime = time.Now()
		}

		// Send, if valid.
		//		if isGPSGroundTrackValid()
		s := fmt.Sprintf("XATTStratux,%d,%f,%f", mySituation.trueCourse, mySituation.pitch, mySituation.roll)

		sendMsg([]byte(s), NETWORK_AHRS)

		mySituation.mu_Attitude.Unlock()
	}
	globalStatus.RY835AI_connected = false
}

func isGPSValid() bool {
	return time.Since(mySituation.lastFixLocalTime).Seconds() < 15
}

func isGPSGroundTrackValid() bool {
	return time.Since(mySituation.lastGroundTrackTime).Seconds() < 15
}

func isAHRSValid() bool {
	return time.Since(mySituation.lastAttitudeTime).Seconds() < 1 // If attitude information gets to be over 1 second old, declare invalid.
}

func isTempPressValid() bool {
	return time.Since(mySituation.lastTempPressTime).Seconds < 15
}

func initAHRS() error {
	if err := initI2C(); err != nil { // I2C bus.
		return err
	}
	if err := initBMP180(); err != nil { // I2C temperature and pressure altitude.
		i2cbus.Close()
		return err
	}
	if err := initMPU6050(); err != nil { // I2C accel/gyro.
		i2cbus.Close()
		myBMP180.Close()
		return err
	}
	globalStatus.RY835AI_connected = true
	go attitudeReaderSender()
	go tempAndPressureReader()

	return nil
}

func pollRY835AI() {
	timer := time.NewTicker(10 * time.Second)
	for {
		<-timer.C
		// GPS enabled, was not connected previously?
		if globalSettings.GPS_Enabled && !globalStatus.GPS_connected {
			globalStatus.GPS_connected = initGPSSerial() // via USB for now.
			if globalStatus.GPS_connected {
				go gpsSerialReader()
			}
		}
		// RY835AI I2C enabled, was not connected previously?
		if globalSettings.AHRS_Enabled && !globalStatus.RY835AI_connected {
			err := initAHRS()
			if err != nil {
				log.Printf("initAHRS(): %s\ndisabling AHRS sensors.\n", err.Error())
				globalStatus.RY835AI_connected = false
			}
		}
	}
}

func initRY835AI() {
	mySituation.mu_GPS = &sync.Mutex{}
	mySituation.mu_Attitude = &sync.Mutex{}

	go pollRY835AI()
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"github.com/streadway/amqp"
)

type RabbitMQConnector struct {
	conn    *amqp.Connection
	channel *amqp.Channel
	tag     string
	done    chan error
}

func BuildRabbitMQConnector(host string, port int, user string, password string) (*RabbitMQConnector, error) {
	var err error
	mq := &RabbitMQConnector{}
	Info("connecting to RabbitMQ")

	url := fmt.Sprintf("amqp://%s:%s@%s:%v", user, password, host, port)

	if mq.conn, err = amqp.Dial(url); err != nil {
		return nil, err
	}
	if mq.channel, err = mq.conn.Channel(); err != nil {
		return nil, err
	}
	if err = mq.channel.ExchangeDeclare("logs", "fanout", true, false, false, false, nil); err != nil {
		return nil, err
	}
	if err = mq.channel.Qos(1, 0, false); err != nil {
		return nil, err
	}
	return mq, nil
}

func (m *RabbitMQConnector) Handle(tag string, f func([]byte) bool) error {

	q, err := m.channel.QueueDeclare(tag, true, false, false, false, nil)
	if err != nil {
		return err
	}

	if err = m.channel.QueueBind(q.Name, "", "logs", false, nil); err != nil {
		return err
	}

	msgs, err := m.channel.Consume(tag, tag, false, false, false, false, nil)
	if err != nil {
		return err
	}
	go func() {
		Info("closing: %s", <-m.conn.NotifyClose(make(chan *amqp.Error)))
		//TODO: try to recconect
	}()
	m.tag = tag

	go handle(msgs, f, m.done)
	return nil
}

func handle(deliveries <-chan amqp.Delivery, f func([]byte) bool, done chan error) {
	for d := range deliveries {
		requeue := !f(d.Body)
		if requeue {
			Log("error processing an element: requeueing")
		}
		d.Ack(requeue)
	}
	Info("handle: deliveries channel closed")
	done <- nil
}

func (c *RabbitMQConnector) Close() error {
	// will close() the deliveries channel
	if err := c.channel.Cancel(c.tag, true); err != nil {
		return fmt.Errorf("Consumer cancel failed: %s", err)
	}
	if err := c.conn.Close(); err != nil {
		return fmt.Errorf("AMQP connection close error: %s", err)
	}
	defer Info("AMQP shutdown OK")
	// wait for handle() to exit
	return <-c.done
}
<commit_msg>add metrycs add rabbitmq reconnection handler<commit_after>package common

import (
	"expvar"
	"fmt"
	"github.com/paulbellamy/ratecounter"
	"github.com/streadway/amqp"
	"time"
)

var (
	counts        = expvar.NewMap("rabbitmq_counters")
	hitspersecond = expvar.NewInt("hits_per_second")
	counter       = ratecounter.NewRateCounter(1 * time.Second)
)

type RabbitMQConnector struct {
	conn     *amqp.Connection
	channel  *amqp.Channel
	tag      string
	url      string
	done     chan error
	internal chan bool
	handler  func([]byte) bool
}

func BuildRabbitMQConnector(host string, port int, user string, password string) (*RabbitMQConnector, error) {
	var err error
	mq := &RabbitMQConnector{}

	mq.url = fmt.Sprintf("amqp://%s:%s@%s:%v", user, password, host, port)

	if err = mq.open(); err != nil {
		return nil, err
	}
	return mq, err
}

func (m *RabbitMQConnector) open() error {
	Log("connecting to RabbitMQ")

	var err error
	if m.conn, err = amqp.Dial(m.url); err != nil {
		return err
	}
	if m.channel, err = m.conn.Channel(); err != nil {
		return err
	}
	if err = m.channel.ExchangeDeclare("logs", "fanout", true, false, false, false, nil); err != nil {
		return err
	}
	if err = m.channel.Qos(1, 0, false); err != nil {
		return err
	}

	go func() {
		Info("rabbitmq connection closed: ", <-m.conn.NotifyClose(make(chan *amqp.Error)))
		ticker := time.NewTicker(time.Second * 5)
		for t := range ticker.C {
			Log("trying to recover connection to rabbitmq")
			if e := m.open(); e != nil {
				Log("cannot connect to RabbitMQ at:", m.url)
				continue
			}
			if e := m.Handle(m.tag, m.handler); e != nil {
				continue
			}
			ticker.Stop()
		}
	}()

	return nil
}

func (m *RabbitMQConnector) Handle(tag string, f func([]byte) bool) error {

	q, err := m.channel.QueueDeclare(tag, true, false, false, false, nil)
	if err != nil {
		return err
	}

	if err = m.channel.QueueBind(q.Name, "", "logs", false, nil); err != nil {
		return err
	}

	msgs, err := m.channel.Consume(tag, tag, false, false, false, false, nil)
	if err != nil {
		return err
	}

	m.tag = tag
	m.handler = f

	go handle(msgs, f, m.done)
	return nil
}

func handle(deliveries <-chan amqp.Delivery, f func([]byte) bool, done chan error) {
	for d := range deliveries {
		counter.Incr(1)
		hitspersecond.Set(counter.Rate())
		requeue := !f(d.Body)
		if requeue {
			Log("error processing an element: requeueing")
			counts.Add("worker_errors", 1)
		}
		d.Ack(requeue)
	}
	Info("handle: deliveries channel closed")
	done <- nil
}

func (c *RabbitMQConnector) Close() error {
	// will close() the deliveries channel
	if err := c.channel.Cancel(c.tag, true); err != nil {
		return fmt.Errorf("Consumer cancel failed: %s", err)
	}
	if err := c.conn.Close(); err != nil {
		return fmt.Errorf("AMQP connection close error: %s", err)
	}
	defer Info("AMQP shutdown OK")
	// wait for handle() to exit
	return <-c.done
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	//"fmt"
	"reflect"
	"regexp"
	"testing"
)

type metricTest struct {
	input    string
	expected *Metric
}

var metricTests = []metricTest{
	{"mycounter:1|c", &Metric{Bucket: "mycounter", Value: int64(1), Type: "c"}},
	{"mycounter:1|c\n", &Metric{Bucket: "mycounter", Value: int64(1), Type: "c"}},
	{"  mycounter:1|c ", &Metric{Bucket: "mycounter", Value: int64(1), Type: "c"}},

	{"mygauge:78|g", &Metric{Bucket: "mygauge", Value: uint64(78), Type: "g"}},
	{"mytimer:123|ms", &Metric{Bucket: "mytimer", Value: uint64(123), Type: "ms"}},
}

// TestParseMetric tests all of the parsing
func TestParseMetric(t *testing.T) {

	for _, tt := range metricTests {
		want := tt.expected
		got, err := parseMetric([]byte(tt.input))

		if err != nil {
			t.Fatal(err)
		}

		if got.Bucket != want.Bucket {
			t.Errorf("parseMetric(%q): got: %s, want %s",
				tt.input, got.Bucket, want.Bucket)
		}

		if got.Value != want.Value {
			t.Errorf("parseMetric(%q): got: %d (%s), want %d (%s)",
				tt.input, got.Value, reflect.TypeOf(got.Value), want.Value,
				reflect.TypeOf(want.Value))
		}

		if got.Type != want.Type {
			t.Errorf("parseMetric(%q): got: %q, want %q",
				tt.input, got.Type, want.Type)
		}
	}
}

func TestHandleMessage(t *testing.T) {
	done := make(chan bool)
	testTable := make(chan metricTest, len(metricTests))

	go func() {
		for {
			select {
			case got := <-In:
				tt := <-testTable
				want := tt.expected

				if got.Bucket != want.Bucket {
					t.Errorf("handleMessage(%q): got: %q, want %q",
						tt.input, got.Bucket, want.Bucket)
				}

				if got.Value != want.Value {
					t.Errorf("handleMessage(%q): got: %d (%s), want %d (%s)",
						tt.input, got.Value, reflect.TypeOf(got.Value), want.Value,
						reflect.TypeOf(want.Value))
				}

				if got.Type != want.Type {
					t.Errorf("handleMessage(%q): got: %q, want %q",
						tt.input, got.Type, want.Type)
				}
			case <-done:
				break
			}
		}
	}()

	for _, tt := range metricTests {
		testTable <- tt
		handleMessage([]byte(tt.input))
	}

	done <- true
}

//-----------------------------------------------------------------------------
// Benchmarks

func getBuf(size int) []byte {
	var buf bytes.Buffer

	for {
		if buf.Len() >= size {
			break
		}

		buf.Write([]byte("mycounter:1|c "))
	}

	return buf.Bytes()
}

func benchmarkHandleMessage(size int, b *testing.B) {
	done := make(chan bool)

	go func() {
		for {
			select {
			case <-In:
			case <-done:
				break
			}
		}
	}()

	buf := getBuf(size)

	for n := 0; n < b.N; n++ {
		handleMessage(buf)
	}

	done <- true
}

func BenchmarkHandleMessage512(b *testing.B)  { benchmarkHandleMessage(512, b) }
func BenchmarkHandleMessage1024(b *testing.B) { benchmarkHandleMessage(1024, b) }
func BenchmarkHandleMessage2048(b *testing.B) { benchmarkHandleMessage(2048, b) }
func BenchmarkHandleMessage4096(b *testing.B) { benchmarkHandleMessage(4096, b) }
func BenchmarkHandleMessage8192(b *testing.B) { benchmarkHandleMessage(8192, b) }

// Benchmark metric parsing using different types
func benchmarkParseMetric(s string, b *testing.B) {
	for n := 0; n < b.N; n++ {
		parseMetric([]byte(s))
	}
}

func BenchmarkParseMetricCounter(b *testing.B) { benchmarkParseMetric("mycounter:1|c", b) }
func BenchmarkParseMetricGauge(b *testing.B)   { benchmarkParseMetric("mygauge:78|g", b) }
func BenchmarkParseMetricTimer(b *testing.B)   { benchmarkParseMetric("mytimer:123|ms", b) }

// Benchmark metric extraction using regular expressions
var statsPattern = regexp.MustCompile(`[\w\.]+:-?\d+\|(?:c|ms|g)(?:\|\@[\d\.]+)?`)

func benchmarkRegexFindAll(size int, b *testing.B) {
	buf := getBuf(size)

	b.ResetTimer()
	b.StartTimer()

	for n := 0; n < b.N; n++ {
		statsPattern.FindAll(buf, -1)
	}

	b.StopTimer()
}

func BenchmarkRegexFindAll512(b *testing.B)  { benchmarkRegexFindAll(512, b) }
func BenchmarkRegexFindAll1024(b *testing.B) { benchmarkRegexFindAll(1024, b) }
func BenchmarkRegexFindAll2048(b *testing.B) { benchmarkRegexFindAll(2048, b) }
func BenchmarkRegexFindAll4096(b *testing.B) { benchmarkRegexFindAll(4096, b) }
func BenchmarkRegexFindAll8192(b *testing.B) { benchmarkRegexFindAll(8192, b) }

// Benchmark metric extraction using bytes.Split()
func benchmarkBytesSplit(size int, b *testing.B) {
	buf := getBuf(size)

	b.ResetTimer()
	b.StartTimer()

	for n := 0; n < b.N; n++ {
		bytes.Split(buf, []byte(" "))
	}

	b.StopTimer()
}

func BenchmarkBytesSplit512(b *testing.B)  { benchmarkBytesSplit(512, b) }
func BenchmarkBytesSplit1024(b *testing.B) { benchmarkBytesSplit(1024, b) }
func BenchmarkBytesSplit2048(b *testing.B) { benchmarkBytesSplit(2048, b) }
func BenchmarkBytesSplit4096(b *testing.B) { benchmarkBytesSplit(4096, b) }
func BenchmarkBytesSplit8192(b *testing.B) { benchmarkBytesSplit(8192, b) }
<commit_msg>Fix BytesSplit test and add more size tests for HandleMessage<commit_after>package main

import (
	"bytes"
	//"fmt"
	"reflect"
	"regexp"
	"testing"
)

type metricTest struct {
	input    string
	expected *Metric
}

var metricTests = []metricTest{
	{"mycounter:1|c", &Metric{Bucket: "mycounter", Value: int64(1), Type: "c"}},
	{"mycounter:1|c\n", &Metric{Bucket: "mycounter", Value: int64(1), Type: "c"}},
	{"  mycounter:1|c ", &Metric{Bucket: "mycounter", Value: int64(1), Type: "c"}},

	{"mygauge:78|g", &Metric{Bucket: "mygauge", Value: uint64(78), Type: "g"}},
	{"mytimer:123|ms", &Metric{Bucket: "mytimer", Value: uint64(123), Type: "ms"}},
}

// TestParseMetric tests all of the parsing
func TestParseMetric(t *testing.T) {

	for _, tt := range metricTests {
		want := tt.expected
		got, err := parseMetric([]byte(tt.input))

		if err != nil {
			t.Fatal(err)
		}

		if got.Bucket != want.Bucket {
			t.Errorf("parseMetric(%q): got: %s, want %s",
				tt.input, got.Bucket, want.Bucket)
		}

		if got.Value != want.Value {
			t.Errorf("parseMetric(%q): got: %d (%s), want %d (%s)",
				tt.input, got.Value, reflect.TypeOf(got.Value), want.Value,
				reflect.TypeOf(want.Value))
		}

		if got.Type != want.Type {
			t.Errorf("parseMetric(%q): got: %q, want %q",
				tt.input, got.Type, want.Type)
		}
	}
}

func TestHandleMessage(t *testing.T) {
	done := make(chan bool)
	testTable := make(chan metricTest, len(metricTests))

	go func() {
		for {
			select {
			case got := <-In:
				tt := <-testTable
				want := tt.expected

				if got.Bucket != want.Bucket {
					t.Errorf("handleMessage(%q): got: %q, want %q",
						tt.input, got.Bucket, want.Bucket)
				}

				if got.Value != want.Value {
					t.Errorf("handleMessage(%q): got: %d (%s), want %d (%s)",
						tt.input, got.Value, reflect.TypeOf(got.Value), want.Value,
						reflect.TypeOf(want.Value))
				}

				if got.Type != want.Type {
					t.Errorf("handleMessage(%q): got: %q, want %q",
						tt.input, got.Type, want.Type)
				}
			case <-done:
				break
			}
		}
	}()

	for _, tt := range metricTests {
		testTable <- tt
		handleMessage([]byte(tt.input))
	}

	done <- true
}

//-----------------------------------------------------------------------------
// Benchmarks

func getBuf(size int) []byte {
	var buf bytes.Buffer

	for {
		if buf.Len() >= size {
			break
		}

		buf.Write([]byte("a:1|c\n"))
	}

	return buf.Bytes()
}

func benchmarkHandleMessage(size int, b *testing.B) {
	done := make(chan bool)
	//num := 0

	go func() {
		for {
			select {
			case <-In:
				//num++
			case <-done:
				break
			}
		}
	}()

	buf := getBuf(size)

	b.ResetTimer()
	b.StartTimer()

	for n := 0; n < b.N; n++ {
		handleMessage(buf)
	}

	b.StopTimer()

	done <- true
	//b.Logf("Handled %d metrics", num)
}

func BenchmarkHandleMessage64(b *testing.B)   { benchmarkHandleMessage(64, b) }
func BenchmarkHandleMessage128(b *testing.B)  { benchmarkHandleMessage(128, b) }
func BenchmarkHandleMessage256(b *testing.B)  { benchmarkHandleMessage(256, b) }
func BenchmarkHandleMessage512(b *testing.B)  { benchmarkHandleMessage(512, b) }
func BenchmarkHandleMessage1024(b *testing.B) { benchmarkHandleMessage(1024, b) }
func BenchmarkHandleMessage2048(b *testing.B) { benchmarkHandleMessage(2048, b) }
func BenchmarkHandleMessage4096(b *testing.B) { benchmarkHandleMessage(4096, b) }
func BenchmarkHandleMessage8192(b *testing.B) { benchmarkHandleMessage(8192, b) }

// Benchmark metric parsing using different types
func benchmarkParseMetric(s string, b *testing.B) {
	for n := 0; n < b.N; n++ {
		parseMetric([]byte(s))
	}
}

func BenchmarkParseMetricCounter(b *testing.B) { benchmarkParseMetric("mycounter:1|c", b) }
func BenchmarkParseMetricGauge(b *testing.B)   { benchmarkParseMetric("mygauge:78|g", b) }
func BenchmarkParseMetricTimer(b *testing.B)   { benchmarkParseMetric("mytimer:123|ms", b) }

// Benchmark metric extraction using regular expressions
var statsPattern = regexp.MustCompile(`[\w\.]+:-?\d+\|(?:c|ms|g)(?:\|\@[\d\.]+)?`)

func benchmarkRegexFindAll(size int, b *testing.B) {
	buf := getBuf(size)

	b.ResetTimer()
	b.StartTimer()

	for n := 0; n < b.N; n++ {
		statsPattern.FindAll(buf, -1)
	}

	b.StopTimer()
}

func BenchmarkRegexFindAll512(b *testing.B)  { benchmarkRegexFindAll(512, b) }
func BenchmarkRegexFindAll1024(b *testing.B) { benchmarkRegexFindAll(1024, b) }
func BenchmarkRegexFindAll2048(b *testing.B) { benchmarkRegexFindAll(2048, b) }
func BenchmarkRegexFindAll4096(b *testing.B) { benchmarkRegexFindAll(4096, b) }
func BenchmarkRegexFindAll8192(b *testing.B) { benchmarkRegexFindAll(8192, b) }

// Benchmark metric extraction using bytes.Split()
func benchmarkBytesSplit(size int, b *testing.B) {
	buf := getBuf(size)

	b.ResetTimer()
	b.StartTimer()

	for n := 0; n < b.N; n++ {
		bytes.Split(buf, []byte("\n"))
	}

	b.StopTimer()
}

func BenchmarkBytesSplit512(b *testing.B)  { benchmarkBytesSplit(512, b) }
func BenchmarkBytesSplit1024(b *testing.B) { benchmarkBytesSplit(1024, b) }
func BenchmarkBytesSplit2048(b *testing.B) { benchmarkBytesSplit(2048, b) }
func BenchmarkBytesSplit4096(b *testing.B) { benchmarkBytesSplit(4096, b) }
func BenchmarkBytesSplit8192(b *testing.B) { benchmarkBytesSplit(8192, b) }
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"math"
	"net"
	"os"
	"os/signal"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"
)

const VERSION = "0.5"

var signalchan chan os.Signal

type Packet struct {
	Bucket   string
	Value    interface{}
	Modifier string
	Sampling float32
}

type Uint64Slice []uint64

func (s Uint64Slice) Len() int           { return len(s) }
func (s Uint64Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s Uint64Slice) Less(i, j int) bool { return s[i] < s[j] }

type Percentiles []*Percentile
type Percentile struct {
	float float64
	str   string
}

func (a *Percentiles) Set(s string) error {
	f, err := strconv.ParseFloat(s, 64)
	if err != nil {
		return err
	}
	*a = append(*a, &Percentile{f, strings.Replace(s, ".", "_", -1)})
	return nil
}
func (p *Percentile) String() string {
	return p.str
}
func (a *Percentiles) String() string {
	return fmt.Sprintf("%v", *a)
}

var (
	serviceAddress   = flag.String("address", ":8125", "UDP service address")
	graphiteAddress  = flag.String("graphite", "127.0.0.1:2003", "Graphite service address (or - to disable)")
	flushInterval    = flag.Int64("flush-interval", 10, "Flush interval (seconds)")
	debug            = flag.Bool("debug", false, "print statistics sent to graphite")
	showVersion      = flag.Bool("version", false, "print version string")
	persistCountKeys = flag.Int64("persist-count-keys", 60, "number of flush-interval's to persist count keys")
	percentThreshold = Percentiles{}
)

func init() {
	flag.Var(&percentThreshold, "percent-threshold", "Threshold percent (may be given multiple times)")
}

var (
	In       = make(chan *Packet, 1000)
	counters = make(map[string]int64)
	gauges   = make(map[string]uint64)
	timers   = make(map[string]Uint64Slice)
)

func monitor() {
	ticker := time.NewTicker(time.Duration(*flushInterval) * time.Second)
	for {
		select {
		case sig := <-signalchan:
			fmt.Printf("!! Caught signal %d... shutting down\n", sig)
			submit()
			return
		case <-ticker.C:
			submit()
		case s := <-In:
			if s.Modifier == "ms" {
				_, ok := timers[s.Bucket]
				if !ok {
					var t Uint64Slice
					timers[s.Bucket] = t
				}
				timers[s.Bucket] = append(timers[s.Bucket], s.Value.(uint64))
			} else if s.Modifier == "g" {
				gauges[s.Bucket] = uint64(s.Value.(int64))
			} else {
				v, ok := counters[s.Bucket]
				if !ok || v < 0 {
					counters[s.Bucket] = 0
				}
				counters[s.Bucket] += int64(float64(s.Value.(int64)) * float64(1/s.Sampling))
			}
		}
	}
}

func submit() {
	client, err := net.Dial("tcp", *graphiteAddress)
	if err != nil {
		log.Printf("Error dialing %s %s", *graphiteAddress, err.Error())
		if *debug == false {
			return
		} else {
			log.Printf("WARNING: in debug mode. resetting counters even though connection to graphite failed")
		}
	} else {
		defer client.Close()
	}

	numStats := 0
	now := time.Now().Unix()
	buffer := bytes.NewBuffer([]byte{})

	// continue sending zeros for counters for a short period of time
	// even if we have no new data. for more context see https://github.com/bitly/statsdaemon/pull/8
	for s, c := range counters {
		switch {
		case c <= *persistCountKeys:
			continue
		case c < 0:
			counters[s] -= 1
			fmt.Fprintf(buffer, "%s %d %d\n", s, 0, now)
		case c >= 0:
			counters[s] = -1
			fmt.Fprintf(buffer, "%s %d %d\n", s, c, now)
		}
		numStats++
	}

	for g, c := range gauges {
		if c == math.MaxUint64 {
			continue
		}
		fmt.Fprintf(buffer, "%s %d %d\n", g, c, now)
		gauges[g] = math.MaxUint64
		numStats++
	}

	for u, t := range timers {
		if len(t) > 0 {
			numStats++
			sort.Sort(t)
			min := t[0]
			max := t[len(t)-1]
			mean := t[len(t)/2]
			maxAtThreshold := max
			count := len(t)

			for _, pct := range percentThreshold {

				if len(t) > 1 {
					indexOfPerc := int(math.Ceil(((pct.float / 100.0) * float64(count)) + 0.5))
					if indexOfPerc >= count {
						indexOfPerc = count - 1
					}
					maxAtThreshold = t[indexOfPerc]
				}

				fmt.Fprintf(buffer, "%s.upper_%s %d %d\n", u, pct.str, maxAtThreshold, now)
			}

			var z Uint64Slice
			timers[u] = z

			fmt.Fprintf(buffer, "%s.mean %d %d\n", u, mean, now)
			fmt.Fprintf(buffer, "%s.upper %d %d\n", u, max, now)
			fmt.Fprintf(buffer, "%s.lower %d %d\n", u, min, now)
			fmt.Fprintf(buffer, "%s.count %d %d\n", u, count, now)
		}
	}
	if numStats == 0 {
		return
	}
	data := buffer.Bytes()
	if client != nil {
		log.Printf("sent %d stats to %s", numStats, *graphiteAddress)
		client.Write(data)
	}
	if *debug {
		lines := bytes.NewBuffer(data)
		for {
			line, err := lines.ReadString([]byte("\n")[0])
			if line == "" || err != nil {
				break
			}
			log.Printf("debug: %s", line)
		}
	}
}

func parseMessage(buf *bytes.Buffer) []*Packet {
	var packetRegexp = regexp.MustCompile("^([^:]+):([0-9]+)\\|(g|c|ms)(\\|@([0-9\\.]+))?\n?$")

	var output []*Packet
	var err error
	var line string
	for {
		if err != nil {
			break
		}
		line, err = buf.ReadString('\n')
		if line != "" {
			item := packetRegexp.FindStringSubmatch(line)
			if len(item) == 0 {
				continue
			}

			var value interface{}
			modifier := item[3]
			switch modifier {
			case "c":
				value, err = strconv.ParseInt(item[2], 10, 64)
				if err != nil {
					log.Printf("ERROR: failed to ParseInt %s - %s", item[2], err.Error())
				}
			default:
				value, err = strconv.ParseUint(item[2], 10, 64)
				if err != nil {
					log.Printf("ERROR: failed to ParseUint %s - %s", item[2], err.Error())
				}
			}
			if err != nil {
				if modifier == "ms" {
					value = 0
				} else {
					value = 1
				}
			}

			sampleRate, err := strconv.ParseFloat(item[5], 32)
			if err != nil {
				sampleRate = 1
			}

			packet := &Packet{
				Bucket:   item[1],
				Value:    value,
				Modifier: modifier,
				Sampling: float32(sampleRate),
			}
			output = append(output, packet)
		}
	}
	return output
}

func udpListener() {
	address, _ := net.ResolveUDPAddr("udp", *serviceAddress)
	log.Printf("Listening on %s", address)
	listener, err := net.ListenUDP("udp", address)
	if err != nil {
		log.Fatalf("ListenAndServe: %s", err.Error())
	}
	defer listener.Close()
	message := make([]byte, 512)
	for {
		n, remaddr, err := listener.ReadFrom(message)
		if err != nil {
			log.Printf("error reading from %v %s", remaddr, err.Error())
			continue
		}
		buf := bytes.NewBuffer(message[0:n])
		packets := parseMessage(buf)
		for _, p := range packets {
			In <- p
		}
	}
}

func main() {
	flag.Parse()
	if *showVersion {
		fmt.Printf("statsdaemon v%s\n", VERSION)
		return
	}
	signalchan = make(chan os.Signal, 1)
	signal.Notify(signalchan, syscall.SIGTERM)
	*persistCountKeys = -1 * (*persistCountKeys)

	go udpListener()
	monitor()
}
<commit_msg>fix unit64 type conversion. closes #2<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"math"
	"net"
	"os"
	"os/signal"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"
)

const VERSION = "0.5"

var signalchan chan os.Signal

type Packet struct {
	Bucket   string
	Value    interface{}
	Modifier string
	Sampling float32
}

type Uint64Slice []uint64

func (s Uint64Slice) Len() int           { return len(s) }
func (s Uint64Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s Uint64Slice) Less(i, j int) bool { return s[i] < s[j] }

type Percentiles []*Percentile
type Percentile struct {
	float float64
	str   string
}

func (a *Percentiles) Set(s string) error {
	f, err := strconv.ParseFloat(s, 64)
	if err != nil {
		return err
	}
	*a = append(*a, &Percentile{f, strings.Replace(s, ".", "_", -1)})
	return nil
}
func (p *Percentile) String() string {
	return p.str
}
func (a *Percentiles) String() string {
	return fmt.Sprintf("%v", *a)
}

var (
	serviceAddress   = flag.String("address", ":8125", "UDP service address")
	graphiteAddress  = flag.String("graphite", "127.0.0.1:2003", "Graphite service address (or - to disable)")
	flushInterval    = flag.Int64("flush-interval", 10, "Flush interval (seconds)")
	debug            = flag.Bool("debug", false, "print statistics sent to graphite")
	showVersion      = flag.Bool("version", false, "print version string")
	persistCountKeys = flag.Int64("persist-count-keys", 60, "number of flush-interval's to persist count keys")
	percentThreshold = Percentiles{}
)

func init() {
	flag.Var(&percentThreshold, "percent-threshold", "Threshold percent (may be given multiple times)")
}

var (
	In       = make(chan *Packet, 1000)
	counters = make(map[string]int64)
	gauges   = make(map[string]uint64)
	timers   = make(map[string]Uint64Slice)
)

func monitor() {
	ticker := time.NewTicker(time.Duration(*flushInterval) * time.Second)
	for {
		select {
		case sig := <-signalchan:
			fmt.Printf("!! Caught signal %d... shutting down\n", sig)
			submit()
			return
		case <-ticker.C:
			submit()
		case s := <-In:
			if s.Modifier == "ms" {
				_, ok := timers[s.Bucket]
				if !ok {
					var t Uint64Slice
					timers[s.Bucket] = t
				}
				timers[s.Bucket] = append(timers[s.Bucket], s.Value.(uint64))
			} else if s.Modifier == "g" {
				gauges[s.Bucket] = s.Value.(uint64)
			} else {
				v, ok := counters[s.Bucket]
				if !ok || v < 0 {
					counters[s.Bucket] = 0
				}
				counters[s.Bucket] += int64(float64(s.Value.(int64)) * float64(1/s.Sampling))
			}
		}
	}
}

func submit() {
	client, err := net.Dial("tcp", *graphiteAddress)
	if err != nil {
		log.Printf("Error dialing %s %s", *graphiteAddress, err.Error())
		if *debug == false {
			return
		} else {
			log.Printf("WARNING: in debug mode. resetting counters even though connection to graphite failed")
		}
	} else {
		defer client.Close()
	}

	numStats := 0
	now := time.Now().Unix()
	buffer := bytes.NewBuffer([]byte{})

	// continue sending zeros for counters for a short period of time
	// even if we have no new data. for more context see https://github.com/bitly/statsdaemon/pull/8
	for s, c := range counters {
		switch {
		case c <= *persistCountKeys:
			continue
		case c < 0:
			counters[s] -= 1
			fmt.Fprintf(buffer, "%s %d %d\n", s, 0, now)
		case c >= 0:
			counters[s] = -1
			fmt.Fprintf(buffer, "%s %d %d\n", s, c, now)
		}
		numStats++
	}

	for g, c := range gauges {
		if c == math.MaxUint64 {
			continue
		}
		fmt.Fprintf(buffer, "%s %d %d\n", g, c, now)
		gauges[g] = math.MaxUint64
		numStats++
	}

	for u, t := range timers {
		if len(t) > 0 {
			numStats++
			sort.Sort(t)
			min := t[0]
			max := t[len(t)-1]
			mean := t[len(t)/2]
			maxAtThreshold := max
			count := len(t)

			for _, pct := range percentThreshold {

				if len(t) > 1 {
					indexOfPerc := int(math.Ceil(((pct.float / 100.0) * float64(count)) + 0.5))
					if indexOfPerc >= count {
						indexOfPerc = count - 1
					}
					maxAtThreshold = t[indexOfPerc]
				}

				fmt.Fprintf(buffer, "%s.upper_%s %d %d\n", u, pct.str, maxAtThreshold, now)
			}

			var z Uint64Slice
			timers[u] = z

			fmt.Fprintf(buffer, "%s.mean %d %d\n", u, mean, now)
			fmt.Fprintf(buffer, "%s.upper %d %d\n", u, max, now)
			fmt.Fprintf(buffer, "%s.lower %d %d\n", u, min, now)
			fmt.Fprintf(buffer, "%s.count %d %d\n", u, count, now)
		}
	}
	if numStats == 0 {
		return
	}
	data := buffer.Bytes()
	if client != nil {
		log.Printf("sent %d stats to %s", numStats, *graphiteAddress)
		client.Write(data)
	}
	if *debug {
		lines := bytes.NewBuffer(data)
		for {
			line, err := lines.ReadString([]byte("\n")[0])
			if line == "" || err != nil {
				break
			}
			log.Printf("debug: %s", line)
		}
	}
}

func parseMessage(buf *bytes.Buffer) []*Packet {
	var packetRegexp = regexp.MustCompile("^([^:]+):([0-9]+)\\|(g|c|ms)(\\|@([0-9\\.]+))?\n?$")

	var output []*Packet
	var err error
	var line string
	for {
		if err != nil {
			break
		}
		line, err = buf.ReadString('\n')
		if line != "" {
			item := packetRegexp.FindStringSubmatch(line)
			if len(item) == 0 {
				continue
			}

			var value interface{}
			modifier := item[3]
			switch modifier {
			case "c":
				value, err = strconv.ParseInt(item[2], 10, 64)
				if err != nil {
					log.Printf("ERROR: failed to ParseInt %s - %s", item[2], err.Error())
				}
			default:
				value, err = strconv.ParseUint(item[2], 10, 64)
				if err != nil {
					log.Printf("ERROR: failed to ParseUint %s - %s", item[2], err.Error())
				}
			}
			if err != nil {
				if modifier == "ms" {
					value = 0
				} else {
					value = 1
				}
			}

			sampleRate, err := strconv.ParseFloat(item[5], 32)
			if err != nil {
				sampleRate = 1
			}

			packet := &Packet{
				Bucket:   item[1],
				Value:    value,
				Modifier: modifier,
				Sampling: float32(sampleRate),
			}
			output = append(output, packet)
		}
	}
	return output
}

func udpListener() {
	address, _ := net.ResolveUDPAddr("udp", *serviceAddress)
	log.Printf("Listening on %s", address)
	listener, err := net.ListenUDP("udp", address)
	if err != nil {
		log.Fatalf("ListenAndServe: %s", err.Error())
	}
	defer listener.Close()
	message := make([]byte, 512)
	for {
		n, remaddr, err := listener.ReadFrom(message)
		if err != nil {
			log.Printf("error reading from %v %s", remaddr, err.Error())
			continue
		}
		buf := bytes.NewBuffer(message[0:n])
		packets := parseMessage(buf)
		for _, p := range packets {
			In <- p
		}
	}
}

func main() {
	flag.Parse()
	if *showVersion {
		fmt.Printf("statsdaemon v%s\n", VERSION)
		return
	}
	signalchan = make(chan os.Signal, 1)
	signal.Notify(signalchan, syscall.SIGTERM)
	*persistCountKeys = -1 * (*persistCountKeys)

	go udpListener()
	monitor()
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the LGPLv3, see LICENCE file for details.

package utils

import (
	"bytes"
	"compress/gzip"
	"crypto/sha256"
	"encoding/hex"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"unicode"

	goyaml "gopkg.in/yaml.v1"
)

// WriteYaml marshals obj as yaml and then writes it to a file, atomically,
// by first writing a sibling with the suffix ".preparing" and then moving
// the sibling to the real path.
func WriteYaml(path string, obj interface{}) error {
	data, err := goyaml.Marshal(obj)
	if err != nil {
		return err
	}
	prep := path + ".preparing"
	f, err := os.OpenFile(prep, os.O_WRONLY|os.O_CREATE|os.O_SYNC, 0644)
	if err != nil {
		return err
	}
	_, err = f.Write(data)
	// Explicitly close the file before moving it. This is needed on Windows
	// where the OS will not allow us to move a file that still has an open file handle
	f.Close()
	if err != nil {
		return err
	}
	return ReplaceFile(prep, path)
}

// ReadYaml unmarshals the yaml contained in the file at path into obj. See
// goyaml.Unmarshal.
func ReadYaml(path string, obj interface{}) error {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	return goyaml.Unmarshal(data, obj)
}

// ShQuote quotes s so that when read by bash, no metacharacters
// within s will be interpreted as such.
func ShQuote(s string) string {
	// single-quote becomes single-quote, double-quote, single-quote, double-quote, single-quote
	return `'` + strings.Replace(s, `'`, `'"'"'`, -1) + `'`
}

// WinPSQuote quotes s so that when read by powershell, no metacharacters
// within s will be interpreted as such.
func WinPSQuote(s string) string {
	// See http://ss64.com/ps/syntax-esc.html#quotes.
	return `'` + strings.Replace(s, `'`, `''`, -1) + `'`
}

// WinCmdQuote quotes s so that when read by cmd.exe, no metacharacters
// within s will be interpreted as such.
func WinCmdQuote(s string) string {
	// See http://blogs.msdn.com/b/twistylittlepassagesallalike/archive/2011/04/23/everyone-quotes-arguments-the-wrong-way.aspx.
	quoted := winCmdQuote(s)
	return winCmdEscapeMeta(quoted)
}

func winCmdQuote(s string) string {
	var escaped string
	for _, c := range s {
		switch c {
		case '\\', '"':
			escaped += `\`
		}
		escaped += string(c)
	}
	return `"` + escaped + `"`
}

func winCmdEscapeMeta(str string) string {
	const meta = `()%!^"<>&|`
	var newStr string
	for _, c := range str {
		if strings.Contains(meta, string(c)) {
			newStr += "^"
		}
		newStr += string(c)
	}
	return newStr
}

// CommandString flattens a sequence of command arguments into a
// string suitable for executing in a shell, escaping slashes,
// variables and quotes as necessary; each argument is double-quoted
// if and only if necessary.
func CommandString(args ...string) string {
	var buf bytes.Buffer
	for i, arg := range args {
		needsQuotes := false
		var argBuf bytes.Buffer
		for _, r := range arg {
			if unicode.IsSpace(r) {
				needsQuotes = true
			} else if r == '"' || r == '$' || r == '\\' {
				needsQuotes = true
				argBuf.WriteByte('\\')
			}
			argBuf.WriteRune(r)
		}
		if i > 0 {
			buf.WriteByte(' ')
		}
		if needsQuotes {
			buf.WriteByte('"')
			argBuf.WriteTo(&buf)
			buf.WriteByte('"')
		} else {
			argBuf.WriteTo(&buf)
		}
	}
	return buf.String()
}

// Gzip compresses the given data.
func Gzip(data []byte) []byte {
	var buf bytes.Buffer
	w := gzip.NewWriter(&buf)
	if _, err := w.Write(data); err != nil {
		// Compression should never fail unless it fails
		// to write to the underlying writer, which is a bytes.Buffer
		// that never fails.
		panic(err)
	}
	if err := w.Close(); err != nil {
		panic(err)
	}
	return buf.Bytes()
}

// Gunzip uncompresses the given data.
func Gunzip(data []byte) ([]byte, error) {
	r, err := gzip.NewReader(bytes.NewReader(data))
	if err != nil {
		return nil, err
	}
	return ioutil.ReadAll(r)
}

// ReadSHA256 returns the SHA256 hash of the contents read from source
// (hex encoded) and the size of the source in bytes.
func ReadSHA256(source io.Reader) (string, int64, error) {
	hash := sha256.New()
	size, err := io.Copy(hash, source)
	if err != nil {
		return "", 0, err
	}
	digest := hex.EncodeToString(hash.Sum(nil))
	return digest, size, nil
}

// ReadFileSHA256 is like ReadSHA256 but reads the contents of the
// given file.
func ReadFileSHA256(filename string) (string, int64, error) {
	f, err := os.Open(filename)
	if err != nil {
		return "", 0, err
	}
	defer f.Close()
	return ReadSHA256(f)
}
<commit_msg>Add a TODO.<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the LGPLv3, see LICENCE file for details.

package utils

import (
	"bytes"
	"compress/gzip"
	"crypto/sha256"
	"encoding/hex"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"unicode"

	goyaml "gopkg.in/yaml.v1"
)

// WriteYaml marshals obj as yaml and then writes it to a file, atomically,
// by first writing a sibling with the suffix ".preparing" and then moving
// the sibling to the real path.
func WriteYaml(path string, obj interface{}) error {
	data, err := goyaml.Marshal(obj)
	if err != nil {
		return err
	}
	prep := path + ".preparing"
	f, err := os.OpenFile(prep, os.O_WRONLY|os.O_CREATE|os.O_SYNC, 0644)
	if err != nil {
		return err
	}
	_, err = f.Write(data)
	// Explicitly close the file before moving it. This is needed on Windows
	// where the OS will not allow us to move a file that still has an open file handle
	f.Close()
	if err != nil {
		return err
	}
	return ReplaceFile(prep, path)
}

// ReadYaml unmarshals the yaml contained in the file at path into obj. See
// goyaml.Unmarshal.
func ReadYaml(path string, obj interface{}) error {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	return goyaml.Unmarshal(data, obj)
}

// TODO(ericsnow) Move the quoting helpers into the shell package?

// ShQuote quotes s so that when read by bash, no metacharacters
// within s will be interpreted as such.
func ShQuote(s string) string {
	// single-quote becomes single-quote, double-quote, single-quote, double-quote, single-quote
	return `'` + strings.Replace(s, `'`, `'"'"'`, -1) + `'`
}

// WinPSQuote quotes s so that when read by powershell, no metacharacters
// within s will be interpreted as such.
func WinPSQuote(s string) string {
	// See http://ss64.com/ps/syntax-esc.html#quotes.
	return `'` + strings.Replace(s, `'`, `''`, -1) + `'`
}

// WinCmdQuote quotes s so that when read by cmd.exe, no metacharacters
// within s will be interpreted as such.
func WinCmdQuote(s string) string {
	// See http://blogs.msdn.com/b/twistylittlepassagesallalike/archive/2011/04/23/everyone-quotes-arguments-the-wrong-way.aspx.
	quoted := winCmdQuote(s)
	return winCmdEscapeMeta(quoted)
}

func winCmdQuote(s string) string {
	var escaped string
	for _, c := range s {
		switch c {
		case '\\', '"':
			escaped += `\`
		}
		escaped += string(c)
	}
	return `"` + escaped + `"`
}

func winCmdEscapeMeta(str string) string {
	const meta = `()%!^"<>&|`
	var newStr string
	for _, c := range str {
		if strings.Contains(meta, string(c)) {
			newStr += "^"
		}
		newStr += string(c)
	}
	return newStr
}

// CommandString flattens a sequence of command arguments into a
// string suitable for executing in a shell, escaping slashes,
// variables and quotes as necessary; each argument is double-quoted
// if and only if necessary.
func CommandString(args ...string) string {
	var buf bytes.Buffer
	for i, arg := range args {
		needsQuotes := false
		var argBuf bytes.Buffer
		for _, r := range arg {
			if unicode.IsSpace(r) {
				needsQuotes = true
			} else if r == '"' || r == '$' || r == '\\' {
				needsQuotes = true
				argBuf.WriteByte('\\')
			}
			argBuf.WriteRune(r)
		}
		if i > 0 {
			buf.WriteByte(' ')
		}
		if needsQuotes {
			buf.WriteByte('"')
			argBuf.WriteTo(&buf)
			buf.WriteByte('"')
		} else {
			argBuf.WriteTo(&buf)
		}
	}
	return buf.String()
}

// Gzip compresses the given data.
func Gzip(data []byte) []byte {
	var buf bytes.Buffer
	w := gzip.NewWriter(&buf)
	if _, err := w.Write(data); err != nil {
		// Compression should never fail unless it fails
		// to write to the underlying writer, which is a bytes.Buffer
		// that never fails.
		panic(err)
	}
	if err := w.Close(); err != nil {
		panic(err)
	}
	return buf.Bytes()
}

// Gunzip uncompresses the given data.
func Gunzip(data []byte) ([]byte, error) {
	r, err := gzip.NewReader(bytes.NewReader(data))
	if err != nil {
		return nil, err
	}
	return ioutil.ReadAll(r)
}

// ReadSHA256 returns the SHA256 hash of the contents read from source
// (hex encoded) and the size of the source in bytes.
func ReadSHA256(source io.Reader) (string, int64, error) {
	hash := sha256.New()
	size, err := io.Copy(hash, source)
	if err != nil {
		return "", 0, err
	}
	digest := hex.EncodeToString(hash.Sum(nil))
	return digest, size, nil
}

// ReadFileSHA256 is like ReadSHA256 but reads the contents of the
// given file.
func ReadFileSHA256(filename string) (string, int64, error) {
	f, err := os.Open(filename)
	if err != nil {
		return "", 0, err
	}
	defer f.Close()
	return ReadSHA256(f)
}
<|endoftext|>


<commit_before>package easygenapi

import (
	"github.com/danverbraganza/varcaser/varcaser"
)

////////////////////////////////////////////////////////////////////////////
// Constant and data type/structure definitions

// pre-configed varcaser caser converters
// the names are self-explanatory from their definitions
var (
	cls2lc = varcaser.Caser{From: varcaser.LowerSnakeCase, To: varcaser.LowerCamelCase}
	cls2uc = varcaser.Caser{From: varcaser.LowerSnakeCase, To: varcaser.UpperCamelCase}
	cls2ss = varcaser.Caser{From: varcaser.LowerSnakeCase, To: varcaser.ScreamingSnakeCase}

	ck2lc = varcaser.Caser{From: varcaser.KebabCase, To: varcaser.LowerCamelCase}
	ck2uc = varcaser.Caser{From: varcaser.KebabCase, To: varcaser.UpperCamelCase}
	ck2ss = varcaser.Caser{From: varcaser.KebabCase, To: varcaser.ScreamingSnakeCase}

	clc2ss = varcaser.Caser{From: varcaser.LowerCamelCase, To: varcaser.ScreamingSnakeCase}
	cuc2ss = varcaser.Caser{From: varcaser.UpperCamelCase, To: varcaser.ScreamingSnakeCase}
)

//==========================================================================
// template functions
<commit_msg>- [+] add ck2ls, KebabCase To lower_snake_case<commit_after>package easygenapi

import (
	"github.com/danverbraganza/varcaser/varcaser"
)

////////////////////////////////////////////////////////////////////////////
// Constant and data type/structure definitions

// pre-configed varcaser caser converters
// the names are self-explanatory from their definitions
var (
	cls2lc = varcaser.Caser{From: varcaser.LowerSnakeCase, To: varcaser.LowerCamelCase}
	cls2uc = varcaser.Caser{From: varcaser.LowerSnakeCase, To: varcaser.UpperCamelCase}
	cls2ss = varcaser.Caser{From: varcaser.LowerSnakeCase, To: varcaser.ScreamingSnakeCase}

	ck2lc = varcaser.Caser{From: varcaser.KebabCase, To: varcaser.LowerCamelCase}
	ck2uc = varcaser.Caser{From: varcaser.KebabCase, To: varcaser.UpperCamelCase}
	ck2ls = varcaser.Caser{From: varcaser.KebabCase, To: varcaser.lower_snake_case}
	ck2ss = varcaser.Caser{From: varcaser.KebabCase, To: varcaser.ScreamingSnakeCase}

	clc2ss = varcaser.Caser{From: varcaser.LowerCamelCase, To: varcaser.ScreamingSnakeCase}
	cuc2ss = varcaser.Caser{From: varcaser.UpperCamelCase, To: varcaser.ScreamingSnakeCase}
)

//==========================================================================
// template functions
<|endoftext|>


<commit_before>// Copyright (c) 2013 David G. Andersen.  All rights reserved.
// Use of this source is goverened by the Apache open source license,
// a copy of which can be found in the LICENSE file.  Please contact
// the author if you would like a copy under another license.  We're
// happy to help out.

// https://github.com/efficient/go-cuckoo

// Package cuckoo provides an implementation of a high-performance,
// memory efficient hash table that supports fast and safe concurrent
// access by multiple threads.
// The default version of the hash table uses string keys and
// interface{} values.  For faster performance and fewer annoying
// typecasting issues, copy this code and change the valuetype
// appropriately.
package cuckoo

import (
	//"math/rand"
	//"time"
	//"sync/atomic"
	"fmt"
	"hash/crc64" // xxx - use city eventually.
	"log"
)

const (
	SLOTS_PER_BUCKET    = 4  // This is kinda hardcoded all over the place
	DEFAULT_START_POWER = 16 // 2^16 keys to start with.
	N_LOCKS             = 2048
	MAX_REACH           = 500 // number of buckets to examine before full
	MAX_PATH_DEPTH      = 5   // must be ceil(log4(MAX_REACH))
)

type keytype string
type valuetype string

type kvtype struct {
	keyhash uint64
	key     *keytype
	value   *valuetype
}

type Table struct {
	storage    []kvtype
	locks      [N_LOCKS]int32
	hashpower  uint
	bucketMask uint64
	// For hashing
	crcTab *crc64.Table
}

func NewTable() *Table {
	return NewTablePowerOfTwo(DEFAULT_START_POWER)
}

func (t *Table) sizeTable(twopower uint) {
	t.hashpower = twopower - 2
	t.bucketMask = (1 << t.hashpower) - 1
}

func NewTablePowerOfTwo(twopower uint) *Table {
	t := &Table{}
	t.sizeTable(twopower)
	// storage holds items, but is organized into N/4 fully
	// associative buckets conceptually, so the hashpower differs
	// from the storage size.
	t.storage = make([]kvtype, 1<<twopower)
	t.crcTab = crc64.MakeTable(crc64.ECMA)
	return t
}

func (t *Table) getKeyhash(k keytype) uint64 {
	return ((1 << 63) | crc64.Checksum([]byte(k), t.crcTab))
}

var _ = fmt.Println
var _ = log.Fatal

func (t *Table) altIndex(bucket, keyhash uint64) uint64 {
	tag := (keyhash & 0xff) + 1
	return (bucket ^ (tag * 0x5bd1e995)) & t.bucketMask
}

func (t *Table) indexes(keyhash uint64) (i1, i2 uint64) {
	tag := (keyhash & 0xff) + 1
	i1 = (keyhash >> 8) & t.bucketMask
	i2 = (i1 ^ (tag * 0x5bd1e995)) & t.bucketMask
	return
}

func (t *Table) tryBucketRead(k keytype, keyhash uint64, bucket uint64) (valuetype, bool, int) {
	storageOffset := bucket * 4
	buckets := t.storage[storageOffset : storageOffset+4]
	for i, b := range buckets {
		if b.keyhash == keyhash {
			if *b.key == k {
				return *b.value, true, i
			}
		}
	}
	return valuetype(0), false, 0
}

func (t Table) hasSpace(bucket uint64) (bool, int) {
	storageOffset := bucket * 4
	buckets := t.storage[storageOffset : storageOffset+4]
	for i, b := range buckets {
		if b.keyhash == 0 {
			return true, i
		}
	}
	return false, 0
}

func (t Table) insert(k keytype, v valuetype, keyhash uint64, bucket uint64, slot int) {
	b := &(t.storage[bucket*4+uint64(slot)])
	b.keyhash = keyhash
	b.key = &k
	b.value = &v
}

func (t *Table) Get(k keytype) (v valuetype, found bool) {
	keyhash := t.getKeyhash(k)
	i1, i2 := t.indexes(keyhash)
	v, found, slot = t.tryBucketRead(k, keyhash, i1)
	if !found {
		v, found, slot = t.tryBucketRead(k, keyhash, i2)
	}
	if !found {
		fmt.Printf("key %s not found.  Indexes %d and %d\n", string(k), i1, i2)
	}

	return
}

type pathEnt struct {
	bucket     uint64
	depth      int
	parent     int
	parentslot int
}

func (t *Table) slotSearchBFS(i1, i2 uint64) (success bool, path [MAX_PATH_DEPTH]uint64, depth int) {
	var queue [500]pathEnt
	queue_head := 0
	queue_tail := 0

	queue[queue_tail].bucket = i1
	queue_tail++

	queue[queue_tail].bucket = i2
	queue_tail++

	for dfspos := 0; dfspos < MAX_REACH; dfspos++ {
		candidate := queue[queue_head]
		candidate_pos := queue_head
		queue_head++
		//log.Printf("BFS examining %v ", candidate)
		if hasit, where := t.hasSpace(candidate.bucket); hasit {
			// log.Printf("BFS found space at bucket %d slot %d (parent %d slot %d)   candidate: %v",
			// 	candidate.bucket, where, candidate.parent, candidate.parentslot, candidate)
			cd := candidate.depth
			path[candidate.depth] = candidate.bucket*SLOTS_PER_BUCKET + uint64(where)
			//log.Printf("path %d = %v", candidate.depth, path[candidate.depth])
			parentslot := candidate.parentslot
			for i := 0; i < cd; i++ {
				candidate = queue[candidate.parent]
				path[candidate.depth] = candidate.bucket*SLOTS_PER_BUCKET + uint64(parentslot)
				parentslot = candidate.parentslot
				//log.Printf("path %d = %v  (%v)", candidate.depth, path[candidate.depth], candidate)
			}
			return true, path, cd
		} else {
			bStart := candidate.bucket * SLOTS_PER_BUCKET
			for i := 0; queue_tail < MAX_REACH && i < SLOTS_PER_BUCKET; i++ {
				buck := bStart + uint64(i)
				kh := t.storage[buck].keyhash
				ai := t.altIndex(candidate.bucket, kh)
				//log.Printf("  enqueue %d (%d) ((%v)) - %d", candidate.bucket, buck, *t.storage[buck].key, ai)
				queue[queue_tail].bucket = ai
				queue[queue_tail].depth = candidate.depth + 1
				queue[queue_tail].parent = candidate_pos
				queue[queue_tail].parentslot = i
				queue_tail++
			}
		}
	}
	return false, path, 0

}

func (t *Table) swap(x, y uint64) {
	// Needs to be made conditional on matching the path for the
	// concurrent version...
	// xkey, ykey := "none", "none"
	// if t.storage[x].key != nil { xkey = string(*t.storage[x].key) }
	// if t.storage[y].key != nil { ykey = string(*t.storage[y].key) }
	//log.Printf("swap %d to %d  (keys %v and %v)  (now: %v and %v)\n", x, y, xkey, ykey, t.storage[x], t.storage[y])
	t.storage[x], t.storage[y] = t.storage[y], t.storage[x]
	//log.Printf("  after %v and %v", t.storage[x], t.storage[y])
}

func (t *Table) Put(k keytype, v valuetype) error {
	keyhash := t.getKeyhash(k)
	i1, i2 := t.indexes(keyhash)
	if hasSpace, where := t.hasSpace(i1); hasSpace {
		t.insert(k, v, keyhash, i1, where)
	} else if hasSpace, where := t.hasSpace(i2); hasSpace {
		t.insert(k, v, keyhash, i2, where)
	} else {
		//log.Printf("BFSing for %v indexes %d %d\n", k, i1, i2)
		found, path, depth := t.slotSearchBFS(i1, i2)
		if !found {
			panic("Crap, table really full, search failed")
		}
		for i := depth; i > 0; i-- {
			t.swap(path[i], path[i-1])
		}
		t.insert(k, v, keyhash, path[0]/4, int(path[0]%4))
		//log.Printf("Insert at %d (now %v)", path[0], t.storage[path[0]])
	}
	return nil
}

func (t *Table) Delete(k keytype) error {
	keyhash := t.getKeyhash(k)
	i1, i2 := t.indexes(keyhash)

	bucket, v, found, slot := i1, t.tryBucketRead(k, keyhash, i1)
	if !found {
		bucket, v, found, slot := t.tryBucketRead(k, keyhash, i2)
	}
	if !found {
		log.Println("Delete of not-found key, return appropriate error message")
		return nil
	}
	buck := bucket*SLOTS_PER_BUCKET+uint64(slot)
	t.storage[buck].keyhash = 0
	t.storage[key].keyhash = nil
	t.storage[value].keyhash = nil
	return nil
}
<commit_msg>Working delete - but needs more extensive testing.  Current delete test is probably too expensive.<commit_after>// Copyright (c) 2013 David G. Andersen.  All rights reserved.
// Use of this source is goverened by the Apache open source license,
// a copy of which can be found in the LICENSE file.  Please contact
// the author if you would like a copy under another license.  We're
// happy to help out.

// https://github.com/efficient/go-cuckoo

// Package cuckoo provides an implementation of a high-performance,
// memory efficient hash table that supports fast and safe concurrent
// access by multiple threads.
// The default version of the hash table uses string keys and
// interface{} values.  For faster performance and fewer annoying
// typecasting issues, copy this code and change the valuetype
// appropriately.
package cuckoo

import (
	//"math/rand"
	//"time"
	//"sync/atomic"
	"fmt"
	"hash/crc64" // xxx - use city eventually.
	"log"
)

const (
	SLOTS_PER_BUCKET    = 4  // This is kinda hardcoded all over the place
	DEFAULT_START_POWER = 16 // 2^16 keys to start with.
	N_LOCKS             = 2048
	MAX_REACH           = 500 // number of buckets to examine before full
	MAX_PATH_DEPTH      = 5   // must be ceil(log4(MAX_REACH))
)

type keytype string
type valuetype string

type kvtype struct {
	keyhash uint64
	key     *keytype
	value   *valuetype
}

type Table struct {
	storage    []kvtype
	locks      [N_LOCKS]int32
	hashpower  uint
	bucketMask uint64
	// For hashing
	crcTab *crc64.Table
}

func NewTable() *Table {
	return NewTablePowerOfTwo(DEFAULT_START_POWER)
}

func (t *Table) sizeTable(twopower uint) {
	t.hashpower = twopower - 2
	t.bucketMask = (1 << t.hashpower) - 1
}

func NewTablePowerOfTwo(twopower uint) *Table {
	t := &Table{}
	t.sizeTable(twopower)
	// storage holds items, but is organized into N/4 fully
	// associative buckets conceptually, so the hashpower differs
	// from the storage size.
	t.storage = make([]kvtype, 1<<twopower)
	t.crcTab = crc64.MakeTable(crc64.ECMA)
	return t
}

func (t *Table) getKeyhash(k keytype) uint64 {
	return ((1 << 63) | crc64.Checksum([]byte(k), t.crcTab))
}

var _ = fmt.Println
var _ = log.Fatal

func (t *Table) altIndex(bucket, keyhash uint64) uint64 {
	tag := (keyhash & 0xff) + 1
	return (bucket ^ (tag * 0x5bd1e995)) & t.bucketMask
}

func (t *Table) indexes(keyhash uint64) (i1, i2 uint64) {
	tag := (keyhash & 0xff) + 1
	i1 = (keyhash >> 8) & t.bucketMask
	i2 = (i1 ^ (tag * 0x5bd1e995)) & t.bucketMask
	return
}

func (t *Table) tryBucketRead(k keytype, keyhash uint64, bucket uint64) (valuetype, bool, int) {
	storageOffset := bucket * 4
	buckets := t.storage[storageOffset : storageOffset+4]
	for i, b := range buckets {
		if b.keyhash == keyhash {
			if *b.key == k {
				return *b.value, true, i
			}
		}
	}
	return valuetype(0), false, 0
}

func (t Table) hasSpace(bucket uint64) (bool, int) {
	storageOffset := bucket * 4
	buckets := t.storage[storageOffset : storageOffset+4]
	for i, b := range buckets {
		if b.keyhash == 0 {
			return true, i
		}
	}
	return false, 0
}

func (t Table) insert(k keytype, v valuetype, keyhash uint64, bucket uint64, slot int) {
	b := &(t.storage[bucket*4+uint64(slot)])
	b.keyhash = keyhash
	b.key = &k
	b.value = &v
}

func (t *Table) Get(k keytype) (v valuetype, found bool) {
	keyhash := t.getKeyhash(k)
	i1, i2 := t.indexes(keyhash)
	v, found, _ = t.tryBucketRead(k, keyhash, i1)
	if !found {
		v, found, _ = t.tryBucketRead(k, keyhash, i2)
	}

	return
}

type pathEnt struct {
	bucket     uint64
	depth      int
	parent     int
	parentslot int
}

func (t *Table) slotSearchBFS(i1, i2 uint64) (success bool, path [MAX_PATH_DEPTH]uint64, depth int) {
	var queue [500]pathEnt
	queue_head := 0
	queue_tail := 0

	queue[queue_tail].bucket = i1
	queue_tail++

	queue[queue_tail].bucket = i2
	queue_tail++

	for dfspos := 0; dfspos < MAX_REACH; dfspos++ {
		candidate := queue[queue_head]
		candidate_pos := queue_head
		queue_head++
		//log.Printf("BFS examining %v ", candidate)
		if hasit, where := t.hasSpace(candidate.bucket); hasit {
			// log.Printf("BFS found space at bucket %d slot %d (parent %d slot %d)   candidate: %v",
			// 	candidate.bucket, where, candidate.parent, candidate.parentslot, candidate)
			cd := candidate.depth
			path[candidate.depth] = candidate.bucket*SLOTS_PER_BUCKET + uint64(where)
			//log.Printf("path %d = %v", candidate.depth, path[candidate.depth])
			parentslot := candidate.parentslot
			for i := 0; i < cd; i++ {
				candidate = queue[candidate.parent]
				path[candidate.depth] = candidate.bucket*SLOTS_PER_BUCKET + uint64(parentslot)
				parentslot = candidate.parentslot
				//log.Printf("path %d = %v  (%v)", candidate.depth, path[candidate.depth], candidate)
			}
			return true, path, cd
		} else {
			bStart := candidate.bucket * SLOTS_PER_BUCKET
			for i := 0; queue_tail < MAX_REACH && i < SLOTS_PER_BUCKET; i++ {
				buck := bStart + uint64(i)
				kh := t.storage[buck].keyhash
				ai := t.altIndex(candidate.bucket, kh)
				//log.Printf("  enqueue %d (%d) ((%v)) - %d", candidate.bucket, buck, *t.storage[buck].key, ai)
				queue[queue_tail].bucket = ai
				queue[queue_tail].depth = candidate.depth + 1
				queue[queue_tail].parent = candidate_pos
				queue[queue_tail].parentslot = i
				queue_tail++
			}
		}
	}
	return false, path, 0

}

func (t *Table) swap(x, y uint64) {
	// Needs to be made conditional on matching the path for the
	// concurrent version...
	// xkey, ykey := "none", "none"
	// if t.storage[x].key != nil { xkey = string(*t.storage[x].key) }
	// if t.storage[y].key != nil { ykey = string(*t.storage[y].key) }
	//log.Printf("swap %d to %d  (keys %v and %v)  (now: %v and %v)\n", x, y, xkey, ykey, t.storage[x], t.storage[y])
	t.storage[x], t.storage[y] = t.storage[y], t.storage[x]
	//log.Printf("  after %v and %v", t.storage[x], t.storage[y])
}

func (t *Table) Put(k keytype, v valuetype) error {
	keyhash := t.getKeyhash(k)
	i1, i2 := t.indexes(keyhash)
	if hasSpace, where := t.hasSpace(i1); hasSpace {
		t.insert(k, v, keyhash, i1, where)
	} else if hasSpace, where := t.hasSpace(i2); hasSpace {
		t.insert(k, v, keyhash, i2, where)
	} else {
		//log.Printf("BFSing for %v indexes %d %d\n", k, i1, i2)
		found, path, depth := t.slotSearchBFS(i1, i2)
		if !found {
			panic("Crap, table really full, search failed")
		}
		for i := depth; i > 0; i-- {
			t.swap(path[i], path[i-1])
		}
		t.insert(k, v, keyhash, path[0]/4, int(path[0]%4))
		//log.Printf("Insert at %d (now %v)", path[0], t.storage[path[0]])
	}
	return nil
}

func (t *Table) Delete(k keytype) error {
	keyhash := t.getKeyhash(k)
	i1, i2 := t.indexes(keyhash)

	bucket := i1
	_, found, slot := t.tryBucketRead(k, keyhash, i1)
	if !found {
		bucket = i2
		_, found, slot = t.tryBucketRead(k, keyhash, i2)
	}
	if !found {
		log.Println("Delete of not-found key, return appropriate error message")
		return nil
	}
	buck := bucket*SLOTS_PER_BUCKET+uint64(slot)
	t.storage[buck].keyhash = 0
	t.storage[buck].key = nil
	t.storage[buck].value = nil
	return nil
}
<|endoftext|>


<commit_before>package aranGO

import (
	"encoding/json"
	"errors"
	"reflect"
	"time"
)

type Cursor struct {
	db *Database `json:"-"`
	Id string    `json:"Id"`

	Index  int           `json:"-"`
	Result []interface{} `json:"result"`
	More   bool          `json:"hasMore"`
	Amount int           `json:"count"`
	Data   Extra         `json:"extra"`

	Err    bool   `json:"error"`
	ErrMsg string `json:"errorMessage"`
	Code   int    `json:"code"`
	max    int
	Time   time.Duration `json:"time"`
}

func NewCursor(db *Database) *Cursor {
	var c Cursor
	if db == nil {
		return nil
	}
	c.db = db
	return &c
}

// Delete cursor in server and free RAM
func (c *Cursor) Delete() error {
	if c.Id == "" {
		return errors.New("Invalid cursor to delete")
	}
	res, err := c.db.send("cursor", c.Id, "DELETE", nil, c, c)
	if err != nil {
		return nil
	}

	switch res.Status() {
	case 202:
		return nil
	case 404:
		return errors.New("Cursor does not exist")
	default:
		return nil
	}

}

func (c *Cursor) FetchBatch(r interface{}) error {
	kind := reflect.ValueOf(r).Elem().Kind()
	if kind != reflect.Slice && kind != reflect.Array {
		return errors.New("Container must be Slice of array kind")
	}
	b, err := json.Marshal(c.Result)
	if err != nil {
		return err
	}
	err = json.Unmarshal(b, r)
	if err != nil {
		return err
	}

	// fetch next batch
	if c.HasMore() {
		res, err := c.db.send("cursor", c.Id, "PUT", nil, c, c)
		if res.Status() == 200 {
			return nil
		}

		if err != nil {
			return err
		}
	}

	return nil
}

// Iterates over cursor, returns false when no more values into batch, fetch next batch if necesary.
func (c *Cursor) FetchOne(r interface{}) bool {
	var max int = len(c.Result) - 1
	if c.Index >= max {
		b, err := json.Marshal(c.Result[c.Index])
		err = json.Unmarshal(b, r)
		if err != nil {
			return false
		} else {
			if c.More {
				//fetch rest from server
				res, _ := c.db.send("cursor", c.Id, "PUT", nil, c, c)

				if err != nil {
					return false
				}

				if res.Status() == 200 {
					c.Index = 0
					return true
				} else {
					return false
				}

			} else {
				// last doc
				return false
			}
		}
	} else {
		b, err := json.Marshal(c.Result[c.Index])
		err = json.Unmarshal(b, r)
		c.Index++ // move to next value into result
		if c.Index == max {
		}
		if err != nil {
			return false
		} else {
			return true
		}
	}
}

// move cursor index by 1
func (c *Cursor) Next(r interface{}) bool {
	if c.Index == c.max {
		return false
	} else {
		c.Index++
		if c.Index == c.max {
			return true
		} else {
			return false
		}
	}
}

type Extra struct {
	Stats Stats `json:"stats"`
}

type Stats struct {
	FullCount int `json:"fullCount"`
}

func (c Cursor) Count() int {
	return c.Amount
}

func (c *Cursor) FullCount() int {
	return c.Data.Stats.FullCount
}

func (c Cursor) HasMore() bool {
	return c.More
}

func (c Cursor) Error() bool {
	return c.Err
}

func (c Cursor) ErrCode() int {
	return c.Code
}
<commit_msg>Fix FetchOne returning not enough elements<commit_after>package aranGO

import (
	"encoding/json"
	"errors"
	"reflect"
	"time"
)

type Cursor struct {
	db *Database `json:"-"`
	Id string    `json:"Id"`

	Index  int           `json:"-"`
	Result []interface{} `json:"result"`
	More   bool          `json:"hasMore"`
	Amount int           `json:"count"`
	Data   Extra         `json:"extra"`

	Err    bool   `json:"error"`
	ErrMsg string `json:"errorMessage"`
	Code   int    `json:"code"`
	max    int
	Time   time.Duration `json:"time"`
}

func NewCursor(db *Database) *Cursor {
	var c Cursor
	if db == nil {
		return nil
	}
	c.db = db
	return &c
}

// Delete cursor in server and free RAM
func (c *Cursor) Delete() error {
	if c.Id == "" {
		return errors.New("Invalid cursor to delete")
	}
	res, err := c.db.send("cursor", c.Id, "DELETE", nil, c, c)
	if err != nil {
		return nil
	}

	switch res.Status() {
	case 202:
		return nil
	case 404:
		return errors.New("Cursor does not exist")
	default:
		return nil
	}

}

func (c *Cursor) FetchBatch(r interface{}) error {
	kind := reflect.ValueOf(r).Elem().Kind()
	if kind != reflect.Slice && kind != reflect.Array {
		return errors.New("Container must be Slice of array kind")
	}
	b, err := json.Marshal(c.Result)
	if err != nil {
		return err
	}
	err = json.Unmarshal(b, r)
	if err != nil {
		return err
	}

	// fetch next batch
	if c.HasMore() {
		res, err := c.db.send("cursor", c.Id, "PUT", nil, c, c)
		if res.Status() == 200 {
			return nil
		}

		if err != nil {
			return err
		}
	}

	return nil
}

// FetchOne iterates over cursor, returns false when no more values into batch, fetch next batch if necesary.
func (c *Cursor) FetchOne(r interface{}) bool {
	if c.Index > c.max {
		if c.More {
			//fetch rest from server
			res, err := c.db.send("cursor", c.Id, "PUT", nil, c, c)

			if err != nil {
				return false
			}

			if res.Status() == 200 {
				c.Index = 0
				return true
			} else {
				return false
			}
		} else {
			// last doc
			return false
		}
	} else {
		b, err := json.Marshal(c.Result[c.Index])
		err = json.Unmarshal(b, r)
		c.Index++ // move to next value into result
		if err != nil {
			return false
		} else {
			return true
		}
	}
}

// move cursor index by 1
func (c *Cursor) Next(r interface{}) bool {
	if c.Index == c.max {
		return false
	} else {
		c.Index++
		if c.Index == c.max {
			return true
		} else {
			return false
		}
	}
}

type Extra struct {
	Stats Stats `json:"stats"`
}

type Stats struct {
	FullCount int `json:"fullCount"`
}

func (c Cursor) Count() int {
	return c.Amount
}

func (c *Cursor) FullCount() int {
	return c.Data.Stats.FullCount
}

func (c Cursor) HasMore() bool {
	return c.More
}

func (c Cursor) Error() bool {
	return c.Err
}

func (c Cursor) ErrCode() int {
	return c.Code
}
<|endoftext|>


<commit_before>package bolster_test

import (
	"flag"
	"reflect"
	"testing"

	"github.com/kylelemons/godebug/pretty"
	"github.com/nochso/bolster"
	"github.com/nochso/bolster/internal"
)

var updateGold = flag.Bool("update", false, "update golden test files")

func TestTx_Insert_withoutAutoincrement(t *testing.T) {
	st, closer := internal.OpenTestStore(t)
	defer closer()

	err := st.Register(structWithID{})
	if err != nil {
		t.Error(err)
	}
	t.Run("first", func(t *testing.T) {
		err := st.Write(func(tx *bolster.Tx) error {
			return tx.Insert(&structWithID{})
		})
		if err != nil {
			t.Error(err)
		}
		internal.GoldStore(t, st, *updateGold)
	})
	t.Run("duplicate", func(t *testing.T) {
		err := st.Write(func(tx *bolster.Tx) error {
			return tx.Insert(&structWithID{})
		})
		if err == nil {
			t.Error("expected error, got nil")
		} else {
			t.Log(err)
		}
		internal.GoldStore(t, st, *updateGold)
	})
	t.Run("duplicateLazy", func(t *testing.T) {
		err := st.Write(func(tx *bolster.Tx) error {
			tx.Insert(&structWithID{})
			tx.Insert(&structWithID{})
			tx.Insert(&structWithID{})
			return nil
		})
		if err == nil {
			t.Error("expected error, got nil")
		} else {
			t.Log(err)
		}
		internal.GoldStore(t, st, *updateGold)
	})
	t.Run("withoutPointer", func(t *testing.T) {
		err := st.Write(func(tx *bolster.Tx) error {
			return tx.Insert(structWithID{})
		})
		if err == nil {
			t.Error("expected error, got nil")
		} else {
			t.Log(err)
		}
	})
	t.Run("pointerToNonStruct", func(t *testing.T) {
		err := st.Write(func(tx *bolster.Tx) error {
			return tx.Insert(new(int))
		})
		if err == nil {
			t.Error("expected error, got nil")
		} else {
			t.Log(err)
		}
	})
	t.Run("surrounding", func(t *testing.T) {
		err := st.Write(func(tx *bolster.Tx) error {
			for i := -5; i < 6; i++ {
				if i == 0 {
					continue
				}
				err := tx.Insert(&structWithID{ID: i})
				if err != nil {
					return err
				}
			}
			return nil
		})
		if err != nil {
			t.Error(err)
		}
		internal.GoldStore(t, st, *updateGold)
	})
}

func TestTx_Get(t *testing.T) {
	st, closer := internal.OpenTestStore(t)
	defer closer()

	err := st.Register(&structWithID{})
	if err != nil {
		t.Error(err)
	}
	exp := &structWithID{ID: 2}
	err = st.Write(func(tx *bolster.Tx) error {
		return tx.Insert(exp)
	})
	if err != nil {
		t.Error(err)
	}
	t.Run("NotFound", func(t *testing.T) {
		act := &structWithID{}
		err := st.Read(func(tx *bolster.Tx) error {
			return tx.Get(act, 1)
		})
		if err != bolster.ErrNotFound {
			t.Errorf("expected ErrNotFound, got %v", err)
		} else {
			t.Log(err)
		}
	})
	t.Run("Success", func(t *testing.T) {
		act := &structWithID{}
		err = st.Read(func(tx *bolster.Tx) error {
			return tx.Get(act, 2)
		})
		if err != nil {
			t.Error(err)
		}
		if !reflect.DeepEqual(act, exp) {
			t.Error(pretty.Compare(act, exp))
		}
	})
	t.Run("wrongTypeOfID", func(t *testing.T) {
		err := st.Read(func(tx *bolster.Tx) error {
			return tx.Get(&structWithID{}, "1")
		})
		if err == nil {
			t.Error("expected error, got nil")
		} else {
			t.Log(err)
		}
	})
}
<commit_msg>Test Get without pointer to struct<commit_after>package bolster_test

import (
	"flag"
	"reflect"
	"testing"

	"github.com/kylelemons/godebug/pretty"
	"github.com/nochso/bolster"
	"github.com/nochso/bolster/internal"
)

var updateGold = flag.Bool("update", false, "update golden test files")

func TestTx_Insert_withoutAutoincrement(t *testing.T) {
	st, closer := internal.OpenTestStore(t)
	defer closer()

	err := st.Register(structWithID{})
	if err != nil {
		t.Error(err)
	}
	t.Run("first", func(t *testing.T) {
		err := st.Write(func(tx *bolster.Tx) error {
			return tx.Insert(&structWithID{})
		})
		if err != nil {
			t.Error(err)
		}
		internal.GoldStore(t, st, *updateGold)
	})
	t.Run("duplicate", func(t *testing.T) {
		err := st.Write(func(tx *bolster.Tx) error {
			return tx.Insert(&structWithID{})
		})
		if err == nil {
			t.Error("expected error, got nil")
		} else {
			t.Log(err)
		}
		internal.GoldStore(t, st, *updateGold)
	})
	t.Run("duplicateLazy", func(t *testing.T) {
		err := st.Write(func(tx *bolster.Tx) error {
			tx.Insert(&structWithID{})
			tx.Insert(&structWithID{})
			tx.Insert(&structWithID{})
			return nil
		})
		if err == nil {
			t.Error("expected error, got nil")
		} else {
			t.Log(err)
		}
		internal.GoldStore(t, st, *updateGold)
	})
	t.Run("withoutPointer", func(t *testing.T) {
		err := st.Write(func(tx *bolster.Tx) error {
			return tx.Insert(structWithID{})
		})
		if err == nil {
			t.Error("expected error, got nil")
		} else {
			t.Log(err)
		}
	})
	t.Run("pointerToNonStruct", func(t *testing.T) {
		err := st.Write(func(tx *bolster.Tx) error {
			return tx.Insert(new(int))
		})
		if err == nil {
			t.Error("expected error, got nil")
		} else {
			t.Log(err)
		}
	})
	t.Run("surrounding", func(t *testing.T) {
		err := st.Write(func(tx *bolster.Tx) error {
			for i := -5; i < 6; i++ {
				if i == 0 {
					continue
				}
				err := tx.Insert(&structWithID{ID: i})
				if err != nil {
					return err
				}
			}
			return nil
		})
		if err != nil {
			t.Error(err)
		}
		internal.GoldStore(t, st, *updateGold)
	})
}

func TestTx_Get(t *testing.T) {
	st, closer := internal.OpenTestStore(t)
	defer closer()

	err := st.Register(&structWithID{})
	if err != nil {
		t.Error(err)
	}
	exp := &structWithID{ID: 2}
	err = st.Write(func(tx *bolster.Tx) error {
		return tx.Insert(exp)
	})
	if err != nil {
		t.Error(err)
	}
	t.Run("NotFound", func(t *testing.T) {
		act := &structWithID{}
		err := st.Read(func(tx *bolster.Tx) error {
			return tx.Get(act, 1)
		})
		if err != bolster.ErrNotFound {
			t.Errorf("expected ErrNotFound, got %v", err)
		} else {
			t.Log(err)
		}
	})
	t.Run("Success", func(t *testing.T) {
		act := &structWithID{}
		err = st.Read(func(tx *bolster.Tx) error {
			return tx.Get(act, 2)
		})
		if err != nil {
			t.Error(err)
		}
		if !reflect.DeepEqual(act, exp) {
			t.Error(pretty.Compare(act, exp))
		}
	})
	t.Run("wrongTypeOfID", func(t *testing.T) {
		err := st.Read(func(tx *bolster.Tx) error {
			return tx.Get(&structWithID{}, "1")
		})
		if err == nil {
			t.Error("expected error, got nil")
		} else {
			t.Log(err)
		}
	})
	t.Run("withoutPointer", func(t *testing.T) {
		err := st.Read(func(tx *bolster.Tx) error {
			return tx.Get(structWithID{}, "1")
		})
		if err == nil {
			t.Error("expected error, got nil")
		} else {
			t.Log(err)
		}
	})
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"

	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/digest"
	"github.com/docker/distribution/manifest/schema1"
	"github.com/docker/libtrust"
)

type manifestStore struct {
	repository                 *repository
	revisionStore              *revisionStore
	tagStore                   *tagStore
	ctx                        context.Context
	skipDependencyVerification bool
}

var _ distribution.ManifestService = &manifestStore{}

func (ms *manifestStore) Exists(dgst digest.Digest) (bool, error) {
	context.GetLogger(ms.ctx).Debug("(*manifestStore).Exists")

	_, err := ms.revisionStore.blobStore.Stat(ms.ctx, dgst)
	if err != nil {
		if err == distribution.ErrBlobUnknown {
			return false, nil
		}

		return false, err
	}

	return true, nil
}

func (ms *manifestStore) Get(dgst digest.Digest) (*schema1.SignedManifest, error) {
	context.GetLogger(ms.ctx).Debug("(*manifestStore).Get")
	return ms.revisionStore.get(ms.ctx, dgst)
}

// SkipLayerVerification allows a manifest to be Put before it's
// layers are on the filesystem
func SkipLayerVerification(ms distribution.ManifestService) error {
	if ms, ok := ms.(*manifestStore); ok {
		ms.skipDependencyVerification = true
		return nil
	}
	return fmt.Errorf("skip layer verification only valid for manifeststore")
}

func (ms *manifestStore) Put(manifest *schema1.SignedManifest) error {
	context.GetLogger(ms.ctx).Debug("(*manifestStore).Put")

	if err := ms.verifyManifest(ms.ctx, manifest); err != nil {
		return err
	}

	// Store the revision of the manifest
	revision, err := ms.revisionStore.put(ms.ctx, manifest)
	if err != nil {
		return err
	}

	// Now, tag the manifest
	return ms.tagStore.tag(manifest.Tag, revision.Digest)
}

// Delete removes the revision of the specified manfiest.
func (ms *manifestStore) Delete(dgst digest.Digest) error {
	context.GetLogger(ms.ctx).Debug("(*manifestStore).Delete")
	return ms.revisionStore.delete(ms.ctx, dgst)
}

func (ms *manifestStore) Tags() ([]string, error) {
	context.GetLogger(ms.ctx).Debug("(*manifestStore).Tags")
	return ms.tagStore.tags()
}

func (ms *manifestStore) ExistsByTag(tag string) (bool, error) {
	context.GetLogger(ms.ctx).Debug("(*manifestStore).ExistsByTag")
	return ms.tagStore.exists(tag)
}

func (ms *manifestStore) DeleteByTag(tag string) (bool, error) {
	context.GetLogger(ms.ctx).Debug("(*manifestStore).DeleteByTag")
	exist, err := ms.tagStore.exists(tag)
	if err != nil {
		return false, err
	} else if !exist {
		return true, nil
	} else {
		err = ms.tagStore.delete(tag)
		if err != nil {
			return false, err
		} else {
			return true, nil
		}
	}
}

func (ms *manifestStore) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) {
	for _, option := range options {
		err := option(ms)
		if err != nil {
			return nil, err
		}
	}

	context.GetLogger(ms.ctx).Debug("(*manifestStore).GetByTag")
	dgst, err := ms.tagStore.resolve(tag)
	if err != nil {
		return nil, err
	}

	return ms.revisionStore.get(ms.ctx, dgst)
}

// verifyManifest ensures that the manifest content is valid from the
// perspective of the registry. It ensures that the signature is valid for the
// enclosed payload. As a policy, the registry only tries to store valid
// content, leaving trust policies of that content up to consumers.
func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.SignedManifest) error {
	var errs distribution.ErrManifestVerification
	if mnfst.Name != ms.repository.Name() {
		errs = append(errs, fmt.Errorf("repository name does not match manifest name"))
	}

	if _, err := schema1.Verify(mnfst); err != nil {
		switch err {
		case libtrust.ErrMissingSignatureKey, libtrust.ErrInvalidJSONContent, libtrust.ErrMissingSignatureKey:
			errs = append(errs, distribution.ErrManifestUnverified{})
		default:
			if err.Error() == "invalid signature" { // TODO(stevvooe): This should be exported by libtrust
				errs = append(errs, distribution.ErrManifestUnverified{})
			} else {
				errs = append(errs, err)
			}
		}
	}

	if !ms.skipDependencyVerification {
		for _, fsLayer := range mnfst.FSLayers {
			_, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.BlobSum)
			if err != nil {
				if err != distribution.ErrBlobUnknown {
					errs = append(errs, err)
				}

				// On error here, we always append unknown blob errors.
				errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: fsLayer.BlobSum})
			}
		}
	}
	if len(errs) != 0 {
		return errs
	}

	return nil
}
<commit_msg>golint fix<commit_after>package storage

import (
	"fmt"

	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/digest"
	"github.com/docker/distribution/manifest/schema1"
	"github.com/docker/libtrust"
)

type manifestStore struct {
	repository                 *repository
	revisionStore              *revisionStore
	tagStore                   *tagStore
	ctx                        context.Context
	skipDependencyVerification bool
}

var _ distribution.ManifestService = &manifestStore{}

func (ms *manifestStore) Exists(dgst digest.Digest) (bool, error) {
	context.GetLogger(ms.ctx).Debug("(*manifestStore).Exists")

	_, err := ms.revisionStore.blobStore.Stat(ms.ctx, dgst)
	if err != nil {
		if err == distribution.ErrBlobUnknown {
			return false, nil
		}

		return false, err
	}

	return true, nil
}

func (ms *manifestStore) Get(dgst digest.Digest) (*schema1.SignedManifest, error) {
	context.GetLogger(ms.ctx).Debug("(*manifestStore).Get")
	return ms.revisionStore.get(ms.ctx, dgst)
}

// SkipLayerVerification allows a manifest to be Put before it's
// layers are on the filesystem
func SkipLayerVerification(ms distribution.ManifestService) error {
	if ms, ok := ms.(*manifestStore); ok {
		ms.skipDependencyVerification = true
		return nil
	}
	return fmt.Errorf("skip layer verification only valid for manifeststore")
}

func (ms *manifestStore) Put(manifest *schema1.SignedManifest) error {
	context.GetLogger(ms.ctx).Debug("(*manifestStore).Put")

	if err := ms.verifyManifest(ms.ctx, manifest); err != nil {
		return err
	}

	// Store the revision of the manifest
	revision, err := ms.revisionStore.put(ms.ctx, manifest)
	if err != nil {
		return err
	}

	// Now, tag the manifest
	return ms.tagStore.tag(manifest.Tag, revision.Digest)
}

// Delete removes the revision of the specified manfiest.
func (ms *manifestStore) Delete(dgst digest.Digest) error {
	context.GetLogger(ms.ctx).Debug("(*manifestStore).Delete")
	return ms.revisionStore.delete(ms.ctx, dgst)
}

func (ms *manifestStore) Tags() ([]string, error) {
	context.GetLogger(ms.ctx).Debug("(*manifestStore).Tags")
	return ms.tagStore.tags()
}

func (ms *manifestStore) ExistsByTag(tag string) (bool, error) {
	context.GetLogger(ms.ctx).Debug("(*manifestStore).ExistsByTag")
	return ms.tagStore.exists(tag)
}

func (ms *manifestStore) DeleteByTag(tag string) (bool, error) {
	context.GetLogger(ms.ctx).Debug("(*manifestStore).DeleteByTag")
	exist, err := ms.tagStore.exists(tag)
	if err != nil {
		return false, err
	} else if !exist {
		return true, nil
	} else {
		err = ms.tagStore.delete(tag)
		if err != nil {
			return false, err
		}
		return true, nil
	}
}

func (ms *manifestStore) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) {
	for _, option := range options {
		err := option(ms)
		if err != nil {
			return nil, err
		}
	}

	context.GetLogger(ms.ctx).Debug("(*manifestStore).GetByTag")
	dgst, err := ms.tagStore.resolve(tag)
	if err != nil {
		return nil, err
	}

	return ms.revisionStore.get(ms.ctx, dgst)
}

// verifyManifest ensures that the manifest content is valid from the
// perspective of the registry. It ensures that the signature is valid for the
// enclosed payload. As a policy, the registry only tries to store valid
// content, leaving trust policies of that content up to consumers.
func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.SignedManifest) error {
	var errs distribution.ErrManifestVerification
	if mnfst.Name != ms.repository.Name() {
		errs = append(errs, fmt.Errorf("repository name does not match manifest name"))
	}

	if _, err := schema1.Verify(mnfst); err != nil {
		switch err {
		case libtrust.ErrMissingSignatureKey, libtrust.ErrInvalidJSONContent, libtrust.ErrMissingSignatureKey:
			errs = append(errs, distribution.ErrManifestUnverified{})
		default:
			if err.Error() == "invalid signature" { // TODO(stevvooe): This should be exported by libtrust
				errs = append(errs, distribution.ErrManifestUnverified{})
			} else {
				errs = append(errs, err)
			}
		}
	}

	if !ms.skipDependencyVerification {
		for _, fsLayer := range mnfst.FSLayers {
			_, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.BlobSum)
			if err != nil {
				if err != distribution.ErrBlobUnknown {
					errs = append(errs, err)
				}

				// On error here, we always append unknown blob errors.
				errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: fsLayer.BlobSum})
			}
		}
	}
	if len(errs) != 0 {
		return errs
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/flowcommerce/tools/executor"
)

func main() {
	executor := executor.Create("proxy")

	executor = executor.Add("dev tag")
	executor = executor.Add("dev build_docker_image")

	executor.Run()
}
<commit_msg>Add release script<commit_after>package main

import (
	"github.com/flowcommerce/tools/executor"
)

func main() {
	executor := executor.Create("proxy")
	executor = executor.Add("dev tag")
	executor = executor.Add("dev build_docker_image")
	executor.Run()
}
<|endoftext|>