<commit_before>package main

/*
 * Copyright 2014 Albert P. Tobey <atobey@datastax.com> @AlTobey
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * pcstat.go - page cache stat
 *
 * uses the mincore(2) syscall to find out which pages (almost always 4k)
 * of a file are currently cached in memory
 *
 */

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"strings"
	"syscall"
	"time"
	"unsafe"
)

// pcStat: page cache status
// Bytes: size of the file (from os.File.Stat())
// Pages: array of booleans: true if cached, false otherwise
type pcStat struct {
	Name      string    `json:"filename"`  // file name as specified on command line
	Size      int64     `json:"size"`      // file size in bytes
	Timestamp time.Time `json:"timestamp"` // time right before calling mincore
	Mtime     time.Time `json:"mtime"`     // last modification time of the file
	Pages     int       `json:"pages"`     // total memory pages
	Cached    int       `json:"cached"`    // number of pages that are cached
	Uncached  int       `json:"uncached"`  // number of pages that are not cached
	Percent   float64   `json:"percent"`   // percentage of pages cached
	PPStat    []bool    `json:"status"`    // per-page status, true if cached, false otherwise
}

type pcStatList []pcStat

// adapted from https://groups.google.com/d/msg/golang-nuts/8d4pOPmSL9Q/H6WUqbGNELEJ
type winsize struct {
	ws_row, ws_col       uint16
	ws_xpixel, ws_ypixel uint16
}

var (
	terseFlag, nohdrFlag, jsonFlag, ppsFlag, histoFlag, bnameFlag bool
)

func init() {
	// TODO: error on useless/broken combinations
	flag.BoolVar(&terseFlag, "terse", false, "show terse output")
	flag.BoolVar(&nohdrFlag, "nohdr", false, "omit the header from terse & text output")
	flag.BoolVar(&jsonFlag, "json", false, "return data in JSON format")
	flag.BoolVar(&ppsFlag, "pps", false, "include the per-page status in JSON output")
	flag.BoolVar(&histoFlag, "histo", false, "print a simple histogram instead of raw data")
	flag.BoolVar(&bnameFlag, "bname", false, "convert paths to basename to narrow the output")
}

func main() {
	flag.Parse()

	// all non-flag arguments are considered to be filenames
	// this works well with shell globbing
	// file order is preserved throughout this program
	stats := make(pcStatList, len(flag.Args()))

	for i, fname := range flag.Args() {
		stats[i] = getMincore(fname, ppsFlag || histoFlag)
	}

	if jsonFlag {
		stats.formatJson()
	} else if terseFlag {
		stats.formatTerse()
	} else if histoFlag {
		stats.formatHistogram()
	} else {
		stats.formatText()
	}
}

func (stats pcStatList) formatText() {
	maxName := stats.maxName()

	// create horizontal grid line
	pad := strings.Repeat("-", len(maxName)+2)
	hr := fmt.Sprintf("|%s+----------------+------------+-----------+---------|", pad)

	fmt.Println(hr)

	// -nohdr may be chosen to save 2 lines of precious vertical space
	if !nohdrFlag {
		pad = strings.Repeat(" ", len(maxName)-4)
		fmt.Printf("| Name%s | Size           | Pages      | Cached    | Percent |\n", pad)
		fmt.Println(hr)
	}

	for _, pcs := range stats {
		pad = strings.Repeat(" ", len(maxName)-len(pcs.Name))

		// %07.3f was chosen to make it easy to scan the percentages vertically
		// I tried a few different formats only this one kept the decimals aligned
		fmt.Printf("| %s%s | %-15d| %-11d| %-10d| %07.3f |\n",
			pcs.Name, pad, pcs.Size, pcs.Pages, pcs.Cached, pcs.Percent)
	}

	fmt.Println(hr)
}

func (stats pcStatList) formatTerse() {
	if !nohdrFlag {
		fmt.Println("name,size,timestamp,mtime,pages,cached,percent")
	}
	for _, pcs := range stats {
		time := pcs.Timestamp.Unix()
		mtime := pcs.Mtime.Unix()
		fmt.Printf("%s,%d,%d,%d,%d,%d,%g\n",
			pcs.Name, pcs.Size, time, mtime, pcs.Pages, pcs.Cached, pcs.Percent)
	}
}

func (stats pcStatList) formatJson() {
	b, err := json.Marshal(stats)
	if err != nil {
		log.Fatalf("JSON formatting failed: %s\n", err)
	}
	os.Stdout.Write(b)
	fmt.Println("")
}

// references:
// http://www.unicode.org/charts/PDF/U2580.pdf
// https://github.com/puppetlabs/mcollective-puppet-agent/blob/master/application/puppet.rb#L143
// https://github.com/holman/spark
func (stats pcStatList) formatHistogram() {
	ws := getwinsize()
	maxName := stats.maxName()

	// block elements are wider than characters, so only use 1/2 the available columns
	buckets := (int(ws.ws_col)-len(maxName))/2 - 10

	for _, pcs := range stats {
		pad := strings.Repeat(" ", len(maxName)-len(pcs.Name))
		fmt.Printf("%s%s % 8d ", pcs.Name, pad, pcs.Pages)

		// when there is enough room display on/off for every page
		if buckets > pcs.Pages {
			for _, v := range pcs.PPStat {
				if v {
					fmt.Print("\u2588") // full block = 100%
				} else {
					fmt.Print("\u2581") // lower 1/8 block
				}
			}
		} else {
			bsz := pcs.Pages / buckets
			fbsz := float64(bsz)
			total := 0.0
			for i, v := range pcs.PPStat {
				if v {
					total++
				}

				if (i+1)%bsz == 0 {
					avg := total / fbsz
					if total == 0 {
						fmt.Print("\u2581") // lower 1/8 block = 0
					} else if avg < 16.67 {
						fmt.Print("\u2582") // lower 2/8 block
					} else if avg < 33.33 {
						fmt.Print("\u2583") // lower 3/8 block
					} else if avg < 50.00 {
						fmt.Print("\u2584") // lower 4/8 block
					} else if avg < 66.66 {
						fmt.Print("\u2585") // lower 5/8 block
					} else if avg < 83.33 {
						fmt.Print("\u2586") // lower 6/8 block
					} else if avg < 1.00 {
						fmt.Print("\u2587") // lower 7/8 block
					} else {
						fmt.Print("\u2588") // full block = 100%
					}

					total = 0
				}
			}
		}
		fmt.Println("")
	}
}

func getMincore(fname string, retpps bool) pcStat {
	f, err := os.Open(fname)
	if err != nil {
		log.Fatalf("Could not open file '%s' for read: %s\n", fname, err)
	}
	defer f.Close()

	// TEST TODO: verify behavior when the file size is changing quickly
	// while this function is running. I assume that the size parameter to
	// mincore will prevent overruns of the output vector, but it's not clear
	// what will be in there when the file is truncated between here and the
	// mincore() call.
	fi, err := f.Stat()
	if err != nil {
		log.Fatalf("Could not stat file %s: %s\n", fname, err)
	}
	if fi.Size() == 0 {
		log.Fatalf("%s appears to be 0 bytes in length\n", fname)
	}

	// []byte slice
	mmap, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_NONE, syscall.MAP_SHARED)
	if err != nil {
		log.Fatalf("Could not mmap file '%s': %s\n", fname, err)
	}
	// TODO: check for MAP_FAILED which is ((void *) -1)
	// but maybe unnecessary since it looks like errno is always set when MAP_FAILED

	// one byte per page, only LSB is used, remainder is reserved and clear
	vecsz := (fi.Size() + int64(os.Getpagesize()) - 1) / int64(os.Getpagesize())
	vec := make([]byte, vecsz)

	// get all of the arguments to the mincore syscall converted to uintptr
	mmap_ptr := uintptr(unsafe.Pointer(&mmap[0]))
	size_ptr := uintptr(fi.Size())
	vec_ptr := uintptr(unsafe.Pointer(&vec[0]))

	// get the timestamp right before the syscall
	ts := time.Now()

	// use Go's ASM to submit directly to the kernel, no C wrapper needed
	// mincore(2): int mincore(void *addr, size_t length, unsigned char *vec);
	// 0 on success, takes the pointer to the mmap, a size, which is the
	// size that came from f.Stat(), and the vector, which is a pointer
	// to the memory behind an []byte
	// this writes a snapshot of the data into vec which a list of 8-bit flags
	// with the LSB set if the page in that position is currently in VFS cache
	ret, _, err := syscall.Syscall(syscall.SYS_MINCORE, mmap_ptr, size_ptr, vec_ptr)
	if ret != 0 {
		log.Fatalf("syscall SYS_MINCORE failed: %s", err)
	}
	defer syscall.Munmap(mmap)

	pcs := pcStat{fname, fi.Size(), ts, fi.ModTime(), int(vecsz), 0, 0, 0.0, []bool{}}

	// only export the per-page cache mapping if it's explicitly enabled
	// an empty "status": [] field, but NBD.
	if retpps {
		pcs.PPStat = make([]bool, vecsz)

		// there is no bitshift only bool
		for i, b := range vec {
			if b%2 == 1 {
				pcs.PPStat[i] = true
			} else {
				pcs.PPStat[i] = false
			}
		}
	}

	// convert long paths to their basename with the -bname flag
	// this overwrites the original filename in pcs but it doesn't matter since
	// it's not used to access the file again -- and should not be!
	if bnameFlag {
		pcs.Name = path.Base(fname)
	}

	for _, b := range vec {
		if b%2 == 1 {
			pcs.Cached++
		} else {
			pcs.Uncached++
		}
	}

	// convert to float for the occasional sparsely-cached file
	// see the README.md for how to produce one
	pcs.Percent = (float64(pcs.Cached) / float64(pcs.Pages)) * 100.00

	return pcs
}

func getwinsize() winsize {
	ws := winsize{}
	_, _, err := syscall.Syscall(syscall.SYS_IOCTL,
		uintptr(0), uintptr(syscall.TIOCGWINSZ),
		uintptr(unsafe.Pointer(&ws)))
	if err != 0 {
		log.Fatalf("TIOCGWINSZ failed to get terminal size: %s\n", err)
	}
	return ws
}

// getLongestName returns the len of longest filename in the stat list
// if the bnameFlag is set, this will return the max basename len
func (stats pcStatList) maxName() string {
	var maxName string
	for _, pcs := range stats {
		if len(pcs.Name) > len(maxName) {
			maxName = pcs.Name
		}
	}
	return maxName
}
<commit_msg>fix stupid off-by-a-couple-orders-of-magnitude bug<commit_after>package main

/*
 * Copyright 2014 Albert P. Tobey <atobey@datastax.com> @AlTobey
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * pcstat.go - page cache stat
 *
 * uses the mincore(2) syscall to find out which pages (almost always 4k)
 * of a file are currently cached in memory
 *
 */

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"strings"
	"syscall"
	"time"
	"unsafe"
)

// pcStat: page cache status
// Bytes: size of the file (from os.File.Stat())
// Pages: array of booleans: true if cached, false otherwise
type pcStat struct {
	Name      string    `json:"filename"`  // file name as specified on command line
	Size      int64     `json:"size"`      // file size in bytes
	Timestamp time.Time `json:"timestamp"` // time right before calling mincore
	Mtime     time.Time `json:"mtime"`     // last modification time of the file
	Pages     int       `json:"pages"`     // total memory pages
	Cached    int       `json:"cached"`    // number of pages that are cached
	Uncached  int       `json:"uncached"`  // number of pages that are not cached
	Percent   float64   `json:"percent"`   // percentage of pages cached
	PPStat    []bool    `json:"status"`    // per-page status, true if cached, false otherwise
}

type pcStatList []pcStat

// adapted from https://groups.google.com/d/msg/golang-nuts/8d4pOPmSL9Q/H6WUqbGNELEJ
type winsize struct {
	ws_row, ws_col       uint16
	ws_xpixel, ws_ypixel uint16
}

var (
	terseFlag, nohdrFlag, jsonFlag, ppsFlag, histoFlag, bnameFlag bool
)

func init() {
	// TODO: error on useless/broken combinations
	flag.BoolVar(&terseFlag, "terse", false, "show terse output")
	flag.BoolVar(&nohdrFlag, "nohdr", false, "omit the header from terse & text output")
	flag.BoolVar(&jsonFlag, "json", false, "return data in JSON format")
	flag.BoolVar(&ppsFlag, "pps", false, "include the per-page status in JSON output")
	flag.BoolVar(&histoFlag, "histo", false, "print a simple histogram instead of raw data")
	flag.BoolVar(&bnameFlag, "bname", false, "convert paths to basename to narrow the output")
}

func main() {
	flag.Parse()

	// all non-flag arguments are considered to be filenames
	// this works well with shell globbing
	// file order is preserved throughout this program
	stats := make(pcStatList, len(flag.Args()))

	for i, fname := range flag.Args() {
		stats[i] = getMincore(fname, ppsFlag || histoFlag)
	}

	if jsonFlag {
		stats.formatJson()
	} else if terseFlag {
		stats.formatTerse()
	} else if histoFlag {
		stats.formatHistogram()
	} else {
		stats.formatText()
	}
}

func (stats pcStatList) formatText() {
	maxName := stats.maxName()

	// create horizontal grid line
	pad := strings.Repeat("-", len(maxName)+2)
	hr := fmt.Sprintf("|%s+----------------+------------+-----------+---------|", pad)

	fmt.Println(hr)

	// -nohdr may be chosen to save 2 lines of precious vertical space
	if !nohdrFlag {
		pad = strings.Repeat(" ", len(maxName)-4)
		fmt.Printf("| Name%s | Size           | Pages      | Cached    | Percent |\n", pad)
		fmt.Println(hr)
	}

	for _, pcs := range stats {
		pad = strings.Repeat(" ", len(maxName)-len(pcs.Name))

		// %07.3f was chosen to make it easy to scan the percentages vertically
		// I tried a few different formats only this one kept the decimals aligned
		fmt.Printf("| %s%s | %-15d| %-11d| %-10d| %07.3f |\n",
			pcs.Name, pad, pcs.Size, pcs.Pages, pcs.Cached, pcs.Percent)
	}

	fmt.Println(hr)
}

func (stats pcStatList) formatTerse() {
	if !nohdrFlag {
		fmt.Println("name,size,timestamp,mtime,pages,cached,percent")
	}
	for _, pcs := range stats {
		time := pcs.Timestamp.Unix()
		mtime := pcs.Mtime.Unix()
		fmt.Printf("%s,%d,%d,%d,%d,%d,%g\n",
			pcs.Name, pcs.Size, time, mtime, pcs.Pages, pcs.Cached, pcs.Percent)
	}
}

func (stats pcStatList) formatJson() {
	b, err := json.Marshal(stats)
	if err != nil {
		log.Fatalf("JSON formatting failed: %s\n", err)
	}
	os.Stdout.Write(b)
	fmt.Println("")
}

// references:
// http://www.unicode.org/charts/PDF/U2580.pdf
// https://github.com/puppetlabs/mcollective-puppet-agent/blob/master/application/puppet.rb#L143
// https://github.com/holman/spark
func (stats pcStatList) formatHistogram() {
	ws := getwinsize()
	maxName := stats.maxName()

	// block elements are wider than characters, so only use 1/2 the available columns
	buckets := (int(ws.ws_col)-len(maxName))/2 - 10

	for _, pcs := range stats {
		pad := strings.Repeat(" ", len(maxName)-len(pcs.Name))
		fmt.Printf("%s%s % 8d ", pcs.Name, pad, pcs.Pages)

		// when there is enough room display on/off for every page
		if buckets > pcs.Pages {
			for _, v := range pcs.PPStat {
				if v {
					fmt.Print("\u2588") // full block = 100%
				} else {
					fmt.Print("\u2581") // lower 1/8 block
				}
			}
		} else {
			bsz := pcs.Pages / buckets
			fbsz := float64(bsz)
			total := 0.0
			for i, v := range pcs.PPStat {
				if v {
					total++
				}

				if (i+1)%bsz == 0 {
					avg := total / fbsz
					if total == 0 {
						fmt.Print("\u2581") // lower 1/8 block = 0
					} else if avg < 0.16 {
						fmt.Print("\u2582") // lower 2/8 block
					} else if avg < 0.33 {
						fmt.Print("\u2583") // lower 3/8 block
					} else if avg < 0.50 {
						fmt.Print("\u2584") // lower 4/8 block
					} else if avg < 0.66 {
						fmt.Print("\u2585") // lower 5/8 block
					} else if avg < 0.83 {
						fmt.Print("\u2586") // lower 6/8 block
					} else if avg < 1.00 {
						fmt.Print("\u2587") // lower 7/8 block
					} else {
						fmt.Print("\u2588") // full block = 100%
					}

					total = 0
				}
			}
		}
		fmt.Println("")
	}
}

func getMincore(fname string, retpps bool) pcStat {
	f, err := os.Open(fname)
	if err != nil {
		log.Fatalf("Could not open file '%s' for read: %s\n", fname, err)
	}
	defer f.Close()

	// TEST TODO: verify behavior when the file size is changing quickly
	// while this function is running. I assume that the size parameter to
	// mincore will prevent overruns of the output vector, but it's not clear
	// what will be in there when the file is truncated between here and the
	// mincore() call.
	fi, err := f.Stat()
	if err != nil {
		log.Fatalf("Could not stat file %s: %s\n", fname, err)
	}
	if fi.Size() == 0 {
		log.Fatalf("%s appears to be 0 bytes in length\n", fname)
	}

	// []byte slice
	mmap, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_NONE, syscall.MAP_SHARED)
	if err != nil {
		log.Fatalf("Could not mmap file '%s': %s\n", fname, err)
	}
	// TODO: check for MAP_FAILED which is ((void *) -1)
	// but maybe unnecessary since it looks like errno is always set when MAP_FAILED

	// one byte per page, only LSB is used, remainder is reserved and clear
	vecsz := (fi.Size() + int64(os.Getpagesize()) - 1) / int64(os.Getpagesize())
	vec := make([]byte, vecsz)

	// get all of the arguments to the mincore syscall converted to uintptr
	mmap_ptr := uintptr(unsafe.Pointer(&mmap[0]))
	size_ptr := uintptr(fi.Size())
	vec_ptr := uintptr(unsafe.Pointer(&vec[0]))

	// get the timestamp right before the syscall
	ts := time.Now()

	// use Go's ASM to submit directly to the kernel, no C wrapper needed
	// mincore(2): int mincore(void *addr, size_t length, unsigned char *vec);
	// 0 on success, takes the pointer to the mmap, a size, which is the
	// size that came from f.Stat(), and the vector, which is a pointer
	// to the memory behind an []byte
	// this writes a snapshot of the data into vec which a list of 8-bit flags
	// with the LSB set if the page in that position is currently in VFS cache
	ret, _, err := syscall.Syscall(syscall.SYS_MINCORE, mmap_ptr, size_ptr, vec_ptr)
	if ret != 0 {
		log.Fatalf("syscall SYS_MINCORE failed: %s", err)
	}
	defer syscall.Munmap(mmap)

	pcs := pcStat{fname, fi.Size(), ts, fi.ModTime(), int(vecsz), 0, 0, 0.0, []bool{}}

	// only export the per-page cache mapping if it's explicitly enabled
	// an empty "status": [] field, but NBD.
	if retpps {
		pcs.PPStat = make([]bool, vecsz)

		// there is no bitshift only bool
		for i, b := range vec {
			if b%2 == 1 {
				pcs.PPStat[i] = true
			} else {
				pcs.PPStat[i] = false
			}
		}
	}

	// convert long paths to their basename with the -bname flag
	// this overwrites the original filename in pcs but it doesn't matter since
	// it's not used to access the file again -- and should not be!
	if bnameFlag {
		pcs.Name = path.Base(fname)
	}

	for _, b := range vec {
		if b%2 == 1 {
			pcs.Cached++
		} else {
			pcs.Uncached++
		}
	}

	// convert to float for the occasional sparsely-cached file
	// see the README.md for how to produce one
	pcs.Percent = (float64(pcs.Cached) / float64(pcs.Pages)) * 100.00

	return pcs
}

func getwinsize() winsize {
	ws := winsize{}
	_, _, err := syscall.Syscall(syscall.SYS_IOCTL,
		uintptr(0), uintptr(syscall.TIOCGWINSZ),
		uintptr(unsafe.Pointer(&ws)))
	if err != 0 {
		log.Fatalf("TIOCGWINSZ failed to get terminal size: %s\n", err)
	}
	return ws
}

// getLongestName returns the len of longest filename in the stat list
// if the bnameFlag is set, this will return the max basename len
func (stats pcStatList) maxName() string {
	var maxName string
	for _, pcs := range stats {
		if len(pcs.Name) > len(maxName) {
			maxName = pcs.Name
		}
	}
	return maxName
}
<|endoftext|>


<commit_before>package main

import (
	"time"
	"flag"

	"github.com/mitchellh/cli"
	"github.com/aws/aws-sdk-go/service/ec2"
	//"os"
)

type CreateCommand struct {
	Ui 			cli.Ui
	InstanceId 	string
	Name 		string
	DryRun		bool
	NoReboot	bool
}

// descriptions for args
var createDscrInstanceId = "The instance from which to create the AMI"
var createDscrName = "The name of the AMI; the current timestamp will be automatically appended"
var createDscrDryRun = "Execute a simulated run"
var createDscrNoReboot = "If true, do not reboot the instance before creating the AMI. It is preferable to reboot the instance to guarantee a consistent filesystem when taking the snapshot, but the likelihood of an inconsistent snapshot is very low."

func (c *CreateCommand) Help() string {
	return `ec2-snapper create <args> [--help]

Create an AMI of the given EC2 instance.

Available args are:
--instance      ` + createDscrInstanceId + `
--name          ` + createDscrName + `
--dry-run       ` + createDscrDryRun + `
--no-reboot     ` + createDscrNoReboot
}

func (c *CreateCommand) Synopsis() string {
	return "Create an AMI of the given EC2 instance"
}

func (c *CreateCommand) Run(args []string) int {

	// Handle the command-line args
	cmdFlags := flag.NewFlagSet("create", flag.ExitOnError)
	cmdFlags.Usage = func() { c.Ui.Output(c.Help()) }

	cmdFlags.StringVar(&c.InstanceId, "instance", "", createDscrInstanceId)
	cmdFlags.StringVar(&c.Name, "name", "", createDscrName)
	cmdFlags.BoolVar(&c.DryRun, "dry-run", false, createDscrDryRun)
	cmdFlags.BoolVar(&c.NoReboot, "no-reboot", true, createDscrNoReboot)

	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	// Check for required command-line args
	if c.InstanceId == "" {
		c.Ui.Error("ERROR: The argument '--instance' is required.")
		return 1
	}

	if c.Name == "" {
		c.Ui.Error("ERROR: The argument '--name' is required.")
		return 1
	}

	// Create an EC2 service object; AWS region is picked up from the "AWS_REGION" env var.
	svc := ec2.New(nil)

	// Generate a nicely formatted timestamp for right now
	const dateLayoutForAmiName = "2006-01-02 at 15_04_05 (MST)"
	t := time.Now()

	// Create the AMI Snapshot
	name := c.Name + " " + t.Format(dateLayoutForAmiName)
	instanceId := c.InstanceId
	dryRun := c.DryRun
	noReboot := c.NoReboot

	c.Ui.Output("==> Creating AMI for " + instanceId + "...")

	resp, err := svc.CreateImage(&ec2.CreateImageInput{
		Name: &name,
		InstanceID: &instanceId,
		DryRun: &dryRun,
		NoReboot: &noReboot })
	if err != nil {
		panic(err)
	}

	// Assign tags to this AMI.  We'll use these when it comes time to delete the AMI
	c.Ui.Output("==> Adding tags to AMI " + *resp.ImageID + "...")

	//const dateLayoutForTags = "2006-01-02 at 15:04:05 (UTC)"
	tagName1 := "ec2-snapper-instance-id"
	tagName2 := "ec2-snapper-snapshot-date"
	tagValue2 := time.Now().Format(time.RFC3339)

	svc.CreateTags(&ec2.CreateTagsInput{
		Resources: []*string{resp.ImageID},
		Tags: []*ec2.Tag{
			&ec2.Tag{ Key: &tagName1, Value: &c.InstanceId },
			&ec2.Tag{ Key: &tagName2, Value: &tagValue2 },
		},
	})

	c.Ui.Info("==> Success! Created " + *resp.ImageID + " named \"" + name + "\"")
	return 0
}

<commit_msg>Removed unneeded variables from create_command.go<commit_after>package main

import (
	"time"
	"flag"

	"github.com/mitchellh/cli"
	"github.com/aws/aws-sdk-go/service/ec2"
	//"os"
	"github.com/aws/aws-sdk-go/aws"
)

type CreateCommand struct {
	Ui 			cli.Ui
	InstanceId 	string
	Name 		string
	DryRun		bool
	NoReboot	bool
}

// descriptions for args
var createDscrInstanceId = "The instance from which to create the AMI"
var createDscrName = "The name of the AMI; the current timestamp will be automatically appended"
var createDscrDryRun = "Execute a simulated run"
var createDscrNoReboot = "If true, do not reboot the instance before creating the AMI. It is preferable to reboot the instance to guarantee a consistent filesystem when taking the snapshot, but the likelihood of an inconsistent snapshot is very low."

func (c *CreateCommand) Help() string {
	return `ec2-snapper create <args> [--help]

Create an AMI of the given EC2 instance.

Available args are:
--instance      ` + createDscrInstanceId + `
--name          ` + createDscrName + `
--dry-run       ` + createDscrDryRun + `
--no-reboot     ` + createDscrNoReboot
}

func (c *CreateCommand) Synopsis() string {
	return "Create an AMI of the given EC2 instance"
}

func (c *CreateCommand) Run(args []string) int {

	// Handle the command-line args
	cmdFlags := flag.NewFlagSet("create", flag.ExitOnError)
	cmdFlags.Usage = func() { c.Ui.Output(c.Help()) }

	cmdFlags.StringVar(&c.InstanceId, "instance", "", createDscrInstanceId)
	cmdFlags.StringVar(&c.Name, "name", "", createDscrName)
	cmdFlags.BoolVar(&c.DryRun, "dry-run", false, createDscrDryRun)
	cmdFlags.BoolVar(&c.NoReboot, "no-reboot", true, createDscrNoReboot)

	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	// Check for required command-line args
	if c.InstanceId == "" {
		c.Ui.Error("ERROR: The argument '--instance' is required.")
		return 1
	}

	if c.Name == "" {
		c.Ui.Error("ERROR: The argument '--name' is required.")
		return 1
	}

	// Create an EC2 service object; AWS region is picked up from the "AWS_REGION" env var.
	svc := ec2.New(nil)

	// Generate a nicely formatted timestamp for right now
	const dateLayoutForAmiName = "2006-01-02 at 15_04_05 (MST)"
	t := time.Now()

	// Create the AMI Snapshot
	name := c.Name + " " + t.Format(dateLayoutForAmiName)

	c.Ui.Output("==> Creating AMI for " + c.InstanceId + "...")

	resp, err := svc.CreateImage(&ec2.CreateImageInput{
		Name: &name,
		InstanceID: &c.InstanceId,
		DryRun: &c.DryRun,
		NoReboot: &c.NoReboot })
	if err != nil {
		panic(err)
	}

	// Assign tags to this AMI.  We'll use these when it comes time to delete the AMI
	c.Ui.Output("==> Adding tags to AMI " + *resp.ImageID + "...")

	dateLayoutForTagValue := time.Now().Format(time.RFC3339)

	svc.CreateTags(&ec2.CreateTagsInput{
		Resources: []*string{resp.ImageID},
		Tags: []*ec2.Tag{
			&ec2.Tag{ Key: aws.String("ec2-snapper-instance-id"), Value: &c.InstanceId },
			&ec2.Tag{ Key: aws.String("ec2-snapper-snapshot-date"), Value: &dateLayoutForTagValue },
		},
	})

	c.Ui.Info("==> Success! Created " + *resp.ImageID + " named \"" + name + "\"")
	return 0
}

<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode

import (
	"fmt"
	"os"
	"path"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

type DirInode struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	id           fuseops.InodeID
	implicitDirs bool

	// The the GCS object backing the inode. The object's name is used as a
	// prefix when listing. Special case: the empty string means this is the root
	// inode.
	//
	// INVARIANT: src != nil
	// INVARIANT: src.Name == "" || src.Name[len(name)-1] == '/'
	src storage.Object
}

var _ Inode = &DirInode{}

// Set notes on NewRootInode.
const RootGen int64 = 0

// Create a directory inode for the root of the file system. For this inode,
// the result of SourceGeneration() is guaranteed to be RootGen.
func NewRootInode(
	bucket gcs.Bucket,
	implicitDirs bool) (d *DirInode) {
	dummy := &storage.Object{
		Name:       "",
		Generation: RootGen,
	}

	d = NewDirInode(bucket, fuseops.RootInodeID, dummy, implicitDirs)
	return
}

// Create a directory inode for the supplied source object. The object's name
// must end with a slash unless this is the root directory, in which case it
// must be empty.
//
// If implicitDirs is set, LookUpChild will use ListObjects to find child
// directories that are "implicitly" defined by the existence of their own
// descendents. For example, if there is an object named "foo/bar/baz" and this
// is the directory "foo", a child directory named "bar" will be implied.
//
// REQUIRES: o != nil
// REQUIRES: o.Name == "" || o.Name[len(o.Name)-1] == '/'
func NewDirInode(
	bucket gcs.Bucket,
	id fuseops.InodeID,
	o *storage.Object,
	implicitDirs bool) (d *DirInode) {
	if o.Name != "" && o.Name[len(o.Name)-1] != '/' {
		panic(fmt.Sprintf("Unexpected name: %s", o.Name))
	}

	// Set up the struct.
	d = &DirInode{
		bucket:       bucket,
		id:           id,
		implicitDirs: implicitDirs,
		src:          *o,
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (d *DirInode) checkInvariants() {
	// INVARIANT: src.Name == "" || src.Name[len(name)-1] == '/'
	if !(d.src.Name == "" || d.src.Name[len(d.src.Name)-1] == '/') {
		panic(fmt.Sprintf("Unexpected name: %s", d.src.Name))
	}
}

func (d *DirInode) clobbered(ctx context.Context) (clobbered bool, err error) {
	// Special case: the root is never clobbered.
	if d.ID() == fuseops.RootInodeID {
		return
	}

	// Stat the backing object.
	req := &gcs.StatObjectRequest{
		Name: d.Name(),
	}

	o, err := d.bucket.StatObject(ctx, req)

	// "Not found" means clobbered.
	if _, ok := err.(*gcs.NotFoundError); ok {
		clobbered = true
		err = nil
		return
	}

	// Propagate other errors.
	if err != nil {
		err = fmt.Errorf("StatObject: %v", err)
		return
	}

	// We are clobbered if the generation number has changed.
	clobbered = o.Generation != d.SourceGeneration()

	return
}

func (d *DirInode) lookUpChildFile(
	ctx context.Context,
	name string) (o *storage.Object, err error) {
	o, err = statObjectMayNotExist(ctx, d.bucket, d.Name()+name)
	if err != nil {
		err = fmt.Errorf("statObjectMayNotExist: %v", err)
		return
	}

	return
}

func (d *DirInode) lookUpChildDir(
	ctx context.Context,
	name string) (o *storage.Object, err error) {
	b := syncutil.NewBundle(ctx)

	// Stat the placeholder object.
	b.Add(func(ctx context.Context) (err error) {
		o, err = statObjectMayNotExist(ctx, d.bucket, d.Name()+name+"/")
		if err != nil {
			err = fmt.Errorf("statObjectMayNotExist: %v", err)
			return
		}

		return
	})

	// If implicit directories are enabled, find out whether the child name is
	// implicitly defined.
	var implicitlyDefined bool
	if d.implicitDirs {
		b.Add(func(ctx context.Context) (err error) {
			implicitlyDefined, err = objectNamePrefixNonEmpty(
				ctx,
				d.bucket,
				d.Name()+name+"/")

			if err != nil {
				err = fmt.Errorf("objectNamePrefixNonEmpty: %v", err)
				return
			}

			return
		})
	}

	// Wait for both.
	err = b.Join()
	if err != nil {
		return
	}

	// If statting failed by the directory is implicitly defined, fake a source
	// object.
	if o == nil && implicitlyDefined {
		o = &storage.Object{
			Name:       d.Name() + name + "/",
			Generation: ImplicitDirGen,
		}
	}

	return
}

// List the supplied object name prefix to find out whether it is non-empty.
func objectNamePrefixNonEmpty(
	ctx context.Context,
	bucket gcs.Bucket,
	name string) (nonEmpty bool, err error)

// Stat the object with the given name, returning (nil, nil) if the object
// doesn't exist rather than failing.
func statObjectMayNotExist(
	ctx context.Context,
	bucket gcs.Bucket,
	name string) (o *storage.Object, err error) {
	// Call the bucket.
	req := &gcs.StatObjectRequest{
		Name: name,
	}

	o, err = bucket.StatObject(ctx, req)

	// Suppress "not found" errors.
	if _, ok := err.(*gcs.NotFoundError); ok {
		err = nil
	}

	// Annotate others.
	if err != nil {
		err = fmt.Errorf("StatObject: %v", err)
		return
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (d *DirInode) Lock() {
	// We don't require locks.
}

func (d *DirInode) Unlock() {
	// We don't require locks.
}

// Return the ID of this inode.
func (d *DirInode) ID() fuseops.InodeID {
	return d.id
}

// Return the full name of the directory object in GCS, including the trailing
// slash (e.g. "foo/bar/").
func (d *DirInode) Name() string {
	return d.src.Name
}

// Return the generation number from which this inode was branched.
func (d *DirInode) SourceGeneration() int64 {
	return d.src.Generation
}

func (d *DirInode) Attributes(
	ctx context.Context) (attrs fuseops.InodeAttributes, err error) {
	// Find out whether the backing object has been clobbered in GCS.
	clobbered, err := d.clobbered(ctx)
	if err != nil {
		err = fmt.Errorf("clobbered: %v", err)
		return
	}

	// Set up basic attributes.
	attrs = fuseops.InodeAttributes{
		Mode: 0700 | os.ModeDir,
	}

	// Modify Nlink as appropriate.
	if !clobbered {
		attrs.Nlink = 1
	}

	return
}

// See notes on DirInode.LookUpChild.
const ImplicitDirGen int64 = -1

// Look up the direct child with the given relative name, returning a record
// for the current object of that name in the GCS bucket. If both a file and a
// directory with the given name exist, be consistent from call to call about
// which is preferred. Return fuse.ENOENT if neither is found.
//
// If this inode was created with implicitDirs is set, this method will use
// ListObjects to find child directories that are "implicitly" defined by the
// existence of their own descendents. For example, if there is an object named
// "foo/bar/baz" and this is the directory "foo", a child directory named "bar"
// will be implied. In this case, the child's generation will be the sentinel
// value ImplicitDirGen.
func (d *DirInode) LookUpChild(
	ctx context.Context,
	name string) (o *storage.Object, err error) {
	b := syncutil.NewBundle(ctx)

	// Stat the child as a file.
	var fileRecord *storage.Object
	b.Add(func(ctx context.Context) (err error) {
		fileRecord, err = d.lookUpChildFile(ctx, name)
		return
	})

	// Stat the child as a directory.
	var dirRecord *storage.Object
	b.Add(func(ctx context.Context) (err error) {
		dirRecord, err = d.lookUpChildDir(ctx, name)
		return
	})

	// Wait for both.
	err = b.Join()
	if err != nil {
		return
	}

	// Prefer directories over files.
	switch {
	case dirRecord != nil:
		o = dirRecord
	case fileRecord != nil:
		o = fileRecord
	default:
		err = fuse.ENOENT
	}

	return
}

// Read some number of entries from the directory, returning a continuation
// token that can be used to pick up the read operation where it left off.
// Supply the empty token on the first call.
//
// At the end of the directory, the returned continuation token will be empty.
// Otherwise it will be non-empty. There is no guarantee about the number of
// entries returned; it may be zero even with a non-empty continuation token.
//
// The contents of the Offset and Inode fields for returned entries is
// undefined.
func (d *DirInode) ReadEntries(
	ctx context.Context,
	tok string) (entries []fuseutil.Dirent, newTok string, err error) {
	// Ask the bucket to list some objects.
	query := &storage.Query{
		Delimiter: "/",
		Prefix:    d.Name(),
		Cursor:    tok,
	}

	listing, err := d.bucket.ListObjects(ctx, query)
	if err != nil {
		err = fmt.Errorf("ListObjects: %v", err)
		return
	}

	// Convert objects to entries for files.
	for _, o := range listing.Results {
		e := fuseutil.Dirent{
			Name: path.Base(o.Name),
			Type: fuseutil.DT_File,
		}

		entries = append(entries, e)
	}

	// Convert prefixes to entries for directories.
	for _, p := range listing.Prefixes {
		e := fuseutil.Dirent{
			Name: path.Base(p),
			Type: fuseutil.DT_Directory,
		}

		entries = append(entries, e)
	}

	// Return an appropriate continuation token, if any.
	if listing.Next != nil {
		newTok = listing.Next.Cursor
	}

	return
}
<commit_msg>Implemented objectNamePrefixNonEmpty.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode

import (
	"fmt"
	"os"
	"path"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

type DirInode struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	id           fuseops.InodeID
	implicitDirs bool

	// The the GCS object backing the inode. The object's name is used as a
	// prefix when listing. Special case: the empty string means this is the root
	// inode.
	//
	// INVARIANT: src != nil
	// INVARIANT: src.Name == "" || src.Name[len(name)-1] == '/'
	src storage.Object
}

var _ Inode = &DirInode{}

// Set notes on NewRootInode.
const RootGen int64 = 0

// Create a directory inode for the root of the file system. For this inode,
// the result of SourceGeneration() is guaranteed to be RootGen.
func NewRootInode(
	bucket gcs.Bucket,
	implicitDirs bool) (d *DirInode) {
	dummy := &storage.Object{
		Name:       "",
		Generation: RootGen,
	}

	d = NewDirInode(bucket, fuseops.RootInodeID, dummy, implicitDirs)
	return
}

// Create a directory inode for the supplied source object. The object's name
// must end with a slash unless this is the root directory, in which case it
// must be empty.
//
// If implicitDirs is set, LookUpChild will use ListObjects to find child
// directories that are "implicitly" defined by the existence of their own
// descendents. For example, if there is an object named "foo/bar/baz" and this
// is the directory "foo", a child directory named "bar" will be implied.
//
// REQUIRES: o != nil
// REQUIRES: o.Name == "" || o.Name[len(o.Name)-1] == '/'
func NewDirInode(
	bucket gcs.Bucket,
	id fuseops.InodeID,
	o *storage.Object,
	implicitDirs bool) (d *DirInode) {
	if o.Name != "" && o.Name[len(o.Name)-1] != '/' {
		panic(fmt.Sprintf("Unexpected name: %s", o.Name))
	}

	// Set up the struct.
	d = &DirInode{
		bucket:       bucket,
		id:           id,
		implicitDirs: implicitDirs,
		src:          *o,
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (d *DirInode) checkInvariants() {
	// INVARIANT: src.Name == "" || src.Name[len(name)-1] == '/'
	if !(d.src.Name == "" || d.src.Name[len(d.src.Name)-1] == '/') {
		panic(fmt.Sprintf("Unexpected name: %s", d.src.Name))
	}
}

func (d *DirInode) clobbered(ctx context.Context) (clobbered bool, err error) {
	// Special case: the root is never clobbered.
	if d.ID() == fuseops.RootInodeID {
		return
	}

	// Stat the backing object.
	req := &gcs.StatObjectRequest{
		Name: d.Name(),
	}

	o, err := d.bucket.StatObject(ctx, req)

	// "Not found" means clobbered.
	if _, ok := err.(*gcs.NotFoundError); ok {
		clobbered = true
		err = nil
		return
	}

	// Propagate other errors.
	if err != nil {
		err = fmt.Errorf("StatObject: %v", err)
		return
	}

	// We are clobbered if the generation number has changed.
	clobbered = o.Generation != d.SourceGeneration()

	return
}

func (d *DirInode) lookUpChildFile(
	ctx context.Context,
	name string) (o *storage.Object, err error) {
	o, err = statObjectMayNotExist(ctx, d.bucket, d.Name()+name)
	if err != nil {
		err = fmt.Errorf("statObjectMayNotExist: %v", err)
		return
	}

	return
}

func (d *DirInode) lookUpChildDir(
	ctx context.Context,
	name string) (o *storage.Object, err error) {
	b := syncutil.NewBundle(ctx)

	// Stat the placeholder object.
	b.Add(func(ctx context.Context) (err error) {
		o, err = statObjectMayNotExist(ctx, d.bucket, d.Name()+name+"/")
		if err != nil {
			err = fmt.Errorf("statObjectMayNotExist: %v", err)
			return
		}

		return
	})

	// If implicit directories are enabled, find out whether the child name is
	// implicitly defined.
	var implicitlyDefined bool
	if d.implicitDirs {
		b.Add(func(ctx context.Context) (err error) {
			implicitlyDefined, err = objectNamePrefixNonEmpty(
				ctx,
				d.bucket,
				d.Name()+name+"/")

			if err != nil {
				err = fmt.Errorf("objectNamePrefixNonEmpty: %v", err)
				return
			}

			return
		})
	}

	// Wait for both.
	err = b.Join()
	if err != nil {
		return
	}

	// If statting failed by the directory is implicitly defined, fake a source
	// object.
	if o == nil && implicitlyDefined {
		o = &storage.Object{
			Name:       d.Name() + name + "/",
			Generation: ImplicitDirGen,
		}
	}

	return
}

// List the supplied object name prefix to find out whether it is non-empty.
func objectNamePrefixNonEmpty(
	ctx context.Context,
	bucket gcs.Bucket,
	prefix string) (nonEmpty bool, err error) {
	query := &storage.Query{
		Prefix:     prefix,
		MaxResults: 1,
	}

	listing, err := bucket.ListObjects(ctx, query)
	if err != nil {
		err = fmt.Errorf("ListObjects: %v", err)
		return
	}

	nonEmpty = len(listing.Results) != 0
	return
}

// Stat the object with the given name, returning (nil, nil) if the object
// doesn't exist rather than failing.
func statObjectMayNotExist(
	ctx context.Context,
	bucket gcs.Bucket,
	name string) (o *storage.Object, err error) {
	// Call the bucket.
	req := &gcs.StatObjectRequest{
		Name: name,
	}

	o, err = bucket.StatObject(ctx, req)

	// Suppress "not found" errors.
	if _, ok := err.(*gcs.NotFoundError); ok {
		err = nil
	}

	// Annotate others.
	if err != nil {
		err = fmt.Errorf("StatObject: %v", err)
		return
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (d *DirInode) Lock() {
	// We don't require locks.
}

func (d *DirInode) Unlock() {
	// We don't require locks.
}

// Return the ID of this inode.
func (d *DirInode) ID() fuseops.InodeID {
	return d.id
}

// Return the full name of the directory object in GCS, including the trailing
// slash (e.g. "foo/bar/").
func (d *DirInode) Name() string {
	return d.src.Name
}

// Return the generation number from which this inode was branched.
func (d *DirInode) SourceGeneration() int64 {
	return d.src.Generation
}

func (d *DirInode) Attributes(
	ctx context.Context) (attrs fuseops.InodeAttributes, err error) {
	// Find out whether the backing object has been clobbered in GCS.
	clobbered, err := d.clobbered(ctx)
	if err != nil {
		err = fmt.Errorf("clobbered: %v", err)
		return
	}

	// Set up basic attributes.
	attrs = fuseops.InodeAttributes{
		Mode: 0700 | os.ModeDir,
	}

	// Modify Nlink as appropriate.
	if !clobbered {
		attrs.Nlink = 1
	}

	return
}

// See notes on DirInode.LookUpChild.
const ImplicitDirGen int64 = -1

// Look up the direct child with the given relative name, returning a record
// for the current object of that name in the GCS bucket. If both a file and a
// directory with the given name exist, be consistent from call to call about
// which is preferred. Return fuse.ENOENT if neither is found.
//
// If this inode was created with implicitDirs is set, this method will use
// ListObjects to find child directories that are "implicitly" defined by the
// existence of their own descendents. For example, if there is an object named
// "foo/bar/baz" and this is the directory "foo", a child directory named "bar"
// will be implied. In this case, the child's generation will be the sentinel
// value ImplicitDirGen.
func (d *DirInode) LookUpChild(
	ctx context.Context,
	name string) (o *storage.Object, err error) {
	b := syncutil.NewBundle(ctx)

	// Stat the child as a file.
	var fileRecord *storage.Object
	b.Add(func(ctx context.Context) (err error) {
		fileRecord, err = d.lookUpChildFile(ctx, name)
		return
	})

	// Stat the child as a directory.
	var dirRecord *storage.Object
	b.Add(func(ctx context.Context) (err error) {
		dirRecord, err = d.lookUpChildDir(ctx, name)
		return
	})

	// Wait for both.
	err = b.Join()
	if err != nil {
		return
	}

	// Prefer directories over files.
	switch {
	case dirRecord != nil:
		o = dirRecord
	case fileRecord != nil:
		o = fileRecord
	default:
		err = fuse.ENOENT
	}

	return
}

// Read some number of entries from the directory, returning a continuation
// token that can be used to pick up the read operation where it left off.
// Supply the empty token on the first call.
//
// At the end of the directory, the returned continuation token will be empty.
// Otherwise it will be non-empty. There is no guarantee about the number of
// entries returned; it may be zero even with a non-empty continuation token.
//
// The contents of the Offset and Inode fields for returned entries is
// undefined.
func (d *DirInode) ReadEntries(
	ctx context.Context,
	tok string) (entries []fuseutil.Dirent, newTok string, err error) {
	// Ask the bucket to list some objects.
	query := &storage.Query{
		Delimiter: "/",
		Prefix:    d.Name(),
		Cursor:    tok,
	}

	listing, err := d.bucket.ListObjects(ctx, query)
	if err != nil {
		err = fmt.Errorf("ListObjects: %v", err)
		return
	}

	// Convert objects to entries for files.
	for _, o := range listing.Results {
		e := fuseutil.Dirent{
			Name: path.Base(o.Name),
			Type: fuseutil.DT_File,
		}

		entries = append(entries, e)
	}

	// Convert prefixes to entries for directories.
	for _, p := range listing.Prefixes {
		e := fuseutil.Dirent{
			Name: path.Base(p),
			Type: fuseutil.DT_Directory,
		}

		entries = append(entries, e)
	}

	// Return an appropriate continuation token, if any.
	if listing.Next != nil {
		newTok = listing.Next.Cursor
	}

	return
}
<|endoftext|>


<commit_before><commit_msg>Many more test cases for Connection<commit_after><|endoftext|>


<commit_before>package scheduling

import (
	"sync"

	"go.skia.org/infra/go/swarming"
	"go.skia.org/infra/go/trie"
	"go.skia.org/infra/task_scheduler/go/db"

	swarming_api "go.chromium.org/luci/common/api/swarming/swarming/v1"
)

const (
	MEASUREMENT_BUSY_BOTS = "busy-bots"
)

// busyBots is a struct used for marking a bot as busy while it runs a Task.
type busyBots struct {
	pendingTasks *trie.Trie
	mtx          sync.Mutex
}

// newBusyBots returns a busyBots instance.
func newBusyBots() *busyBots {
	return &busyBots{
		pendingTasks: trie.New(),
	}
}

// Filter returns a copy of the given slice of bots with the busy bots removed.
func (b *busyBots) Filter(bots []*swarming_api.SwarmingRpcsBotInfo) []*swarming_api.SwarmingRpcsBotInfo {
	b.mtx.Lock()
	defer b.mtx.Unlock()
	matched := make(map[string]bool, len(bots))
	rv := make([]*swarming_api.SwarmingRpcsBotInfo, 0, len(bots))
	for _, bot := range bots {
		// Find matching tasks.
		matches := b.pendingTasks.SearchSubset(swarming.BotDimensionsToStringSlice(bot.Dimensions))
		// Choose the first non-empty entry and pretend that
		// this bot is busy with that task.
		var e string
		for _, match := range matches {
			m := match.(string)
			if _, ok := matched[m]; !ok {
				e = m
				break
			}
		}
		if e != "" {
			matched[e] = true
		} else {
			rv = append(rv, bot)
		}
	}
	return rv
}

// RefreshTasks updates the contents of busyBots based on the cached tasks.
func (b *busyBots) RefreshTasks(pending []*swarming_api.SwarmingRpcsTaskResult) {
	b.mtx.Lock()
	defer b.mtx.Unlock()

	b.pendingTasks = trie.New()
	for _, t := range pending {
		dims := db.DimensionsFromTags(t.Tags)
		b.pendingTasks.Insert(dims, t.TaskId)
	}
}
<commit_msg>[task scheduler] Add metrics for free bot count.<commit_after>package scheduling

import (
	"sort"
	"strings"
	"sync"

	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/swarming"
	"go.skia.org/infra/go/trie"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/task_scheduler/go/db"

	swarming_api "go.chromium.org/luci/common/api/swarming/swarming/v1"
)

const (
	// Metric name for free bots.
	MEASUREMENT_FREE_BOT_COUNT = "free_bot_count"

	// FILTER_* are used as the value of the "filter" key in metrics; we record counts for all free
	// bots and all free bots after allocating pending tasks to bots.
	FILTER_ALL_FREE_BOTS       = "all_free_bots"
	FILTER_MINUS_PENDING_TASKS = "minus_pending_tasks"
)

var (
	// dimensionWhitelist includes all dimensions used in
	// https://skia.googlesource.com/skia/+/42974b73cd6f3515af69c553aac8dd15e3fc1927/infra/bots/gen_tasks.go
	// (except for "image" which has a TODO to remove).
	dimensionWhitelist = []string{
		"cpu",
		"device",
		"device_os",
		"device_type",
		"gpu",
		"machine_type",
		"os",
		"release_version",
		"valgrind",
	}
)

func init() {
	sort.Strings(dimensionWhitelist)
}

// busyBots is a struct used for marking a bot as busy while it runs a Task.
type busyBots struct {
	// map[<filter>]map[<dimensionsString>]<count of bots>
	freeBotMetrics map[string]map[string]metrics2.Int64Metric
	pendingTasks   *trie.Trie
	mtx            sync.Mutex
}

// newBusyBots returns a busyBots instance.
func newBusyBots() *busyBots {
	return &busyBots{
		freeBotMetrics: map[string]map[string]metrics2.Int64Metric{},
		pendingTasks:   trie.New(),
	}
}

// Return a space-separated string of sorted dimensions and values, filtered by dimensionWhitelist.
// Similar to flatten in task_scheduler.go. When there are multiple values for a dimension, the
// longest is used. (The longest value is usually the most interesting.)
func dimensionsString(dims []*swarming_api.SwarmingRpcsStringListPair) string {
	vals := make(map[string]string, len(dimensionWhitelist))
	for _, dim := range dims {
		if util.In(dim.Key, dimensionWhitelist) {
			for _, val := range dim.Value {
				if len(val) > len(vals[dim.Key]) {
					vals[dim.Key] = val
				}
			}
		}
	}
	rv := make([]string, 0, 2*len(vals))
	for _, key := range dimensionWhitelist {
		if vals[key] != "" {
			rv = append(rv, key, vals[key])
		}
	}
	return strings.Join(rv, " ")
}

// recordBotMetrics updates MEASUREMENT_FREE_BOT_COUNT for the given filter based on bots. Assumes
// b.mtx is locked.
func (b *busyBots) recordBotMetrics(filter string, bots []*swarming_api.SwarmingRpcsBotInfo) {
	metrics, ok := b.freeBotMetrics[filter]
	if !ok {
		metrics = map[string]metrics2.Int64Metric{}
		b.freeBotMetrics[filter] = metrics
	}
	counts := map[string]int64{}
	for _, bot := range bots {
		counts[dimensionsString(bot.Dimensions)]++
	}
	for dims, count := range counts {
		metric, ok := metrics[dims]
		if !ok {
			metric = metrics2.GetInt64Metric(MEASUREMENT_FREE_BOT_COUNT, map[string]string{
				"filter":     filter,
				"dimensions": dims,
			})
			metrics[dims] = metric
		}
		metric.Update(count)
	}
	for dims, metric := range metrics {
		_, ok := counts[dims]
		if !ok {
			metric.Update(0)
			delete(metrics, dims)
		}
	}
}

// Filter returns a copy of the given slice of bots with the busy bots removed.
func (b *busyBots) Filter(bots []*swarming_api.SwarmingRpcsBotInfo) []*swarming_api.SwarmingRpcsBotInfo {
	b.mtx.Lock()
	defer b.mtx.Unlock()
	b.recordBotMetrics(FILTER_ALL_FREE_BOTS, bots)
	matched := make(map[string]bool, len(bots))
	rv := make([]*swarming_api.SwarmingRpcsBotInfo, 0, len(bots))
	for _, bot := range bots {
		// Find matching tasks.
		matches := b.pendingTasks.SearchSubset(swarming.BotDimensionsToStringSlice(bot.Dimensions))
		// Choose the first non-empty entry and pretend that
		// this bot is busy with that task.
		var e string
		for _, match := range matches {
			m := match.(string)
			if _, ok := matched[m]; !ok {
				e = m
				break
			}
		}
		if e != "" {
			matched[e] = true
		} else {
			rv = append(rv, bot)
		}
	}
	b.recordBotMetrics(FILTER_MINUS_PENDING_TASKS, bots)
	return rv
}

// RefreshTasks updates the contents of busyBots based on the cached tasks.
func (b *busyBots) RefreshTasks(pending []*swarming_api.SwarmingRpcsTaskResult) {
	b.mtx.Lock()
	defer b.mtx.Unlock()

	b.pendingTasks = trie.New()
	for _, t := range pending {
		dims := db.DimensionsFromTags(t.Tags)
		b.pendingTasks.Insert(dims, t.TaskId)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/dgryski/go-tsz"
	"github.com/gocql/gocql"
	"github.com/grafana/grafana/pkg/log"
	"github.com/raintank/met"
)

// write aggregated data to cassandra.

const month_sec = 60 * 60 * 24 * 28

const keyspace_schema = `CREATE KEYSPACE IF NOT EXISTS raintank WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}  AND durable_writes = true`
const table_schema = `CREATE TABLE IF NOT EXISTS raintank.metric (
    key ascii,
    ts int,
    data blob,
    PRIMARY KEY (key, ts)
) WITH COMPACT STORAGE
    AND CLUSTERING ORDER BY (ts DESC)
    AND compaction = {'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy'}
    AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'}
    AND read_repair_chance = 0.0
    AND dclocal_read_repair_chance = 0`

/*
https://godoc.org/github.com/gocql/gocql#Session
Session is the interface used by users to interact with the database.
It's safe for concurrent use by multiple goroutines and a typical usage scenario is to have one global session
object to interact with the whole Cassandra cluster.
*/

type cassandraStore struct {
	session          *gocql.Session
	writeQueues      []chan *ChunkWriteRequest
	readQueue        chan *ChunkReadRequest
	writeQueueMeters []met.Meter
}

func NewCassandraStore(stats met.Backend, addrs, consistency string, timeout, readers, writers int) (*cassandraStore, error) {
	cluster := gocql.NewCluster(strings.Split(addrs, ",")...)
	cluster.Consistency = gocql.ParseConsistency(consistency)
	cluster.Timeout = time.Duration(timeout) * time.Millisecond
	cluster.NumConns = writers
	var err error
	tmpSession, err := cluster.CreateSession()
	if err != nil {
		return nil, err
	}
	// ensure the keyspace and table exist.
	err = tmpSession.Query(keyspace_schema).Exec()
	if err != nil {
		return nil, err
	}
	err = tmpSession.Query(table_schema).Exec()
	if err != nil {
		return nil, err
	}
	tmpSession.Close()
	cluster.Keyspace = "raintank"
	session, err := cluster.CreateSession()
	if err != nil {
		return nil, err
	}
	c := &cassandraStore{
		session:          session,
		writeQueues:      make([]chan *ChunkWriteRequest, writers),
		readQueue:        make(chan *ChunkReadRequest, *cassandraReadQueueSize),
		writeQueueMeters: make([]met.Meter, writers),
	}

	for i := 0; i < *cassandraWriteConcurrency; i++ {
		c.writeQueues[i] = make(chan *ChunkWriteRequest, *cassandraWriteQueueSize)
		c.writeQueueMeters[i] = stats.NewMeter(fmt.Sprintf("cassandra.write_queue.%d.items", i+1), 0)
		go c.processWriteQueue(c.writeQueues[i], c.writeQueueMeters[i])
	}

	for i := 0; i < *cassandraReadConcurrency; i++ {
		go c.processReadQueue()
	}

	return c, err
}

func (c *cassandraStore) Add(cwr *ChunkWriteRequest) {
	sum := 0
	for _, char := range cwr.key {
		sum += int(char)
	}
	which := sum % *cassandraWriteConcurrency
	c.writeQueueMeters[which].Value(int64(len(c.writeQueues[which])))
	c.writeQueues[which] <- cwr
	c.writeQueueMeters[which].Value(int64(len(c.writeQueues[which])))
}

/* process writeQueue.
 */
func (c *cassandraStore) processWriteQueue(queue chan *ChunkWriteRequest, meter met.Meter) {
	tick := time.Tick(time.Duration(1) * time.Second)
	for {
		select {
		case <-tick:
			meter.Value(int64(len(queue)))
		case cwr := <-queue:
			meter.Value(int64(len(queue)))
			log.Debug("starting to save %s:%d %v", cwr.key, cwr.chunk.T0, cwr.chunk)
			//log how long the chunk waited in the queue before we attempted to save to cassandra
			cassBlockDuration.Value(time.Now().Sub(cwr.timestamp))

			data := cwr.chunk.Series.Bytes()
			chunkSizeAtSave.Value(int64(len(data)))
			version := FormatStandardGoTsz
			buf := new(bytes.Buffer)
			err := binary.Write(buf, binary.LittleEndian, uint8(version))
			if err != nil {
				// TODO
			}
			_, err = buf.Write(data)
			if err != nil {
				// TODO
			}
			success := false
			attempts := 0
			for !success {
				err := c.insertChunk(cwr.key, cwr.chunk.T0, buf.Bytes(), int(cwr.ttl))
				if err == nil {
					success = true
					cwr.chunk.Saved = true
					SendPersistMessage(cwr.key, cwr.chunk.T0)
					log.Debug("save complete. %s:%d %v", cwr.key, cwr.chunk.T0, cwr.chunk)
					chunkSaveOk.Inc(1)
				} else {
					if (attempts % 20) == 0 {
						log.Warn("failed to save chunk to cassandra after %d attempts. %v, %s", attempts+1, cwr.chunk, err)
					}
					chunkSaveFail.Inc(1)
					sleepTime := 100 * attempts
					if sleepTime > 2000 {
						sleepTime = 2000
					}
					time.Sleep(time.Duration(sleepTime) * time.Millisecond)
					attempts++
				}
			}
		}
	}
}

// Insert Chunks into Cassandra.
//
// key: is the metric_id
// ts: is the start of the aggregated time range.
// data: is the payload as bytes.
func (c *cassandraStore) insertChunk(key string, t0 uint32, data []byte, ttl int) error {
	// for unit tests
	if c.session == nil {
		return nil
	}
	query := fmt.Sprintf("INSERT INTO metric (key, ts, data) values(?,?,?) USING TTL %d", ttl)
	row_key := fmt.Sprintf("%s_%d", key, t0/month_sec) // "month number" based on unix timestamp (rounded down)
	pre := time.Now()
	ret := c.session.Query(query, row_key, t0, data).Exec()
	cassPutDuration.Value(time.Now().Sub(pre))
	return ret
}

type outcome struct {
	month   uint32
	sortKey uint32
	i       *gocql.Iter
}
type asc []outcome

func (o asc) Len() int           { return len(o) }
func (o asc) Swap(i, j int)      { o[i], o[j] = o[j], o[i] }
func (o asc) Less(i, j int) bool { return o[i].sortKey < o[j].sortKey }

func (c *cassandraStore) processReadQueue() {
	for crr := range c.readQueue {
		crr.out <- outcome{crr.month, crr.sortKey, c.session.Query(crr.q, crr.p...).Iter()}
	}
}

// Basic search of cassandra.
// start inclusive, end exclusive
func (c *cassandraStore) Search(key string, start, end uint32) ([]Iter, error) {
	iters := make([]Iter, 0)
	if start > end {
		return iters, fmt.Errorf("start must be before end.")
	}

	results := make(chan outcome)
	numQueries := 0

	query := func(month, sortKey uint32, q string, p ...interface{}) {
		numQueries += 1
		c.readQueue <- &ChunkReadRequest{month, sortKey, q, p, results}
	}

	start_month := start - (start % month_sec)       // starting row has to be at, or before, requested start
	end_month := (end - 1) - ((end - 1) % month_sec) // ending row has to be include the last point we might need

	pre := time.Now()

	// unfortunately in the database we only have the t0's of all chunks.
	// this means we can easily make sure to include the correct last chunk (just query for a t0 < end, the last chunk will contain the last needed data)
	// but it becomes hard to find which should be the first chunk to include. we can't just query for start <= t0 because than will miss some data at the beginning
	// we can't assume we know the chunkSpan so we can't just calculate the t0 >= start - <some-predefined-number> because chunkSpans may change over time.
	// we effectively need all chunks with a t0 > start, as well as the last chunk with a t0 <= start.
	// since we make sure that you can only use chunkSpans so that month_sec % chunkSpan == 0, we know that this previous chunk will always be in the same row
	// as the one that has start_month.

	row_key := fmt.Sprintf("%s_%d", key, start_month/month_sec)
	query(start_month, start_month, "SELECT ts, data FROM metric WHERE key=? AND ts <= ? Limit 1", row_key, start)

	if start_month == end_month {
		// we need a selection of the row between startTs and endTs
		row_key = fmt.Sprintf("%s_%d", key, start_month/month_sec)
		query(start_month, start_month+1, "SELECT ts, data FROM metric WHERE key = ? AND ts > ? AND ts < ? ORDER BY ts ASC", row_key, start, end)
	} else {
		// get row_keys for each row we need to query.
		for month := start_month; month <= end_month; month += month_sec {
			row_key = fmt.Sprintf("%s_%d", key, month/month_sec)
			if month == start_month {
				// we want from startTs to the end of the row.
				query(month, month+1, "SELECT ts, data FROM metric WHERE key = ? AND ts >= ? ORDER BY ts ASC", row_key, start+1)
			} else if month == end_month {
				// we want from start of the row till the endTs.
				query(month, month, "SELECT ts, data FROM metric WHERE key = ? AND ts <= ? ORDER BY ts ASC", row_key, end-1)
			} else {
				// we want all columns
				query(month, month, "SELECT ts, data FROM metric WHERE key = ? ORDER BY ts ASC", row_key)
			}
		}
	}
	outcomes := make([]outcome, 0, numQueries)

	seen := 0
	for o := range results {
		seen += 1
		outcomes = append(outcomes, o)
		if seen == numQueries {
			close(results)
			cassGetDuration.Value(time.Now().Sub(pre))
			break
		}
	}
	pre = time.Now()
	// we have all of the results, but they could have arrived in any order.
	sort.Sort(asc(outcomes))

	var b []byte
	var ts int
	for _, outcome := range outcomes {
		chunks := int64(0)
		for outcome.i.Scan(&ts, &b) {
			chunks += 1
			chunkSizeAtLoad.Value(int64(len(b)))
			if len(b) < 2 {
				err := errors.New("unpossibly small chunk in cassandra")
				log.Error(3, err.Error())
				return iters, err
			}
			if Format(b[0]) != FormatStandardGoTsz {
				err := errors.New("unrecognized chunk format in cassandra")
				log.Error(3, err.Error())
				return iters, err
			}
			iter, err := tsz.NewIterator(b[1:])
			if err != nil {
				log.Error(3, "failed to unpack cassandra payload. %s", err)
				return iters, err
			}
			iters = append(iters, NewIter(iter, "cassandra month=%d t0=%d", outcome.month, ts))
		}
		cassChunksPerRow.Value(chunks)
		err := outcome.i.Close()
		if err != nil {
			log.Error(3, "cassandra query error. %s", err)
		}
	}
	cassToIterDuration.Value(time.Now().Sub(pre))
	cassRowsPerResponse.Value(int64(len(outcomes)))
	log.Debug("searchCassandra(): %d outcomes (queries), %d total iters", len(outcomes), len(iters))
	return iters, nil
}

func (c *cassandraStore) Stop() {
	c.session.Close()
}
<commit_msg>prevent processReadQueue blocking while we're not ready for results yet<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/dgryski/go-tsz"
	"github.com/gocql/gocql"
	"github.com/grafana/grafana/pkg/log"
	"github.com/raintank/met"
)

// write aggregated data to cassandra.

const month_sec = 60 * 60 * 24 * 28

const keyspace_schema = `CREATE KEYSPACE IF NOT EXISTS raintank WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}  AND durable_writes = true`
const table_schema = `CREATE TABLE IF NOT EXISTS raintank.metric (
    key ascii,
    ts int,
    data blob,
    PRIMARY KEY (key, ts)
) WITH COMPACT STORAGE
    AND CLUSTERING ORDER BY (ts DESC)
    AND compaction = {'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy'}
    AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'}
    AND read_repair_chance = 0.0
    AND dclocal_read_repair_chance = 0`

/*
https://godoc.org/github.com/gocql/gocql#Session
Session is the interface used by users to interact with the database.
It's safe for concurrent use by multiple goroutines and a typical usage scenario is to have one global session
object to interact with the whole Cassandra cluster.
*/

type cassandraStore struct {
	session          *gocql.Session
	writeQueues      []chan *ChunkWriteRequest
	readQueue        chan *ChunkReadRequest
	writeQueueMeters []met.Meter
}

func NewCassandraStore(stats met.Backend, addrs, consistency string, timeout, readers, writers int) (*cassandraStore, error) {
	cluster := gocql.NewCluster(strings.Split(addrs, ",")...)
	cluster.Consistency = gocql.ParseConsistency(consistency)
	cluster.Timeout = time.Duration(timeout) * time.Millisecond
	cluster.NumConns = writers
	var err error
	tmpSession, err := cluster.CreateSession()
	if err != nil {
		return nil, err
	}
	// ensure the keyspace and table exist.
	err = tmpSession.Query(keyspace_schema).Exec()
	if err != nil {
		return nil, err
	}
	err = tmpSession.Query(table_schema).Exec()
	if err != nil {
		return nil, err
	}
	tmpSession.Close()
	cluster.Keyspace = "raintank"
	session, err := cluster.CreateSession()
	if err != nil {
		return nil, err
	}
	c := &cassandraStore{
		session:          session,
		writeQueues:      make([]chan *ChunkWriteRequest, writers),
		readQueue:        make(chan *ChunkReadRequest, *cassandraReadQueueSize),
		writeQueueMeters: make([]met.Meter, writers),
	}

	for i := 0; i < *cassandraWriteConcurrency; i++ {
		c.writeQueues[i] = make(chan *ChunkWriteRequest, *cassandraWriteQueueSize)
		c.writeQueueMeters[i] = stats.NewMeter(fmt.Sprintf("cassandra.write_queue.%d.items", i+1), 0)
		go c.processWriteQueue(c.writeQueues[i], c.writeQueueMeters[i])
	}

	for i := 0; i < *cassandraReadConcurrency; i++ {
		go c.processReadQueue()
	}

	return c, err
}

func (c *cassandraStore) Add(cwr *ChunkWriteRequest) {
	sum := 0
	for _, char := range cwr.key {
		sum += int(char)
	}
	which := sum % *cassandraWriteConcurrency
	c.writeQueueMeters[which].Value(int64(len(c.writeQueues[which])))
	c.writeQueues[which] <- cwr
	c.writeQueueMeters[which].Value(int64(len(c.writeQueues[which])))
}

/* process writeQueue.
 */
func (c *cassandraStore) processWriteQueue(queue chan *ChunkWriteRequest, meter met.Meter) {
	tick := time.Tick(time.Duration(1) * time.Second)
	for {
		select {
		case <-tick:
			meter.Value(int64(len(queue)))
		case cwr := <-queue:
			meter.Value(int64(len(queue)))
			log.Debug("starting to save %s:%d %v", cwr.key, cwr.chunk.T0, cwr.chunk)
			//log how long the chunk waited in the queue before we attempted to save to cassandra
			cassBlockDuration.Value(time.Now().Sub(cwr.timestamp))

			data := cwr.chunk.Series.Bytes()
			chunkSizeAtSave.Value(int64(len(data)))
			version := FormatStandardGoTsz
			buf := new(bytes.Buffer)
			err := binary.Write(buf, binary.LittleEndian, uint8(version))
			if err != nil {
				// TODO
			}
			_, err = buf.Write(data)
			if err != nil {
				// TODO
			}
			success := false
			attempts := 0
			for !success {
				err := c.insertChunk(cwr.key, cwr.chunk.T0, buf.Bytes(), int(cwr.ttl))
				if err == nil {
					success = true
					cwr.chunk.Saved = true
					SendPersistMessage(cwr.key, cwr.chunk.T0)
					log.Debug("save complete. %s:%d %v", cwr.key, cwr.chunk.T0, cwr.chunk)
					chunkSaveOk.Inc(1)
				} else {
					if (attempts % 20) == 0 {
						log.Warn("failed to save chunk to cassandra after %d attempts. %v, %s", attempts+1, cwr.chunk, err)
					}
					chunkSaveFail.Inc(1)
					sleepTime := 100 * attempts
					if sleepTime > 2000 {
						sleepTime = 2000
					}
					time.Sleep(time.Duration(sleepTime) * time.Millisecond)
					attempts++
				}
			}
		}
	}
}

// Insert Chunks into Cassandra.
//
// key: is the metric_id
// ts: is the start of the aggregated time range.
// data: is the payload as bytes.
func (c *cassandraStore) insertChunk(key string, t0 uint32, data []byte, ttl int) error {
	// for unit tests
	if c.session == nil {
		return nil
	}
	query := fmt.Sprintf("INSERT INTO metric (key, ts, data) values(?,?,?) USING TTL %d", ttl)
	row_key := fmt.Sprintf("%s_%d", key, t0/month_sec) // "month number" based on unix timestamp (rounded down)
	pre := time.Now()
	ret := c.session.Query(query, row_key, t0, data).Exec()
	cassPutDuration.Value(time.Now().Sub(pre))
	return ret
}

type outcome struct {
	month   uint32
	sortKey uint32
	i       *gocql.Iter
}
type asc []outcome

func (o asc) Len() int           { return len(o) }
func (o asc) Swap(i, j int)      { o[i], o[j] = o[j], o[i] }
func (o asc) Less(i, j int) bool { return o[i].sortKey < o[j].sortKey }

func (c *cassandraStore) processReadQueue() {
	for crr := range c.readQueue {
		crr.out <- outcome{crr.month, crr.sortKey, c.session.Query(crr.q, crr.p...).Iter()}
	}
}

// Basic search of cassandra.
// start inclusive, end exclusive
func (c *cassandraStore) Search(key string, start, end uint32) ([]Iter, error) {
	iters := make([]Iter, 0)
	if start > end {
		return iters, fmt.Errorf("start must be before end.")
	}

	crrs := make([]*ChunkReadRequest, 0)

	query := func(month, sortKey uint32, q string, p ...interface{}) {
		crrs = append(crrs, &ChunkReadRequest{month, sortKey, q, p, nil})
	}

	start_month := start - (start % month_sec)       // starting row has to be at, or before, requested start
	end_month := (end - 1) - ((end - 1) % month_sec) // ending row has to be include the last point we might need

	pre := time.Now()

	// unfortunately in the database we only have the t0's of all chunks.
	// this means we can easily make sure to include the correct last chunk (just query for a t0 < end, the last chunk will contain the last needed data)
	// but it becomes hard to find which should be the first chunk to include. we can't just query for start <= t0 because than will miss some data at the beginning
	// we can't assume we know the chunkSpan so we can't just calculate the t0 >= start - <some-predefined-number> because chunkSpans may change over time.
	// we effectively need all chunks with a t0 > start, as well as the last chunk with a t0 <= start.
	// since we make sure that you can only use chunkSpans so that month_sec % chunkSpan == 0, we know that this previous chunk will always be in the same row
	// as the one that has start_month.

	row_key := fmt.Sprintf("%s_%d", key, start_month/month_sec)

	query(start_month, start_month, "SELECT ts, data FROM metric WHERE key=? AND ts <= ? Limit 1", row_key, start)

	if start_month == end_month {
		// we need a selection of the row between startTs and endTs
		row_key = fmt.Sprintf("%s_%d", key, start_month/month_sec)
		query(start_month, start_month+1, "SELECT ts, data FROM metric WHERE key = ? AND ts > ? AND ts < ? ORDER BY ts ASC", row_key, start, end)
	} else {
		// get row_keys for each row we need to query.
		for month := start_month; month <= end_month; month += month_sec {
			row_key = fmt.Sprintf("%s_%d", key, month/month_sec)
			if month == start_month {
				// we want from startTs to the end of the row.
				query(month, month+1, "SELECT ts, data FROM metric WHERE key = ? AND ts >= ? ORDER BY ts ASC", row_key, start+1)
			} else if month == end_month {
				// we want from start of the row till the endTs.
				query(month, month, "SELECT ts, data FROM metric WHERE key = ? AND ts <= ? ORDER BY ts ASC", row_key, end-1)
			} else {
				// we want all columns
				query(month, month, "SELECT ts, data FROM metric WHERE key = ? ORDER BY ts ASC", row_key)
			}
		}
	}
	numQueries := len(crrs)
	results := make(chan outcome, numQueries)
	for i := range crrs {
		crrs[i].out = results
		c.readQueue <- crrs[i]
	}
	outcomes := make([]outcome, 0, numQueries)

	seen := 0
	for o := range results {
		seen += 1
		outcomes = append(outcomes, o)
		if seen == numQueries {
			close(results)
			cassGetDuration.Value(time.Now().Sub(pre))
			break
		}
	}
	pre = time.Now()
	// we have all of the results, but they could have arrived in any order.
	sort.Sort(asc(outcomes))

	var b []byte
	var ts int
	for _, outcome := range outcomes {
		chunks := int64(0)
		for outcome.i.Scan(&ts, &b) {
			chunks += 1
			chunkSizeAtLoad.Value(int64(len(b)))
			if len(b) < 2 {
				err := errors.New("unpossibly small chunk in cassandra")
				log.Error(3, err.Error())
				return iters, err
			}
			if Format(b[0]) != FormatStandardGoTsz {
				err := errors.New("unrecognized chunk format in cassandra")
				log.Error(3, err.Error())
				return iters, err
			}
			iter, err := tsz.NewIterator(b[1:])
			if err != nil {
				log.Error(3, "failed to unpack cassandra payload. %s", err)
				return iters, err
			}
			iters = append(iters, NewIter(iter, "cassandra month=%d t0=%d", outcome.month, ts))
		}
		cassChunksPerRow.Value(chunks)
		err := outcome.i.Close()
		if err != nil {
			log.Error(3, "cassandra query error. %s", err)
		}
	}
	cassToIterDuration.Value(time.Now().Sub(pre))
	cassRowsPerResponse.Value(int64(len(outcomes)))
	log.Debug("searchCassandra(): %d outcomes (queries), %d total iters", len(outcomes), len(iters))
	return iters, nil
}

func (c *cassandraStore) Stop() {
	c.session.Close()
}
<|endoftext|>


<commit_before>package bench

import (
	"reflect"
	"testing"
)

// Just getting the reflect.Value of a string.
func Benchmark_ReflectGetValueOfString(b *testing.B) {
	var slot string
	for i := 0; i < b.N; i++ {
		reflect.ValueOf(slot)
	}
	_ = slot
}

// This is fascinating because it's actually much faster.
// Like 5ns compared to 60 for the direct value.
func Benchmark_ReflectGetValueOfStringRef(b *testing.B) {
	var slot string
	var slotAddr_rv reflect.Value
	for i := 0; i < b.N; i++ {
		slotAddr_rv = reflect.ValueOf(&slot)
	}
	_ = slot
	_ = slotAddr_rv
}

// Getting a reflect.Value of a string's address, then `Elem()`'ing back to the string type.
// This is the hop you need to do to have an *addressable* value that you can set.
// Like 8ns compared to the 9ns (appears `Elem()` adds ~3.5ns).
func Benchmark_ReflectGetValueOfStringRefElem(b *testing.B) {
	var slot string
	var slot_rav reflect.Value
	for i := 0; i < b.N; i++ {
		_ = reflect.ValueOf(&slot).Elem()
	}
	_ = slot
	_ = slot_rav
}

// About 24ns, by the same rulers as the others.
func Benchmark_ReflectSetValue(b *testing.B) {
	var slot string
	slot_rav := reflect.ValueOf(&slot).Elem()
	var val = "x"
	val_rv := reflect.ValueOf(val)
	for i := 0; i < b.N; i++ {
		slot_rav.Set(val_rv)
	}
}

// Just setting something through an address, full types.
// Very fast (obviously): less than a single nano.
func Benchmark_DirectSetValue(b *testing.B) {
	var slot string
	slotAddr_v := &slot
	var val = "x"
	for i := 0; i < b.N; i++ {
		*slotAddr_v = val
	}
}

// Fit the address of a primitive into an `interface{}`, then type-switch
// it back to a primitive so we can directly set it.
// Still very fast: 2ns.
// Context: looking 4/5x faster than the equivalent ops with reflect
// (but that's maybe less of a margin than I might've expected).
func Benchmark_DirectInterfacedSetValue(b *testing.B) {
	var slot string
	var slotAddr_v interface{} = &slot
	var val = "x"
	for i := 0; i < b.N; i++ {
		switch v2 := slotAddr_v.(type) {
		case *interface{}:
			// sigh
		case *string:
			*v2 = val
		}
	}
}

// Use a func to get that `interface{}` that's a pointer.
// This is emulating the fastest path we can do with an atlas with user-written functions.
// Still fast: 3ns.
func Benchmark_FuncedDirectInterfacedSetValue(b *testing.B) {
	var slot string
	addrFunc := func() interface{} { return &slot }
	var val = "x"
	var slotAddr_v interface{}
	for i := 0; i < b.N; i++ {
		slotAddr_v = addrFunc()
		switch v2 := slotAddr_v.(type) {
		case *interface{}:
			// sigh
		case *string:
			*v2 = val
		}
	}
}
<commit_msg>Making observations with and without GC, and with and without benchmem.<commit_after>package bench

import (
	"reflect"
	"testing"
)

// Just getting the reflect.Value of a string.
// 45ns.
// Or, 60ns with GC enabled.  Yes, quite a difference.
func Benchmark_ReflectGetValueOfString(b *testing.B) {
	var slot string
	for i := 0; i < b.N; i++ {
		reflect.ValueOf(slot)
	}
	_ = slot
}

// This is fascinating because it's actually much faster.
// Like 5ns compared to 60 for the direct value.
// GC: no impact.  0 allocs (!)
func Benchmark_ReflectGetValueOfStringRef(b *testing.B) {
	var slot string
	var slotAddr_rv reflect.Value
	for i := 0; i < b.N; i++ {
		slotAddr_rv = reflect.ValueOf(&slot)
	}
	_ = slot
	_ = slotAddr_rv
}

// Getting a reflect.Value of a string's address, then `Elem()`'ing back to the string type.
// This is the hop you need to do to have an *addressable* value that you can set.
// Like 8ns compared to the 9ns (appears `Elem()` adds ~3.5ns).
// GC: no impact.  0 allocs (!)
func Benchmark_ReflectGetValueOfStringRefElem(b *testing.B) {
	var slot string
	var slot_rav reflect.Value
	for i := 0; i < b.N; i++ {
		_ = reflect.ValueOf(&slot).Elem()
	}
	_ = slot
	_ = slot_rav
}

// About 24ns, by the same rulers as the others.
// GC: no impact.  0 allocs (!)
func Benchmark_ReflectSetValue(b *testing.B) {
	var slot string
	slot_rav := reflect.ValueOf(&slot).Elem()
	var val = "x"
	val_rv := reflect.ValueOf(val)
	for i := 0; i < b.N; i++ {
		slot_rav.Set(val_rv)
	}
}

// Just setting something through an address, full types.
// Very fast (obviously): less than a single nano.
// GC: no impact.  0 allocs (!)
func Benchmark_DirectSetValue(b *testing.B) {
	var slot string
	slotAddr_v := &slot
	var val = "x"
	for i := 0; i < b.N; i++ {
		*slotAddr_v = val
	}
}

// Fit the address of a primitive into an `interface{}`, then type-switch
// it back to a primitive so we can directly set it.
// Still very fast: 2ns.
// GC: no impact.  0 allocs (!)
// Context: looking 4/5x faster than the equivalent ops with reflect
// (but that's maybe less of a margin than I might've expected).
func Benchmark_DirectInterfacedSetValue(b *testing.B) {
	var slot string
	var slotAddr_v interface{} = &slot
	var val = "x"
	for i := 0; i < b.N; i++ {
		switch v2 := slotAddr_v.(type) {
		case *interface{}:
			// sigh
		case *string:
			*v2 = val
		}
	}
}

// Use a func to get that `interface{}` that's a pointer.
// This is emulating the fastest path we can do with an atlas with user-written functions.
// Still fast: 3ns.
// GC: no impact.  0 allocs (!)
func Benchmark_FuncedDirectInterfacedSetValue(b *testing.B) {
	var slot string
	addrFunc := func() interface{} { return &slot }
	var val = "x"
	var slotAddr_v interface{}
	for i := 0; i < b.N; i++ {
		slotAddr_v = addrFunc()
		switch v2 := slotAddr_v.(type) {
		case *interface{}:
			// sigh
		case *string:
			*v2 = val
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"image"
	"image/color"
	"image/draw"
	"math"
	"strings"
	"time"

	"github.com/davecgh/go-spew/spew"
	"github.com/lucasb-eyer/go-colorful"
	"github.com/ninjasphere/gestic-tools/go-gestic-sdk"
	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/channels"
	"github.com/ninjasphere/go-ninja/model"
	"github.com/ninjasphere/sphere-go-led-controller/fonts/clock"
	"github.com/ninjasphere/sphere-go-led-controller/ui"
	"github.com/ninjasphere/sphere-go-led-controller/util"
	"github.com/ninjasphere/usvc-lib/config"
)

var enableACPane = config.Bool(true, "ac.enable")
var tempAdjustInterval = config.Duration(time.Millisecond*300, "ac.airwheel.interval")
var airWheelReset = config.Duration(time.Millisecond*500, "ac.airwheel.reset")
var tapTimeout = config.Duration(time.Millisecond*500, "ac.tap.timeout")

var (
	red, _   = colorful.Hex("#fd9720")
	blue, _  = colorful.Hex("#45aff9")
	green, _ = colorful.Hex("#45f96b")
)

type ACPane struct {
	mode        string
	images      map[string]util.Image
	targetTemp  int
	currentTemp int
	controlled  bool

	thermostat *ninja.ServiceClient
	acstat     *ninja.ServiceClient

	lastTempAdjust   time.Time
	lastAirWheelTime time.Time
	lastAirWheel     *uint8
	countSinceLast   int
	ignoringTap      bool
	ignoreTapTimer   *time.Timer
}

func NewACPane(conn *ninja.Connection) *ACPane {

	pane := &ACPane{
		mode:        "off",
		controlled:  false,
		targetTemp:  666,
		currentTemp: 666,
		images: map[string]util.Image{
			"off":  util.LoadImage(util.ResolveImagePath("fan.gif")),
			"fan":  util.LoadImage(util.ResolveImagePath("fan-on.gif")),
			"cool": util.LoadImage(util.ResolveImagePath("fan-on-blue.gif")),
			"heat": util.LoadImage(util.ResolveImagePath("fan-on-red.gif")),
		},
	}

	pane.ignoreTapTimer = time.AfterFunc(0, func() {
		pane.ignoringTap = false
	})

	listening := make(map[string]bool)

	onState := func(protocol, event string, cb func(params *json.RawMessage)) {
		ui.GetChannelServicesContinuous("aircon", protocol, func(thing *model.Thing) bool {
			return true
		}, func(devices []*ninja.ServiceClient, err error) {
			if err != nil {
				log.Infof("Failed to update %s device: %s", protocol, err)
			} else {
				log.Infof("Got %d %s devices", len(devices), protocol)

				for _, device := range devices {

					log.Debugf("Checking %s device %s", protocol, device.Topic)

					if _, ok := listening[device.Topic]; !ok {

						// New device
						log.Infof("Got new %s device: %s", protocol, device.Topic)

						if protocol == "thermostat" {
							pane.thermostat = device
						}

						if protocol == "acstat" {
							pane.acstat = device
						}

						if protocol == "demand" {
							go func() {
								for {
									time.Sleep(time.Second)

									var state StateChangeNotification
									err := device.Call("getControlState", nil, &state, time.Second*10)
									if err != nil {
										log.Errorf("Failed to fetch state from demand channel: %s", err)

										if strings.Contains(err.Error(), "timed out") {
											continue
										} else {
											break
										}

									}

									spew.Dump("Got demand state", state)

									pane.controlled = state.State == "STATE_ACTIVE"
								}
							}()
						}

						listening[device.Topic] = true

						device.OnEvent(event, func(params *json.RawMessage, values map[string]string) bool {
							cb(params)

							return true
						})
					}
				}
			}
		})
	}

	onState("temperature", "state", func(params *json.RawMessage) {
		var temp float64
		err := json.Unmarshal(*params, &temp)
		if err != nil {
			log.Infof("Failed to unmarshal temp from %s error:%s", *params, err)
		}

		pane.currentTemp = int(temp)

		log.Infof("Got the temp %d", pane.currentTemp)
	})

	onState("thermostat", "state", func(params *json.RawMessage) {
		var temp float64
		err := json.Unmarshal(*params, &temp)
		if err != nil {
			log.Infof("Failed to unmarshal thermostat from %s error:%s", *params, err)
		}

		pane.targetTemp = int(temp)

		log.Infof("Got the thermostat %d", pane.targetTemp)
	})

	onState("acstat", "state", func(params *json.RawMessage) {
		var state channels.ACState
		err := json.Unmarshal(*params, &state)
		if err != nil {
			log.Infof("Failed to unmarshal acstat from %s error:%s", *params, err)
		}

		pane.mode = *state.Mode

		log.Infof("Got the ac mode %d", pane.mode)
	})

	onState("demand", "controlstate", func(params *json.RawMessage) {

		spew.Dump("demand/controlstate", params)

		var state StateChangeNotification
		err := json.Unmarshal(*params, &state)
		if err != nil {
			log.Infof("Failed to unmarshal demandstat state from %s error:%s", *params, err)
		}

		pane.controlled = state.State == "STATE_ACTIVE"

		log.Infof("Got the demandstat state %d", pane.mode)
	})

	go ui.StartSearchTasks(conn)

	return pane
}

type StateChangeNotification struct {
	State string `json:"state"` // the current state of the controller
}

func (p *ACPane) KeepAwake() bool {
	return true
}

func (p *ACPane) Locked() bool {
	return true
}

func (p *ACPane) IsEnabled() bool {
	return enableACPane
}

func (p *ACPane) Gesture(gesture *gestic.GestureMessage) {

	if p.acstat != nil && !p.ignoringTap && gesture.Tap.Active() {
		log.Infof("AC tap!")

		p.ignoringTap = true
		p.ignoreTapTimer.Reset(tapTimeout)

		switch p.mode {
		case "off":
			p.mode = "fan"
		case "fan":
			p.mode = "cool"
		case "cool":
			p.mode = "heat"
		case "heat":
			p.mode = "off"
		}

		p.acstat.Call("set", channels.ACState{
			Mode: &p.mode,
		}, nil, 0)

	}

	if p.thermostat != nil && p.targetTemp != 666 && p.lastAirWheel == nil || gesture.AirWheel.Counter != int(*p.lastAirWheel) {

		if time.Since(p.lastAirWheelTime) > airWheelReset {
			p.lastAirWheel = nil
		}

		if p.countSinceLast > gesture.AirWheel.CountSinceLast {
			p.lastAirWheel = nil
		}

		p.countSinceLast = gesture.AirWheel.CountSinceLast

		p.lastAirWheelTime = time.Now()

		log.Debugf("Airwheel: %d", gesture.AirWheel.Counter)

		if p.lastAirWheel != nil {
			offset := int(gesture.AirWheel.Counter) - int(*p.lastAirWheel)

			if offset > 30 {
				offset -= 255
			}

			if offset < -30 {
				offset += 255
			}

			log.Debugf("Airwheel New: %d Offset: %d Last: %d", gesture.AirWheel.Counter, offset, *p.lastAirWheel)

			log.Debugf("Current temp %f", p.targetTemp)

			log.Debugf("Temp offset %f", float64(offset)/255.0)

			if time.Since(p.lastTempAdjust) < tempAdjustInterval {
				log.Debugf("Temp rate limited")
			} else {
				p.lastTempAdjust = time.Now()
				log.Debugf("Temp NOT rate limited")
				if offset > 0 {
					p.targetTemp += 1
				} else {
					p.targetTemp -= 1
				}

				p.thermostat.Call("set", p.targetTemp, nil, 0)

			}

		}

		val := uint8(gesture.AirWheel.Counter)
		p.lastAirWheel = &val
	}
}

func (p *ACPane) Render() (*image.RGBA, error) {

	img := image.NewRGBA(image.Rect(0, 0, 16, 16))

	drawText := func(text string, col color.Color, x, y int) {
		clock.Font.DrawString(img, x, y, text, col)
	}

	draw.Draw(img, img.Bounds(), p.images[p.mode].GetNextFrame(), img.Bounds().Min, draw.Src)

	targetCol := green
	if p.controlled {
		//drawText("$", green, 0, 11)
		h, s, v := targetCol.Hsv()
		targetCol = colorful.Hsv(sinTime(h), s, v)
	}

	if p.currentTemp != 666 {

		if p.currentTemp > p.targetTemp {
			drawText(fmt.Sprintf("%02d", p.currentTemp), red, 0, 11)
		} else if p.currentTemp < p.targetTemp {
			drawText(fmt.Sprintf("%02d", p.currentTemp), blue, 0, 11)
		}

	}

	if p.targetTemp != 666 {
		drawText(fmt.Sprintf("%02d", p.targetTemp), targetCol, 9, 11)
	}

	return img, nil
}

func (p *ACPane) IsDirty() bool {
	return true
}

func sinTime(x float64) float64 {
	t := float64(time.Now().Nanosecond()/int(time.Millisecond)) / 1000.0
	t = math.Sin((t - 0.5) * (2 * math.Pi))
	t = (t + 1) / 2
	return x * t
}
<commit_msg>"STATE_ACTIVE" -> "ACTIVE"<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"image"
	"image/color"
	"image/draw"
	"math"
	"strings"
	"time"

	"github.com/davecgh/go-spew/spew"
	"github.com/lucasb-eyer/go-colorful"
	"github.com/ninjasphere/gestic-tools/go-gestic-sdk"
	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/channels"
	"github.com/ninjasphere/go-ninja/model"
	"github.com/ninjasphere/sphere-go-led-controller/fonts/clock"
	"github.com/ninjasphere/sphere-go-led-controller/ui"
	"github.com/ninjasphere/sphere-go-led-controller/util"
	"github.com/ninjasphere/usvc-lib/config"
)

var enableACPane = config.Bool(true, "ac.enable")
var tempAdjustInterval = config.Duration(time.Millisecond*300, "ac.airwheel.interval")
var airWheelReset = config.Duration(time.Millisecond*500, "ac.airwheel.reset")
var tapTimeout = config.Duration(time.Millisecond*500, "ac.tap.timeout")

var (
	red, _   = colorful.Hex("#fd9720")
	blue, _  = colorful.Hex("#45aff9")
	green, _ = colorful.Hex("#45f96b")
)

type ACPane struct {
	mode        string
	images      map[string]util.Image
	targetTemp  int
	currentTemp int
	controlled  bool

	thermostat *ninja.ServiceClient
	acstat     *ninja.ServiceClient

	lastTempAdjust   time.Time
	lastAirWheelTime time.Time
	lastAirWheel     *uint8
	countSinceLast   int
	ignoringTap      bool
	ignoreTapTimer   *time.Timer
}

func NewACPane(conn *ninja.Connection) *ACPane {

	pane := &ACPane{
		mode:        "off",
		controlled:  false,
		targetTemp:  666,
		currentTemp: 666,
		images: map[string]util.Image{
			"off":  util.LoadImage(util.ResolveImagePath("fan.gif")),
			"fan":  util.LoadImage(util.ResolveImagePath("fan-on.gif")),
			"cool": util.LoadImage(util.ResolveImagePath("fan-on-blue.gif")),
			"heat": util.LoadImage(util.ResolveImagePath("fan-on-red.gif")),
		},
	}

	pane.ignoreTapTimer = time.AfterFunc(0, func() {
		pane.ignoringTap = false
	})

	listening := make(map[string]bool)

	onState := func(protocol, event string, cb func(params *json.RawMessage)) {
		ui.GetChannelServicesContinuous("aircon", protocol, func(thing *model.Thing) bool {
			return true
		}, func(devices []*ninja.ServiceClient, err error) {
			if err != nil {
				log.Infof("Failed to update %s device: %s", protocol, err)
			} else {
				log.Infof("Got %d %s devices", len(devices), protocol)

				for _, device := range devices {

					log.Debugf("Checking %s device %s", protocol, device.Topic)

					if _, ok := listening[device.Topic]; !ok {

						// New device
						log.Infof("Got new %s device: %s", protocol, device.Topic)

						if protocol == "thermostat" {
							pane.thermostat = device
						}

						if protocol == "acstat" {
							pane.acstat = device
						}

						if protocol == "demand" {
							go func() {
								for {
									time.Sleep(time.Second)

									var state StateChangeNotification
									err := device.Call("getControlState", nil, &state, time.Second*10)
									if err != nil {
										log.Errorf("Failed to fetch state from demand channel: %s", err)

										if strings.Contains(err.Error(), "timed out") {
											continue
										} else {
											break
										}

									}

									spew.Dump("Got demand state", state)

									pane.controlled = (state.State == "ACTIVE")
								}
							}()
						}

						listening[device.Topic] = true

						device.OnEvent(event, func(params *json.RawMessage, values map[string]string) bool {
							cb(params)

							return true
						})
					}
				}
			}
		})
	}

	onState("temperature", "state", func(params *json.RawMessage) {
		var temp float64
		err := json.Unmarshal(*params, &temp)
		if err != nil {
			log.Infof("Failed to unmarshal temp from %s error:%s", *params, err)
		}

		pane.currentTemp = int(temp)

		log.Infof("Got the temp %d", pane.currentTemp)
	})

	onState("thermostat", "state", func(params *json.RawMessage) {
		var temp float64
		err := json.Unmarshal(*params, &temp)
		if err != nil {
			log.Infof("Failed to unmarshal thermostat from %s error:%s", *params, err)
		}

		pane.targetTemp = int(temp)

		log.Infof("Got the thermostat %d", pane.targetTemp)
	})

	onState("acstat", "state", func(params *json.RawMessage) {
		var state channels.ACState
		err := json.Unmarshal(*params, &state)
		if err != nil {
			log.Infof("Failed to unmarshal acstat from %s error:%s", *params, err)
		}

		pane.mode = *state.Mode

		log.Infof("Got the ac mode %d", pane.mode)
	})

	onState("demand", "controlstate", func(params *json.RawMessage) {

		spew.Dump("demand/controlstate", params)

		var state StateChangeNotification
		err := json.Unmarshal(*params, &state)
		if err != nil {
			log.Infof("Failed to unmarshal demandstat state from %s error:%s", *params, err)
		}

		pane.controlled = state.State == "ACTIVE"

		log.Infof("Got the demandstat state %d", pane.mode)
	})

	go ui.StartSearchTasks(conn)

	return pane
}

type StateChangeNotification struct {
	State string `json:"state"` // the current state of the controller
}

func (p *ACPane) KeepAwake() bool {
	return true
}

func (p *ACPane) Locked() bool {
	return true
}

func (p *ACPane) IsEnabled() bool {
	return enableACPane
}

func (p *ACPane) Gesture(gesture *gestic.GestureMessage) {

	if p.acstat != nil && !p.ignoringTap && gesture.Tap.Active() {
		log.Infof("AC tap!")

		p.ignoringTap = true
		p.ignoreTapTimer.Reset(tapTimeout)

		switch p.mode {
		case "off":
			p.mode = "fan"
		case "fan":
			p.mode = "cool"
		case "cool":
			p.mode = "heat"
		case "heat":
			p.mode = "off"
		}

		p.acstat.Call("set", channels.ACState{
			Mode: &p.mode,
		}, nil, 0)

	}

	if p.thermostat != nil && p.targetTemp != 666 && p.lastAirWheel == nil || gesture.AirWheel.Counter != int(*p.lastAirWheel) {

		if time.Since(p.lastAirWheelTime) > airWheelReset {
			p.lastAirWheel = nil
		}

		if p.countSinceLast > gesture.AirWheel.CountSinceLast {
			p.lastAirWheel = nil
		}

		p.countSinceLast = gesture.AirWheel.CountSinceLast

		p.lastAirWheelTime = time.Now()

		log.Debugf("Airwheel: %d", gesture.AirWheel.Counter)

		if p.lastAirWheel != nil {
			offset := int(gesture.AirWheel.Counter) - int(*p.lastAirWheel)

			if offset > 30 {
				offset -= 255
			}

			if offset < -30 {
				offset += 255
			}

			log.Debugf("Airwheel New: %d Offset: %d Last: %d", gesture.AirWheel.Counter, offset, *p.lastAirWheel)

			log.Debugf("Current temp %f", p.targetTemp)

			log.Debugf("Temp offset %f", float64(offset)/255.0)

			if time.Since(p.lastTempAdjust) < tempAdjustInterval {
				log.Debugf("Temp rate limited")
			} else {
				p.lastTempAdjust = time.Now()
				log.Debugf("Temp NOT rate limited")
				if offset > 0 {
					p.targetTemp += 1
				} else {
					p.targetTemp -= 1
				}

				p.thermostat.Call("set", p.targetTemp, nil, 0)

			}

		}

		val := uint8(gesture.AirWheel.Counter)
		p.lastAirWheel = &val
	}
}

func (p *ACPane) Render() (*image.RGBA, error) {

	img := image.NewRGBA(image.Rect(0, 0, 16, 16))

	drawText := func(text string, col color.Color, x, y int) {
		clock.Font.DrawString(img, x, y, text, col)
	}

	draw.Draw(img, img.Bounds(), p.images[p.mode].GetNextFrame(), img.Bounds().Min, draw.Src)

	targetCol := green
	if p.controlled {
		//drawText("$", green, 0, 11)
		h, s, v := targetCol.Hsv()
		targetCol = colorful.Hsv(sinTime(h), s, v)
	}

	if p.currentTemp != 666 {

		if p.currentTemp > p.targetTemp {
			drawText(fmt.Sprintf("%02d", p.currentTemp), red, 0, 11)
		} else if p.currentTemp < p.targetTemp {
			drawText(fmt.Sprintf("%02d", p.currentTemp), blue, 0, 11)
		}

	}

	if p.targetTemp != 666 {
		drawText(fmt.Sprintf("%02d", p.targetTemp), targetCol, 9, 11)
	}

	return img, nil
}

func (p *ACPane) IsDirty() bool {
	return true
}

func sinTime(x float64) float64 {
	t := float64(time.Now().Nanosecond()/int(time.Millisecond)) / 1000.0
	t = math.Sin((t - 0.5) * (2 * math.Pi))
	t = (t + 1) / 2
	return x * t
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsS3BucketPolicy() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsS3BucketPolicyCreate,
		Read:   resourceAwsS3BucketPolicyRead,
		Update: resourceAwsS3BucketPolicyUpdate,
		Delete: resourceAwsS3BucketPolicyDelete,

		Schema: map[string]*schema.Schema{
			"address": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
		},
	}
}

func resourceAwsS3BucketPolicyCreate(d *schema.ResourceData, m interface{}) error {
	return nil
}

func resourceAwsS3BucketPolicyRead(d *schema.ResourceData, m interface{}) error {
	return nil
}

func resourceAwsS3BucketPolicyUpdate(d *schema.ResourceData, m interface{}) error {
	return nil
}

func resourceAwsS3BucketPolicyDelete(d *schema.ResourceData, m interface{}) error {
	return nil
}
<commit_msg>moved policy to it's own provider<commit_after>package aws

import (
	"fmt"
	"log"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/service/s3"
	"github.com/awslabs/aws-sdk-go/aws/awsutil"
)

func resourceAwsS3BucketPolicy() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsS3BucketPolicyPut,
		Update: resourceAwsS3BucketPolicyPut,
		Read:   resourceAwsS3BucketPolicyRead,
		Delete: resourceAwsS3BucketPolicyDelete,

		Schema: map[string]*schema.Schema{
			"bucket": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"policy": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
		},
	}
}

func resourceAwsS3BucketPolicyPut(d *schema.ResourceData, meta interface{}) error {

	s3conn := meta.(*AWSClient).s3conn
	bucket := d.Get("bucket").(string)
	policy := d.Get("policy").(string)

	resp, err := s3conn.PutBucketPolicy(
		&s3.PutBucketPolicyInput{
			Bucket: aws.String(bucket),
			Policy: aws.String(policy),
		})

	log.Printf("[DEBUG] S3 bucket policy set (response): %s", awsutil.StringValue(resp));

	if err != nil {
		return fmt.Errorf("Error adding policy to S3 bucket: %s", err)
	}

	return nil
}

func resourceAwsS3BucketPolicyRead(d *schema.ResourceData, m interface{}) error {
	return nil
}

func resourceAwsS3BucketPolicyUpdate(d *schema.ResourceData, m interface{}) error {
	return nil
}

func resourceAwsS3BucketPolicyDelete(d *schema.ResourceData, m interface{}) error {
	return nil
}
<|endoftext|>