<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/coreos/go-etcd/etcd"
	//	_ "github.com/davecgh/go-spew/spew"
	"github.com/jessevdk/go-flags"
	"github.com/kelseyhightower/envconfig"
	"github.com/rcrowley/go-metrics"
	"github.com/rcrowley/go-metrics/librato"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/http"
	//_ "net/http/pprof"
	"net/url"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	// DefaultInterval is the check check interval for a url specified in seconds
	DefaultInterval int = 15

	// DefaultSplay is the rand splay to introduce on checks so we don't flood
	DefaultSplay int = 5

	// DefaultAlertLevel is the level we alert at when no level is specified
	DefaultAlertLevel string = "WARN"
)

type Options struct {
	Etcd     string `short:"e" long:"etcd" description:"Etcd Server url. Comma separate multiple servers. default:'http://localhost:4001'"`
	Prefix   string `short:"p" long:"prefix" description:"the prefix to use in etcd for storing check urls. default:'urlmon'"`
	User     string `short:"u" long:"user" description:"librato user"`
	Token    string `short:"t" long:"token"  description:"librato token"`
	Port     int    `short:"P" long:"port"  description:"The port to start the status interface on. default: '0.0.0.0:9731'"`
	Sensu    string `short:"s" long:"sensu" description:"Sensu client address. default:'localhost:3030'"`
	Handlers string `short:"H" long:"handlers" description:"Sensu handler to use for alert message. comma separatemultiples. default:'hipchat'"`
}

type Check struct {
	Id           string
	URL          *url.URL
	Content      string
	Level        string
	ContentRegex *regexp.Regexp
	Interval     int
	Splay        int
	Registry     metrics.Registry
	shutdown     chan struct{}
}

type SensuEvent struct {
	Name     string   `json:"name"`
	Handlers []string `json:"handlers"`
	Output   string   `json:"output"`
	Status   int      `json:"status"`
}

type LazyResponse map[string]interface{}

var (
	Checks []Check
	opts   Options
)

func (c *Check) setupInterval() {
	if c.Interval <= 0 {
		c.Interval = DefaultInterval
	}

	if c.Splay <= 0 {
		c.Splay = DefaultSplay
	}

	// build up splay and add to interval
	c.Interval += rand.Intn(c.Splay)
}

// Validate takes a response body and runs the check validations on it.
// the response body will be closed in the process
func (c *Check) Validate(res *http.Response) (code int, status string) {
	// let checks set weather this is a crit or not, default is WARN which doesn't trigger sensu
	errorVal := 1
	if c.Level == "CRITICAL" {
		errorVal = 2
	}
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return errorVal, fmt.Sprintf("Error reading response: %s", err.Error())
	}

	if c.Content != "" && !strings.Contains(string(body), c.Content) {
		return errorVal, "Content match failed"
	}

	if c.ContentRegex != nil && !c.ContentRegex.Match(body) {
		return errorVal, "Content Regex did not match"
	}

	// everyting is groovy, return ok with status
	return 0, res.Status
}

// reportStatus reports if
func (c *Check) reportStatus(status int, msg string) {
	s := "Success:"
	if status != 0 {
		s = "Fail:"
	}
	log.Println(s, c.Id, c.URL.String(), msg)

	// for now handler is hardcode TODO: move it to default/param/env
	event := SensuEvent{
		Name:     c.Id,
		Output:   fmt.Sprintf("%s::%s", msg, c.URL),
		Handlers: strings.Split(opts.Handlers, ","),
		Status:   status,
	}

	event.send()
}

// Monitor sets up the monitoring loop
func (c *Check) Monitor() {
	// build the splay/random timing into the interval
	c.setupInterval()

	// setup the guage
	g := metrics.NewGauge()

	metrics.Register(fmt.Sprintf("urlmon.request.%s", c.Id), g)
	// loop for shutdown and spaly/sleep
	for {
		select {
		case <-c.shutdown:
			log.Printf("%s Shutdown", c.Id)
			return
		default:
			// we sleep first here so that on startup we don't overwhelm
			s := time.Duration(c.Interval) * time.Second
			time.Sleep(s)

			// time.Since reports in nanoseocnds
			start := time.Now()
			r, err := http.Get(c.URL.String())
			g.Update(int64(time.Since(start)))
			if err != nil {
				c.reportStatus(2, err.Error())
			} else {
				c.reportStatus(c.Validate(r))
			}
			r.Body.Close()
		}
	}
}

// easy way to signal the monitor to shutdown
func (c *Check) Shutdown() {
	c.shutdown <- struct{}{}
}

// String Adds stringer interface to the lazy response
func (r LazyResponse) String() (s string) {
	b, err := json.MarshalIndent(r, "", "  ")
	if err != nil {
		s = ""
		return
	}
	s = string(b)
	return
}

// send the sensu event to the local sensu port
func (e *SensuEvent) send() {
	conn, err := net.Dial("tcp", opts.Sensu)
	if err != nil {
		log.Println("Error connecting to sensu client socket.", err.Error())
		return
	}
	defer conn.Close()

	j, err := json.Marshal(e)
	if err != nil {
		log.Println("Error marshaling event data.", err.Error())
		return
	}
	fmt.Fprintf(conn, string(j))
}

// libratoMetrics starts the reporting metrics on this check to librato
func libratoMetrics() {
	host, err := os.Hostname()
	if err != nil {
		host = "localhost"
	}

	// start the metrics collector should be per-check
	log.Println("starting metrics")
	librato.Librato(metrics.DefaultRegistry,
		60*time.Second, // interval
		opts.User,      // account owner email address
		opts.Token,     // Librato API token
		// This should be per check I think
		host,             // source
		[]float64{95},    // precentiles to send
		time.Millisecond, // time unit
	)
}

// valueStr Converts a string to an int where a blank string or errvalue returns a 0
// This is a helper mostly for creating a Check where the interval or splay may be an empty string.
func valueStr(value string) int {
	v, err := strconv.Atoi(value)
	if err != nil {
		v = 0
	}
	return v
}

// createCheck populates check struct with data from etcd check
func createCheck(node *etcd.Node) (*Check, error) {
	// create a check
	c := Check{Id: path.Base(node.Key)}
	c.shutdown = make(chan struct{}, 1)
	for _, child := range node.Nodes {
		log.Printf("  - %s", path.Base(child.Key))
		switch strings.ToUpper(path.Base(child.Key)) {
		case "URL":
			u, err := url.Parse(child.Value)
			if err != nil {
				log.Printf("Url isn't valid for key: %s, %s", node.Key, err.Error())
				continue
			}
			c.URL = u
		case "CONTENT":
			c.Content = child.Value
		case "REGEX":
			r, err := regexp.Compile(child.Value)
			if err != nil {
				log.Fatalf("couldn't compile regex for Check: %s, %s", node.Key, err.Error())
			}
			c.ContentRegex = r
		case "LEVEL":
			l := strings.ToUpper(child.Value)
			if l == "" {
				l = DefaultAlertLevel
			}
			c.Level = l
		case "SPLAY":
			c.Splay = valueStr(child.Value)
		case "INTERVAL":
			c.Interval = valueStr(child.Value)
		}
	}

	// set defaults
	if c.Splay == 0 {
		c.Splay = DefaultSplay
	}
	if c.Interval == 0 {
		c.Interval = DefaultInterval
	}
	if c.Level == "" {
		c.Level = DefaultAlertLevel
	}
	if c.URL == nil {
		return &c, errors.New("No URL for check")
	}

	// BUG: Checks may be malformed in various ways, createCheck needs to implment more validations before returning the check
	return &c, nil
}

// loadChecks reads etcd popilates Checks, and starts their monitor
func loadChecks(client *etcd.Client) {
	resp, err := client.Get(fmt.Sprintf("%s/checks", opts.Prefix), true, true)
	if err != nil {
		log.Fatalf("Problem fetching Checks from etcd: %s", err)
	}

	for _, n := range Checks {
		// signal that monitor to shutdown
		log.Println("Shutting down ", n.Id)
		n.Shutdown()
	}

	// this clears the slice.
	Checks = nil

	for _, n := range resp.Node.Nodes {
		// these top level nodes should be a directory
		if !n.Dir {
			log.Printf("Error loading config %s is not a dir, skipping", n.Key)
			continue
		}

		log.Printf("Loading: %s: %s\n", n.Key, n.Value)
		c, err := createCheck(n)
		if err != nil {
			log.Println("Failed to create check, skipping: ", n.Key, err)
			continue
		}
		// start monitoring this check
		go c.Monitor()
		Checks = append(Checks, *c)
	}
}

// Create our Etcd Dir structure
func setupEtcd(client *etcd.Client) {
	for _, path := range []string{opts.Prefix, fmt.Sprintf("%s/checks", opts.Prefix)} {
		if _, err := client.Get(path, false, false); err != nil {
			log.Printf("Creating dir in etcd: %s ", path)
			if _, err := client.CreateDir(path, 0); err != nil {
				log.Fatal("Couldn't create etcd dir: ", err)
			}
		}
	}
}

func statusHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprint(w, LazyResponse{"checkdata": Checks})
}

// cliDefaults sets the options to default vaules if env or switches haven't specified them
func cliDefaults() {
	if opts.Etcd == "" {
		opts.Etcd = "http://localhost:4001"
	}

	if opts.Sensu == "" {
		opts.Sensu = "localhost:3030"
	}

	if opts.Handlers == "" {
		opts.Handlers = "hipchat"
	}

	if opts.Port == 0 {
		opts.Port = 9731
	}

	if opts.Prefix == "" {
		opts.Prefix = "urlmon"
	}
}

func main() {
	err := envconfig.Process("urlmon", &opts)
	if err != nil {
		log.Fatalf("Error parsing ENV vars %s", err)
	}

	if _, err := flags.Parse(&opts); err != nil {
		if err.(*flags.Error).Type == flags.ErrHelp {
			os.Exit(0)
		} else {
			log.Println(err.Error())
			os.Exit(1)
		}
	}

	// make sure defaults after env/parsing has been done
	cliDefaults()

	etcdServers := strings.Split(opts.Etcd, ",")
	etcdClient := etcd.NewClient(etcdServers)
	setupEtcd(etcdClient)

	watchChan := make(chan *etcd.Response)
	// setup recursive watch of the keyspace
	go etcdClient.Watch(opts.Prefix, 0, true, watchChan, nil)

	// start the metrics
	if opts.User != "" && opts.Token != "" {
		go libratoMetrics()
	}

	loadChecks(etcdClient)

	http.HandleFunc("/status", statusHandler)
	go http.ListenAndServe(fmt.Sprintf(":%d", opts.Port), nil)
	log.Printf("Status server up and running on ':%d'", opts.Port)

	// loop and reload checks when etcd changes
	for {
		r := <-watchChan
		log.Printf("Reloading checks from etcd, triggered by '%s' on '%s' with value: '%s' ", r.Action, r.Node.Key, r.Node.Value)
		loadChecks(etcdClient)
	}
}
<commit_msg> only close body when non-nil response is recieved<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/coreos/go-etcd/etcd"
	//	_ "github.com/davecgh/go-spew/spew"
	"github.com/jessevdk/go-flags"
	"github.com/kelseyhightower/envconfig"
	"github.com/rcrowley/go-metrics"
	"github.com/rcrowley/go-metrics/librato"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/http"
	//_ "net/http/pprof"
	"net/url"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	// DefaultInterval is the check check interval for a url specified in seconds
	DefaultInterval int = 15

	// DefaultSplay is the rand splay to introduce on checks so we don't flood
	DefaultSplay int = 5

	// DefaultAlertLevel is the level we alert at when no level is specified
	DefaultAlertLevel string = "WARN"
)

type Options struct {
	Etcd     string `short:"e" long:"etcd" description:"Etcd Server url. Comma separate multiple servers. default:'http://localhost:4001'"`
	Prefix   string `short:"p" long:"prefix" description:"the prefix to use in etcd for storing check urls. default:'urlmon'"`
	User     string `short:"u" long:"user" description:"librato user"`
	Token    string `short:"t" long:"token"  description:"librato token"`
	Port     int    `short:"P" long:"port"  description:"The port to start the status interface on. default: '0.0.0.0:9731'"`
	Sensu    string `short:"s" long:"sensu" description:"Sensu client address. default:'localhost:3030'"`
	Handlers string `short:"H" long:"handlers" description:"Sensu handler to use for alert message. comma separatemultiples. default:'hipchat'"`
}

type Check struct {
	Id           string
	URL          *url.URL
	Content      string
	Level        string
	ContentRegex *regexp.Regexp
	Interval     int
	Splay        int
	Registry     metrics.Registry
	shutdown     chan struct{}
}

type SensuEvent struct {
	Name     string   `json:"name"`
	Handlers []string `json:"handlers"`
	Output   string   `json:"output"`
	Status   int      `json:"status"`
}

type LazyResponse map[string]interface{}

var (
	Checks []Check
	opts   Options
)

func (c *Check) setupInterval() {
	if c.Interval <= 0 {
		c.Interval = DefaultInterval
	}

	if c.Splay <= 0 {
		c.Splay = DefaultSplay
	}

	// build up splay and add to interval
	c.Interval += rand.Intn(c.Splay)
}

// Validate takes a response body and runs the check validations on it.
// the response body will be closed in the process
func (c *Check) Validate(res *http.Response) (code int, status string) {
	// let checks set weather this is a crit or not, default is WARN which doesn't trigger sensu
	errorVal := 1
	if c.Level == "CRITICAL" {
		errorVal = 2
	}
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return errorVal, fmt.Sprintf("Error reading response: %s", err.Error())
	}

	if c.Content != "" && !strings.Contains(string(body), c.Content) {
		return errorVal, "Content match failed"
	}

	if c.ContentRegex != nil && !c.ContentRegex.Match(body) {
		return errorVal, "Content Regex did not match"
	}

	// everyting is groovy, return ok with status
	return 0, res.Status
}

// reportStatus reports if
func (c *Check) reportStatus(status int, msg string) {
	s := "Success:"
	if status != 0 {
		s = "Fail:"
	}
	log.Println(s, c.Id, c.URL.String(), msg)

	// for now handler is hardcode TODO: move it to default/param/env
	event := SensuEvent{
		Name:     c.Id,
		Output:   fmt.Sprintf("%s::%s", msg, c.URL),
		Handlers: strings.Split(opts.Handlers, ","),
		Status:   status,
	}

	event.send()
}

// Monitor sets up the monitoring loop
func (c *Check) Monitor() {
	// build the splay/random timing into the interval
	c.setupInterval()

	// setup the guage
	g := metrics.NewGauge()

	metrics.Register(fmt.Sprintf("urlmon.request.%s", c.Id), g)
	// loop for shutdown and spaly/sleep
	for {
		select {
		case <-c.shutdown:
			log.Printf("%s Shutdown", c.Id)
			return
		default:
			// we sleep first here so that on startup we don't overwhelm
			s := time.Duration(c.Interval) * time.Second
			time.Sleep(s)

			// time.Since reports in nanoseocnds
			start := time.Now()
			r, err := http.Get(c.URL.String())
			g.Update(int64(time.Since(start)))
			if err != nil {
				c.reportStatus(2, err.Error())
			} else {
				c.reportStatus(c.Validate(r))
				r.Body.Close()
			}
		}
	}
}

// easy way to signal the monitor to shutdown
func (c *Check) Shutdown() {
	c.shutdown <- struct{}{}
}

// String Adds stringer interface to the lazy response
func (r LazyResponse) String() (s string) {
	b, err := json.MarshalIndent(r, "", "  ")
	if err != nil {
		s = ""
		return
	}
	s = string(b)
	return
}

// send the sensu event to the local sensu port
func (e *SensuEvent) send() {
	conn, err := net.Dial("tcp", opts.Sensu)
	if err != nil {
		log.Println("Error connecting to sensu client socket.", err.Error())
		return
	}
	defer conn.Close()

	j, err := json.Marshal(e)
	if err != nil {
		log.Println("Error marshaling event data.", err.Error())
		return
	}
	fmt.Fprintf(conn, string(j))
}

// libratoMetrics starts the reporting metrics on this check to librato
func libratoMetrics() {
	host, err := os.Hostname()
	if err != nil {
		host = "localhost"
	}

	// start the metrics collector should be per-check
	log.Println("starting metrics")
	librato.Librato(metrics.DefaultRegistry,
		60*time.Second, // interval
		opts.User,      // account owner email address
		opts.Token,     // Librato API token
		// This should be per check I think
		host,             // source
		[]float64{95},    // precentiles to send
		time.Millisecond, // time unit
	)
}

// valueStr Converts a string to an int where a blank string or errvalue returns a 0
// This is a helper mostly for creating a Check where the interval or splay may be an empty string.
func valueStr(value string) int {
	v, err := strconv.Atoi(value)
	if err != nil {
		v = 0
	}
	return v
}

// createCheck populates check struct with data from etcd check
func createCheck(node *etcd.Node) (*Check, error) {
	// create a check
	c := Check{Id: path.Base(node.Key)}
	c.shutdown = make(chan struct{}, 1)
	for _, child := range node.Nodes {
		log.Printf("  - %s", path.Base(child.Key))
		switch strings.ToUpper(path.Base(child.Key)) {
		case "URL":
			u, err := url.Parse(child.Value)
			if err != nil {
				log.Printf("Url isn't valid for key: %s, %s", node.Key, err.Error())
				continue
			}
			c.URL = u
		case "CONTENT":
			c.Content = child.Value
		case "REGEX":
			r, err := regexp.Compile(child.Value)
			if err != nil {
				log.Fatalf("couldn't compile regex for Check: %s, %s", node.Key, err.Error())
			}
			c.ContentRegex = r
		case "LEVEL":
			l := strings.ToUpper(child.Value)
			if l == "" {
				l = DefaultAlertLevel
			}
			c.Level = l
		case "SPLAY":
			c.Splay = valueStr(child.Value)
		case "INTERVAL":
			c.Interval = valueStr(child.Value)
		}
	}

	// set defaults
	if c.Splay == 0 {
		c.Splay = DefaultSplay
	}
	if c.Interval == 0 {
		c.Interval = DefaultInterval
	}
	if c.Level == "" {
		c.Level = DefaultAlertLevel
	}
	if c.URL == nil {
		return &c, errors.New("No URL for check")
	}

	// BUG: Checks may be malformed in various ways, createCheck needs to implment more validations before returning the check
	return &c, nil
}

// loadChecks reads etcd popilates Checks, and starts their monitor
func loadChecks(client *etcd.Client) {
	resp, err := client.Get(fmt.Sprintf("%s/checks", opts.Prefix), true, true)
	if err != nil {
		log.Fatalf("Problem fetching Checks from etcd: %s", err)
	}

	for _, n := range Checks {
		// signal that monitor to shutdown
		log.Println("Shutting down ", n.Id)
		n.Shutdown()
	}

	// this clears the slice.
	Checks = nil

	for _, n := range resp.Node.Nodes {
		// these top level nodes should be a directory
		if !n.Dir {
			log.Printf("Error loading config %s is not a dir, skipping", n.Key)
			continue
		}

		log.Printf("Loading: %s: %s\n", n.Key, n.Value)
		c, err := createCheck(n)
		if err != nil {
			log.Println("Failed to create check, skipping: ", n.Key, err)
			continue
		}
		// start monitoring this check
		go c.Monitor()
		Checks = append(Checks, *c)
	}
}

// Create our Etcd Dir structure
func setupEtcd(client *etcd.Client) {
	for _, path := range []string{opts.Prefix, fmt.Sprintf("%s/checks", opts.Prefix)} {
		if _, err := client.Get(path, false, false); err != nil {
			log.Printf("Creating dir in etcd: %s ", path)
			if _, err := client.CreateDir(path, 0); err != nil {
				log.Fatal("Couldn't create etcd dir: ", err)
			}
		}
	}
}

func statusHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprint(w, LazyResponse{"checkdata": Checks})
}

// cliDefaults sets the options to default vaules if env or switches haven't specified them
func cliDefaults() {
	if opts.Etcd == "" {
		opts.Etcd = "http://localhost:4001"
	}

	if opts.Sensu == "" {
		opts.Sensu = "localhost:3030"
	}

	if opts.Handlers == "" {
		opts.Handlers = "hipchat"
	}

	if opts.Port == 0 {
		opts.Port = 9731
	}

	if opts.Prefix == "" {
		opts.Prefix = "urlmon"
	}
}

func main() {
	err := envconfig.Process("urlmon", &opts)
	if err != nil {
		log.Fatalf("Error parsing ENV vars %s", err)
	}

	if _, err := flags.Parse(&opts); err != nil {
		if err.(*flags.Error).Type == flags.ErrHelp {
			os.Exit(0)
		} else {
			log.Println(err.Error())
			os.Exit(1)
		}
	}

	// make sure defaults after env/parsing has been done
	cliDefaults()

	etcdServers := strings.Split(opts.Etcd, ",")
	etcdClient := etcd.NewClient(etcdServers)
	setupEtcd(etcdClient)

	watchChan := make(chan *etcd.Response)
	// setup recursive watch of the keyspace
	go etcdClient.Watch(opts.Prefix, 0, true, watchChan, nil)

	// start the metrics
	if opts.User != "" && opts.Token != "" {
		go libratoMetrics()
	}

	loadChecks(etcdClient)

	http.HandleFunc("/status", statusHandler)
	go http.ListenAndServe(fmt.Sprintf(":%d", opts.Port), nil)
	log.Printf("Status server up and running on ':%d'", opts.Port)

	// loop and reload checks when etcd changes
	for {
		r := <-watchChan
		log.Printf("Reloading checks from etcd, triggered by '%s' on '%s' with value: '%s' ", r.Action, r.Node.Key, r.Node.Value)
		loadChecks(etcdClient)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"strings"

	"github.com/dgageot/getme/cache"
	"github.com/dgageot/getme/files"
	"github.com/dgageot/getme/tar"
	"github.com/dgageot/getme/zip"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

var authToken string

func main() {
	var rootCmd = &cobra.Command{Use: "getme"}

	rootCmd.PersistentFlags().StringVar(&authToken, "authToken", "", "Api authentication token")

	rootCmd.AddCommand(&cobra.Command{
		Use: "Download",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) < 1 {
				return errors.New("An url must be provided")
			}
			url := args[0]

			return Download(url)
		},
	})

	rootCmd.AddCommand(&cobra.Command{
		Use: "Copy",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) < 2 {
				return errors.New("An url and a destination must be provided")
			}
			url := args[0]
			destination := args[1]

			return Copy(url, destination)
		},
	})

	rootCmd.AddCommand(&cobra.Command{
		Use: "Unzip",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) < 2 {
				return errors.New("An url and a destination folder must be provided")
			}
			url := args[0]
			destinationFolder := args[1]

			return Unzip(url, destinationFolder)
		},
	})

	rootCmd.AddCommand(&cobra.Command{
		Use: "UnzipSingleFile",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) < 3 {
				return errors.New("An url, a file name and a destination must be provided")
			}
			url := args[0]
			name := args[1]
			destination := args[2]

			return UnzipSingleFile(url, name, destination)
		},
	})

	rootCmd.AddCommand(&cobra.Command{
		Use: "Prune",
		RunE: func(cmd *cobra.Command, args []string) error {
			return Prune()
		},
	})

	if err := rootCmd.Execute(); err != nil {
		log.Fatal(err)
	}
}

// Download retrieves an url from the cache or download it if it's absent.
// Then print the path to that file to stdout.
func Download(url string) error {
	// Discard all the logs. We only want to output the path to the file
	log.SetOutput(ioutil.Discard)

	source, err := cache.Download(url, headers())
	if err != nil {
		return err
	}

	fmt.Println(source)

	return nil
}

// Copy retrieves an url from the cache or download it if it's absent.
// Then it copies the file to a destination path.
func Copy(url string, destination string) error {
	source, err := cache.Download(url, headers())
	if err != nil {
		return err
	}

	log.Println("Copy", url, "to", destination)

	return files.Copy(source, destination)
}

// Download retrieves an url from the cache or download it if it's absent.
// Then it unzips the file to a destination directory.
func Unzip(url string, destinationDirectory string) error {
	source, err := cache.Download(url, headers())
	if err != nil {
		return err
	}

	log.Println("Unzip", url, "to", destinationDirectory)

	if strings.HasSuffix(source, ".zip") {
		return zip.Extract(source, destinationDirectory)
	}
	if strings.HasSuffix(source, ".tar") || strings.HasSuffix(source, ".tar.gz") || strings.HasSuffix(source, ".tgz") {
		return tar.Extract(source, destinationDirectory)
	}

	return errors.New("Unsupported archive: " + source)
}

// Download retrieves an url from the cache or download it if it's absent.
// Then it unzips a single file from that zip to a destination path.
func UnzipSingleFile(url string, name string, destination string) error {
	source, err := cache.Download(url, headers())
	if err != nil {
		return err
	}

	log.Println("Unzip", name, "from", url, "to", destination)

	if strings.HasSuffix(source, ".zip") {
		return zip.ExtractFile(source, name, destination)
	}
	if strings.HasSuffix(source, ".tar") || strings.HasSuffix(source, ".tar.gz") || strings.HasSuffix(source, ".tgz") {
		return tar.ExtractFile(source, name, destination)
	}

	return errors.New("Unsupported archive: " + source)
}

// Prune prunes the cache.
func Prune() error {
	return cache.Prune()
}

func headers() []string {
	if authToken == "" {
		return nil
	}

	return []string{fmt.Sprintf("Authorization=Bearer %s", authToken)}
}
<commit_msg>Rename Unzip* actions<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"strings"

	"github.com/dgageot/getme/cache"
	"github.com/dgageot/getme/files"
	"github.com/dgageot/getme/tar"
	"github.com/dgageot/getme/zip"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

var authToken string

func main() {
	var rootCmd = &cobra.Command{Use: "getme"}

	rootCmd.PersistentFlags().StringVar(&authToken, "authToken", "", "Api authentication token")

	rootCmd.AddCommand(&cobra.Command{
		Use: "Download",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) < 1 {
				return errors.New("An url must be provided")
			}
			url := args[0]

			return Download(url)
		},
	})

	rootCmd.AddCommand(&cobra.Command{
		Use: "Copy",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) < 2 {
				return errors.New("An url and a destination must be provided")
			}
			url := args[0]
			destination := args[1]

			return Copy(url, destination)
		},
	})

	rootCmd.AddCommand(&cobra.Command{
		Use:     "Extract",
		Aliases: []string{"Unzip"},
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) < 2 {
				return errors.New("An url and a destination folder must be provided")
			}
			url := args[0]
			destinationFolder := args[1]

			return Unzip(url, destinationFolder)
		},
	})

	rootCmd.AddCommand(&cobra.Command{
		Use:     "ExtractSingleFile",
		Aliases: []string{"UnzipSingleFile"},
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) < 3 {
				return errors.New("An url, a file name and a destination must be provided")
			}
			url := args[0]
			name := args[1]
			destination := args[2]

			return UnzipSingleFile(url, name, destination)
		},
	})

	rootCmd.AddCommand(&cobra.Command{
		Use: "Prune",
		RunE: func(cmd *cobra.Command, args []string) error {
			return Prune()
		},
	})

	if err := rootCmd.Execute(); err != nil {
		log.Fatal(err)
	}
}

// Download retrieves an url from the cache or download it if it's absent.
// Then print the path to that file to stdout.
func Download(url string) error {
	// Discard all the logs. We only want to output the path to the file
	log.SetOutput(ioutil.Discard)

	source, err := cache.Download(url, headers())
	if err != nil {
		return err
	}

	fmt.Println(source)

	return nil
}

// Copy retrieves an url from the cache or download it if it's absent.
// Then it copies the file to a destination path.
func Copy(url string, destination string) error {
	source, err := cache.Download(url, headers())
	if err != nil {
		return err
	}

	log.Println("Copy", url, "to", destination)

	return files.Copy(source, destination)
}

// Download retrieves an url from the cache or download it if it's absent.
// Then it unzips the file to a destination directory.
func Unzip(url string, destinationDirectory string) error {
	source, err := cache.Download(url, headers())
	if err != nil {
		return err
	}

	log.Println("Unzip", url, "to", destinationDirectory)

	if strings.HasSuffix(source, ".zip") {
		return zip.Extract(source, destinationDirectory)
	}
	if strings.HasSuffix(source, ".tar") || strings.HasSuffix(source, ".tar.gz") || strings.HasSuffix(source, ".tgz") {
		return tar.Extract(source, destinationDirectory)
	}

	return errors.New("Unsupported archive: " + source)
}

// Download retrieves an url from the cache or download it if it's absent.
// Then it unzips a single file from that zip to a destination path.
func UnzipSingleFile(url string, name string, destination string) error {
	source, err := cache.Download(url, headers())
	if err != nil {
		return err
	}

	log.Println("Unzip", name, "from", url, "to", destination)

	if strings.HasSuffix(source, ".zip") {
		return zip.ExtractFile(source, name, destination)
	}
	if strings.HasSuffix(source, ".tar") || strings.HasSuffix(source, ".tar.gz") || strings.HasSuffix(source, ".tgz") {
		return tar.ExtractFile(source, name, destination)
	}

	return errors.New("Unsupported archive: " + source)
}

// Prune prunes the cache.
func Prune() error {
	return cache.Prune()
}

func headers() []string {
	if authToken == "" {
		return nil
	}

	return []string{fmt.Sprintf("Authorization=Bearer %s", authToken)}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
)

type Child struct {
	Path     []int   `json:"path,omitempty"`
	X        *int    `json:"x,omitempty"`
	Y        *int    `json:"y,omitempty"`
	Val      *int    `json:"val,omitempty"`
	Children []Child `json:"children,omitempty`
}

type Root struct {
	Val      *int    `json:"val,omitempty"`
	Children []Child `json:"children,omitempty"`
}

type NanocubeResponse struct {
	Layers []string `json:"layers"`
	Root   Root     `json:"root"`
}

// merge merges the two interfaces (supposedly json objects) passed in according
// to the API.md file in https://github.com/laurolins/nanocube.
func merge(dest, src *NanocubeResponse) {
	// Little check for layers.
	if dest.Layers == nil {
		dest.Layers = []string{}
	}

	if dest.Root.Val != nil {
		*(dest.Root.Val) += *(src.Root.Val)
		return
	}
}

func unmarshalNanocubeResponse(b []byte, dest *NanocubeResponse) {
	// TODO(asubiotto): Handle error.
	err := json.Unmarshal(b, dest)
	if err != nil {
		log.Fatal(err)
	}
}

// TODO(asubiotto): Validate the request.
func handler(w http.ResponseWriter, r *http.Request) {
	// Our actual response to the request.
	var response *NanocubeResponse

	// Mutex to protect concurrent modification of response and WaitGroup to
	// wait for all responses.
	var mtx = &sync.Mutex{}
	var wg = &sync.WaitGroup{}

	wg.Add(5)
	// Send off the request to each server that we know exists.
	for i := 1; i <= 5; i++ {
		go func(counter int) {
			defer wg.Done()

			// TODO(asubiotto): Was getting an annoying EOF error. Put the check
			// back in for an error.
			rawResponse, err := http.Get(
				fmt.Sprintf("http://localhost:900%v%v", counter, r.URL.Path),
			)

			if err != nil {
				return
			}

			// Read the response and unmarshal into an unknown interface object.
			defer rawResponse.Body.Close()
			content, _ := ioutil.ReadAll(rawResponse.Body)

			partitionResponse := new(NanocubeResponse)
			unmarshalNanocubeResponse(content, partitionResponse)

			// Update our global response.
			mtx.Lock()
			defer mtx.Unlock()
			if response == nil {
				// First one to respond, we just set the global response to be
				// our response.
				response = partitionResponse
				return
			}

			// Otherwise we merge with what we have already.
			merge(response, partitionResponse)
		}(i)
	}

	wg.Wait()
	b, err := json.Marshal(response)
	if err != nil {
		log.Fatal(err)
	}

	w.Write(b)
}

func main() {
	// Start a server to listen for queries and forward them.
	http.HandleFunc("/", handler)
	http.ListenAndServe(":29512", nil)
}
<commit_msg>Added code to merge children arrays. Not tested yet. Next on the list: re-organize the code and add tests<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
)

type Child struct {
	Path     []byte  `json:"path,omitempty"`
	X        *int    `json:"x,omitempty"`
	Y        *int    `json:"y,omitempty"`
	Val      *int    `json:"val,omitempty"`
	Children []Child `json:"children,omitempty`
}

type Root struct {
	Val      *int    `json:"val,omitempty"`
	Children []Child `json:"children,omitempty"`
}

type NanocubeResponse struct {
	Layers []string `json:"layers"`
	Root   Root     `json:"root"`
}

// TODO(asubiotto): The whole error handling has to be looked at more closely.

// mergeChildren merges the children array by aggregating "val"s at the base
// level and using either path as key or x and y as keys.
func mergeChildren(dest, src []Child) {
	if dest == nil || src == nil {
		if dest == nil {
			dest = src
		}
		return
	}

	destMap := make(map[string]int)
	for _, child := range dest {
		// If there's a path there's a value.
		if child.Val == nil {
			log.Fatal("Child value was nil in dest")
		}
		destMap[string(child.Path)] = *(child.Val)
	}

	for _, child := range src {
		if child.Val == nil {
			log.Fatal("Child value was nil in src")
		}

		if _, e := destMap[string(child.Path)]; e {
			// If it exists, we add the value that src has for this key.
			destMap[string(child.Path)] += *(child.Val)
		} else {
			// Otherwise we add it.
			destMap[string(child.Path)] = *(child.Val)
		}
	}

	// Now that we have the results in destMap, we are going to update dest's
	// children slice.
	for _, child := range dest {
		key := string(child.Path)
		*(child.Val) = destMap[key]
		delete(destMap, key)
	}

	if len(destMap) == 0 {
		return
	}

	// The keys that are left after this are paths that were not in dest to
	// begin with.
	newDest := make([]Child, len(dest)+len(destMap))
	copy(newDest, dest)
	i := len(dest)
	for k, v := range destMap {
		newDest[i] = Child{Path: []byte(k), Val: &v}
		i++
	}
	dest = newDest
}

// merge merges src into dest according to the API.md document found in
// github.com/laurolins/nanocube.
func merge(dest, src *NanocubeResponse) {
	// Little check for layers. TODO(asubiotto): Think about this case a bit
	// more when we have empty responses or we want to return an error.
	if dest.Layers == nil {
		dest.Layers = []string{}
	}

	if dest.Root.Val != nil {
		*(dest.Root.Val) += *(src.Root.Val)
		return
	}

	mergeChildren(dest.Root.Children, src.Root.Children)
}

func unmarshalNanocubeResponse(b []byte, dest *NanocubeResponse) {
	// TODO(asubiotto): Handle error. Do we really need this to be a separate
	// function?
	err := json.Unmarshal(b, dest)
	if err != nil {
		log.Fatal(err)
	}
}

// TODO(asubiotto): Validate the request.
func handler(w http.ResponseWriter, r *http.Request) {
	// Our actual response to the request.
	var response *NanocubeResponse

	// Mutex to protect concurrent modification of response and WaitGroup to
	// wait for all responses.
	var mtx = &sync.Mutex{}
	var wg = &sync.WaitGroup{}

	wg.Add(5)
	// Send off the request to each server that we know exists.
	for i := 1; i <= 5; i++ {
		go func(counter int) {
			defer wg.Done()

			// TODO(asubiotto): Was getting an annoying EOF error. Put the check
			// back in for an error.
			rawResponse, err := http.Get(
				fmt.Sprintf("http://localhost:900%v%v", counter, r.URL.Path),
			)

			if err != nil {
				return
			}

			// Read the response and unmarshal into a NanocubeResponse object.
			defer rawResponse.Body.Close()
			content, _ := ioutil.ReadAll(rawResponse.Body)

			partitionResponse := new(NanocubeResponse)
			unmarshalNanocubeResponse(content, partitionResponse)

			// Update our global response.
			mtx.Lock()
			defer mtx.Unlock()
			if response == nil {
				// First one to respond, we just set the global response to be
				// our response.
				response = partitionResponse
				return
			}

			// TODO(asubiotto): We can use a more intelligent merging strategy
			// when we have a lot of nodes (in a tree-like fashion).

			// Otherwise we merge with what we have already.
			merge(response, partitionResponse)
		}(i)
	}

	wg.Wait()
	b, err := json.Marshal(response)
	if err != nil {
		log.Fatal(err)
	}

	w.Write(b)
}

func main() {
	// Start a server to listen for queries and forward them.
	http.HandleFunc("/", handler)
	http.ListenAndServe(":29512", nil)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"strings"

	flags "github.com/jessevdk/go-flags"
	"github.com/pkg/errors"
)

var opts struct {
	File     string    `short:"f" long:"file" description:"Input file in /etc/hosts format" default:"/etc/hosts" value-name:"HOSTFILE"`
	Networks []CIDRNet `long:"network" description:"Filter by CIDR network" value-name:"x.x.x.x/len"`
	Domain   string    `short:"d" long:"domain" description:"Domain to update records in" required:"true"`
	TTL      int64     `long:"ttl" description:"TTL to use for Route53 records" default:"3600"`
}

func parseOpts() {
	parser := flags.NewParser(&opts, flags.Default)

	if _, err := parser.Parse(); err != nil {
		if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	// Accept trailing dot, but ignore it for consistency sake
	if strings.HasSuffix(opts.Domain, ".") {
		opts.Domain = opts.Domain[:len(opts.Domain)-1]
	}
}

func canonifyHostname(hostname string) string {
	hostname = strings.ToLower(hostname)
	// Accept trailing dot, but ignore it for consistency sake
	if strings.HasSuffix(hostname, ".") {
		hostname = hostname[:len(hostname)-1]
	}

	return hostname
}

// compareHosts takes the contents of the local /etc/hosts file and the
// route53 hosts that should be compared and produces two arrays.  The first
// array is a list of Route53 records that need to be updated, and the second
// array is a list of Route53 records that should be deleted.
func compareHosts(hosts hostList, r53hosts hostList) (hostList, hostList) {
	// Build index on name, we'll delete entries out of here a we match them
	// against /etc/hosts entries.  The remaining entries aren't present
	// locally anymore and will need to be deleted.
	rhByName := map[string]hostEntry{}
	for _, rh := range r53hosts {
		rhByName[rh.hostname] = rh
	}

	toUpdate := hostList{}
	// Find existing hosts
	for _, h := range hosts {
		rh, ok := rhByName[h.hostname]
		if ok {
			delete(rhByName, h.hostname)
			if !h.ip.Equal(rh.ip) {
				toUpdate = append(toUpdate, h)
			}
		} else {
			toUpdate = append(toUpdate, h)
		}
	}

	toDelete := make(hostList, 0, len(rhByName))
	for _, rh := range rhByName {
		toDelete = append(toDelete, rh)
	}

	return toUpdate, toDelete
}

func main() {
	parseOpts()
	hosts := readHosts(opts.File)
	hosts = filterHosts(hosts, opts.Networks)

	r53 := newRoute53()
	r53Hosts, err := r53.getHosts(opts.Domain)
	if err != nil {
		log.Fatalln(errors.Wrap(err, "error when retrieving zones"))
	}
	r53Hosts = filterHosts(r53Hosts, opts.Networks)

	toUpdate, toDelete := compareHosts(hosts, r53Hosts)
	if len(toUpdate) > 0 || len(toDelete) > 0 {
		if err := r53.sync(opts.Domain, opts.TTL, toUpdate, toDelete); err != nil {
			log.Fatalln(errors.Wrap(err, "Could not sync records to Route 53"))
		}
	} else {
		log.Println("No changes needed.  Everything in sync.")
	}
}
<commit_msg>Add code to remove dupes from /etc/hosts<commit_after>package main

import (
	"log"
	"os"
	"sort"
	"strings"

	flags "github.com/jessevdk/go-flags"
	"github.com/pkg/errors"
)

var opts struct {
	File     string    `short:"f" long:"file" description:"Input file in /etc/hosts format" default:"/etc/hosts" value-name:"HOSTFILE"`
	Networks []CIDRNet `long:"network" description:"Filter by CIDR network" value-name:"x.x.x.x/len"`
	Domain   string    `short:"d" long:"domain" description:"Domain to update records in" required:"true"`
	TTL      int64     `long:"ttl" description:"TTL to use for Route53 records" default:"3600"`
}

func parseOpts() {
	parser := flags.NewParser(&opts, flags.Default)

	if _, err := parser.Parse(); err != nil {
		if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp {
			os.Exit(0)
		} else {
			os.Exit(1)
		}
	}

	// Accept trailing dot, but ignore it for consistency sake
	if strings.HasSuffix(opts.Domain, ".") {
		opts.Domain = opts.Domain[:len(opts.Domain)-1]
	}
}

func canonifyHostname(hostname string) string {
	hostname = strings.ToLower(hostname)
	// Accept trailing dot, but ignore it for consistency sake
	if strings.HasSuffix(hostname, ".") {
		hostname = hostname[:len(hostname)-1]
	}

	return hostname
}

// compareHosts takes the contents of the local /etc/hosts file and the
// route53 hosts that should be compared and produces two arrays.  The first
// array is a list of Route53 records that need to be updated, and the second
// array is a list of Route53 records that should be deleted.
func compareHosts(hosts hostList, r53hosts hostList) (hostList, hostList) {
	// Build index on name, we'll delete entries out of here a we match them
	// against /etc/hosts entries.  The remaining entries aren't present
	// locally anymore and will need to be deleted.
	rhByName := map[string]hostEntry{}
	for _, rh := range r53hosts {
		rhByName[rh.hostname] = rh
	}

	toUpdate := hostList{}
	// Find existing hosts
	for _, h := range hosts {
		rh, ok := rhByName[h.hostname]
		if ok {
			delete(rhByName, h.hostname)
			if !h.ip.Equal(rh.ip) {
				toUpdate = append(toUpdate, h)
			}
		} else {
			toUpdate = append(toUpdate, h)
		}
	}

	toDelete := make(hostList, 0, len(rhByName))
	for _, rh := range rhByName {
		toDelete = append(toDelete, rh)
	}

	return toUpdate, toDelete
}

func removeDupes(hosts hostList) hostList {
	found := make(map[string]bool, len(hosts))

	// Sort hostlist to ensure stable duplication suppression.  We don't want
	// to ping pont between choosing different options because of parse order.
	sort.Sort(hosts)

	dupCount := 0
	result := make(hostList, 0, len(hosts))
	for _, h := range hosts {
		if _, ok := found[h.hostname]; ok {
			log.Printf("Duplicate hostname found in /etc/hosts, ignoring (%v/%v)",
				h.hostname, h.ip.String())
			dupCount++
		} else {
			found[h.hostname] = true
			result = append(result, h)
		}
	}

	return result
}

func main() {
	parseOpts()
	hosts := readHosts(opts.File)
	hosts = filterHosts(hosts, opts.Networks)
	hosts = removeDupes(hosts)

	r53 := newRoute53()
	r53Hosts, err := r53.getHosts(opts.Domain)
	if err != nil {
		log.Fatalln(errors.Wrap(err, "error when retrieving zones"))
	}
	r53Hosts = filterHosts(r53Hosts, opts.Networks)

	toUpdate, toDelete := compareHosts(hosts, r53Hosts)
	if len(toUpdate) > 0 || len(toDelete) > 0 {
		if err := r53.sync(opts.Domain, opts.TTL, toUpdate, toDelete); err != nil {
			log.Fatalln(errors.Wrap(err, "Could not sync records to Route 53"))
		}
	} else {
		log.Println("No changes needed.  Everything in sync.")
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Lieven Govaerts. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"code.google.com/p/gopacket"
	"code.google.com/p/gopacket/layers"
	"code.google.com/p/gopacket/pcap"
	"code.google.com/p/gopacket/tcpassembly"
	"code.google.com/p/gopacket/tcpassembly/tcpreader"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"time"
)

/* Command line arguments */
var iface = flag.String("i", "en0", "Interface to get packets from")
var logAllPackets = flag.Bool("v", false, "Logs every packet in great detail")
var launchCmd = flag.String("e", "", "Launches the command and logs its traffic")

type interval struct {
	req  *http.Request
	resp *http.Response
}

type BidiStream struct {
	key     uint64
	in, out *TCPStream
}

// TCPStream will handle the actual decoding of http requests and responses.
type TCPStream struct {
	netFlow, tcpFlow gopacket.Flow
	readStream       tcpreader.ReaderStream
	storage          *Storage
	bidikey          uint64
}

/* This reads both HTTP requests and HTTP responses in two separate streams */
func (h *TCPStream) runOut() {
	buf := bufio.NewReader(&h.readStream)
	var reqID int64
	for {
		req, err := http.ReadRequest(buf)
		if err == io.EOF {
			// We must read until we see an EOF... very important!
			return
		} else if err != nil {
			log.Println("Error reading stream", h.netFlow, h.tcpFlow, ":", err)
		} else {
			// bodyBytes := tcpreader.DiscardBytesToEOF(req.Body)
			req.Body.Close()
			err = h.storage.SentRequest(h.bidikey, reqID, time.Now(), req)
			if err != nil {
				log.Println("Error storing request", err)
			}

			reqID++
			//			fmt.Print(".")
			// log.Println("Received request from stream", h.netFlow, h.tcpFlow,
			// 	":", req, "with", bodyBytes, "bytes in request body")
		}
	}
}

func (h *TCPStream) runIn() {
	buf := bufio.NewReader(&h.readStream)
	var reqID int64
	for {

		/* Don't start reading a response if no data is available */
		_, err := buf.Peek(1)
		if err == io.EOF {
			return
		}
		/* Data available, read response */
		resp, err := http.ReadResponse(buf, nil) // TODO: request
		if err == io.EOF {
			// We must read until we see an EOF... very important!
			log.Println("EOF while reading stream", h.netFlow, h.tcpFlow, ":", err)
			return
		} else if err != nil {
			log.Println("Error reading stream", h.netFlow, h.tcpFlow, ":", err)
		} else {
			_, err := tcpreader.DiscardBytesToFirstError(resp.Body)
			if err != nil && err != io.EOF {
				log.Println(err)
			}
			resp.Body.Close()
			err = h.storage.ReceivedResponse(h.bidikey, reqID, time.Now(), resp)
			if err != nil {
				log.Println("Error storing response", err)
			}
			reqID++
			// fmt.Print(".")
			//log.Println("Received response from stream", h.netFlow, h.tcpFlow,
			//	":", resp, "with", bodyBytes, "bytes in response body")
		}

		/* Match the response with the next request */
	}

}

/* httpStreamFactory implements tcpassembly.StreamFactory */
type httpStreamFactory struct {
	bidiStreams map[uint64]*BidiStream
	storage     *Storage
}

func (h *httpStreamFactory) New(netFlow, tcpFlow gopacket.Flow) tcpassembly.Stream {

	/* First the outgoing stream, then the incoming stream */
	key := netFlow.FastHash() ^ tcpFlow.FastHash()

	hstream := &TCPStream{
		netFlow:    netFlow,
		tcpFlow:    tcpFlow,
		readStream: tcpreader.NewReaderStream(),
		storage:    h.storage,
		bidikey:    key,
	}

	bds := h.bidiStreams[key]
	if bds == nil {
		log.Println("reading stream", netFlow, tcpFlow)
		bds = &BidiStream{out: hstream, key: key}
		h.bidiStreams[key] = bds
		// Start a coroutine per stream, to ensure that all data is read from
		// the reader stream
		go hstream.runOut()
	} else {
		bds.in = hstream
		err := h.storage.OpenTCPConnection(key, time.Now())
		if err != nil {
			log.Println("Error storing connection", err)
		}
		// Start a coroutine per stream, to ensure that all data is read from
		// the reader stream
		go hstream.runIn()
	}

	// ReaderStream implements tcpassembly.Stream, so we can return a pointer to it.
	return &hstream.readStream
}

func (h *httpStreamFactory) LogPacketSize(packet gopacket.Packet) {
	netFlow := packet.NetworkLayer().NetworkFlow()
	tcpFlow := packet.TransportLayer().TransportFlow()
	key := netFlow.FastHash() ^ tcpFlow.FastHash()

	ipv4Layer := packet.Layer(layers.LayerTypeIPv4)
	ipv4, _ := ipv4Layer.(*layers.IPv4)

	tcpLayer := packet.Layer(layers.LayerTypeTCP)
	tcp, _ := tcpLayer.(*layers.TCP)

	bds := h.bidiStreams[key]
	if bds == nil || bds.in == nil || bds.out == nil {
		return
	}

	payloadLength := uint32(ipv4.Length - uint16(ipv4.IHL)*4 - uint16(tcp.DataOffset)*4)

	if bds.in.netFlow == netFlow {
		/* incoming */
		err := h.storage.IncomingTCPPacket(key, payloadLength)
		if err != nil {
			panic(err)
		}
	} else {
		/* outgoing */
		err := h.storage.OutgoingTCPPacket(key, payloadLength)
		if err != nil {
			panic(err)
		}
	}
}

type Assembler struct {
	assembler *tcpassembly.Assembler
}

func NewAssembler(streamPool *tcpassembly.StreamPool) *Assembler {
	return &Assembler{tcpassembly.NewAssembler(streamPool)}
}

/* PFF, a lot of abstractions that make things more difficult then they should be! */
func (a *Assembler) AssembleWithTimestamp(netFlow gopacket.Flow, t *layers.TCP,
	timestamp time.Time) {
	a.assembler.AssembleWithTimestamp(netFlow, t, timestamp)
}

func create_process_ended_channel(cmd *exec.Cmd) (cmd_done chan error) {
	cmd_done = make(chan error, 1)
	go func() {
		cmd_done <- cmd.Wait()
	}()
	return cmd_done
}

/* Create a channel and send any CTRL-C signal over it */
func create_ctrl_c_handler() (timeout chan os.Signal) {
	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)
	return ctrlc
}

/* Create a channel that will contain TRUE after T seconds. */
func create_timeout_channel(t time.Duration) (timeout chan bool) {
	timeout = make(chan bool, 1)
	go func() {
		time.Sleep(t * time.Second)
		timeout <- true
	}()
	return timeout
}

func main() {
	flag.Parse()
	log.SetFlags(log.Ltime | log.Lmicroseconds)
	//	log.SetOutput(ioutil.Discard)

	log.Printf("starting capture on interface %q", *iface)

	/* TODO: we need a storage layer per goroutine! */
	// Set up storage layer
	storage, err := NewStorage()
	if err != nil {
		panic(err)
	}

	// Set up assembly
	streamFactory := &httpStreamFactory{bidiStreams: make(map[uint64]*BidiStream),
		storage: storage}
	streamPool := tcpassembly.NewStreamPool(streamFactory)
	assembler := NewAssembler(streamPool)

	// Setup CTRL-C handler channel
	ctrlc := create_ctrl_c_handler()

	// Setup channel that reports all socket kernel events (Mac OS X only)
	netDescSource := NewOSXNetDescSource()
	descriptors := netDescSource.Descriptors()

	// Setup packet capture
	handle, err := pcap.OpenLive(*iface, 1600, true, pcap.BlockForever)
	if err != nil {
		panic(err)
	} else if err := handle.SetBPFFilter("tcp and port 80"); err != nil {
		panic(err)
	}
	log.Println("reading in packets. Press CTRL-C to end and report.")
	packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
	packets := packetSource.Packets()

	// Run the external command
	pid := uint32(0)
	var cmd_done chan error
	var start_time time.Time
	if *launchCmd != "" {
		s := *launchCmd
		//		n := strings.Index(*launchCmd, " ")
		//		fmt.Println("Launching ", s[0:n], " --- ", s[n:])
		//		cmd := exec.Command(s[0:n], strings.Split(s[n:], " "))
		args := strings.Split(s, " ")
		cmd := exec.Command(args[0], args[1:]...)
		start_time = time.Now()
		err := cmd.Start()
		if err != nil {
			panic(err)
		}
		// Create the channel that listens for the end of the command.
		cmd_done = create_process_ended_channel(cmd)
		pid = uint32(cmd.Process.Pid)
	}

	var timeout chan bool
	for {
		select {
		case netDesc := <-descriptors:
			if netDesc.Pid == pid {
				fmt.Println("event received ", netDesc)
			}
		case packet := <-packets:
			if *logAllPackets {
				log.Println(packet)
			}

			if storage.PacketInScope(packet) {
				streamFactory.LogPacketSize(packet)
				netFlow := packet.NetworkLayer().NetworkFlow()
				tcp := packet.TransportLayer().(*layers.TCP)

				assembler.AssembleWithTimestamp(netFlow, tcp,
					packet.Metadata().Timestamp)
			}
		case err := <-cmd_done:
			if err != nil {
				log.Printf("process done with error = %v\n", err)
			}
			log.Println("Process took: ", time.Now().Sub(start_time))

			log.Println("Waiting for the remaining responses to arrive.")

			/* Wait for a couple of seconds, just enough to get the events
			   handled by the main function. */
			timeout = create_timeout_channel(2)

		case <-ctrlc:
			/* Don't wait. */
			timeout = create_timeout_channel(0)

		case <-timeout:
			reporting, err := NewReporting()
			if err != nil {
				panic(err)
			}

			if err = reporting.Report(); err != nil {
				log.Panic(err)
			}

			os.Exit(0)
		}
	}
}
<commit_msg>Log child's stdout and stderr to console.<commit_after>// Copyright 2014 Lieven Govaerts. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"code.google.com/p/gopacket"
	"code.google.com/p/gopacket/layers"
	"code.google.com/p/gopacket/pcap"
	"code.google.com/p/gopacket/tcpassembly"
	"code.google.com/p/gopacket/tcpassembly/tcpreader"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"time"
)

/* Command line arguments */
var iface = flag.String("i", "en0", "Interface to get packets from")
var logAllPackets = flag.Bool("v", false, "Logs every packet in great detail")
var launchCmd = flag.String("e", "", "Launches the command and logs its traffic")

type interval struct {
	req  *http.Request
	resp *http.Response
}

type BidiStream struct {
	key     uint64
	in, out *TCPStream
}

// TCPStream will handle the actual decoding of http requests and responses.
type TCPStream struct {
	netFlow, tcpFlow gopacket.Flow
	readStream       tcpreader.ReaderStream
	storage          *Storage
	bidikey          uint64
}

/* This reads both HTTP requests and HTTP responses in two separate streams */
func (h *TCPStream) runOut() {
	buf := bufio.NewReader(&h.readStream)
	var reqID int64
	for {
		req, err := http.ReadRequest(buf)
		if err == io.EOF {
			// We must read until we see an EOF... very important!
			return
		} else if err != nil {
			log.Println("Error reading stream", h.netFlow, h.tcpFlow, ":", err)
		} else {
			// bodyBytes := tcpreader.DiscardBytesToEOF(req.Body)
			req.Body.Close()
			err = h.storage.SentRequest(h.bidikey, reqID, time.Now(), req)
			if err != nil {
				log.Println("Error storing request", err)
			}

			reqID++
			//			fmt.Print(".")
			// log.Println("Received request from stream", h.netFlow, h.tcpFlow,
			// 	":", req, "with", bodyBytes, "bytes in request body")
		}
	}
}

func (h *TCPStream) runIn() {
	buf := bufio.NewReader(&h.readStream)
	var reqID int64
	for {

		/* Don't start reading a response if no data is available */
		_, err := buf.Peek(1)
		if err == io.EOF {
			return
		}
		/* Data available, read response */
		resp, err := http.ReadResponse(buf, nil) // TODO: request
		if err == io.EOF {
			// We must read until we see an EOF... very important!
			log.Println("EOF while reading stream", h.netFlow, h.tcpFlow, ":", err)
			return
		} else if err != nil {
			log.Println("Error reading stream", h.netFlow, h.tcpFlow, ":", err)
		} else {
			_, err := tcpreader.DiscardBytesToFirstError(resp.Body)
			if err != nil && err != io.EOF {
				log.Println(err)
			}
			resp.Body.Close()
			err = h.storage.ReceivedResponse(h.bidikey, reqID, time.Now(), resp)
			if err != nil {
				log.Println("Error storing response", err)
			}
			reqID++
			// fmt.Print(".")
			//log.Println("Received response from stream", h.netFlow, h.tcpFlow,
			//	":", resp, "with", bodyBytes, "bytes in response body")
		}

		/* Match the response with the next request */
	}

}

/* httpStreamFactory implements tcpassembly.StreamFactory */
type httpStreamFactory struct {
	bidiStreams map[uint64]*BidiStream
	storage     *Storage
}

func (h *httpStreamFactory) New(netFlow, tcpFlow gopacket.Flow) tcpassembly.Stream {

	/* First the outgoing stream, then the incoming stream */
	key := netFlow.FastHash() ^ tcpFlow.FastHash()

	hstream := &TCPStream{
		netFlow:    netFlow,
		tcpFlow:    tcpFlow,
		readStream: tcpreader.NewReaderStream(),
		storage:    h.storage,
		bidikey:    key,
	}

	bds := h.bidiStreams[key]
	if bds == nil {
		log.Println("reading stream", netFlow, tcpFlow)
		bds = &BidiStream{out: hstream, key: key}
		h.bidiStreams[key] = bds
		// Start a coroutine per stream, to ensure that all data is read from
		// the reader stream
		go hstream.runOut()
	} else {
		bds.in = hstream
		err := h.storage.OpenTCPConnection(key, time.Now())
		if err != nil {
			log.Println("Error storing connection", err)
		}
		// Start a coroutine per stream, to ensure that all data is read from
		// the reader stream
		go hstream.runIn()
	}

	// ReaderStream implements tcpassembly.Stream, so we can return a pointer to it.
	return &hstream.readStream
}

func (h *httpStreamFactory) LogPacketSize(packet gopacket.Packet) {
	netFlow := packet.NetworkLayer().NetworkFlow()
	tcpFlow := packet.TransportLayer().TransportFlow()
	key := netFlow.FastHash() ^ tcpFlow.FastHash()

	ipv4Layer := packet.Layer(layers.LayerTypeIPv4)
	ipv4, _ := ipv4Layer.(*layers.IPv4)

	tcpLayer := packet.Layer(layers.LayerTypeTCP)
	tcp, _ := tcpLayer.(*layers.TCP)

	bds := h.bidiStreams[key]
	if bds == nil || bds.in == nil || bds.out == nil {
		return
	}

	payloadLength := uint32(ipv4.Length - uint16(ipv4.IHL)*4 - uint16(tcp.DataOffset)*4)

	if bds.in.netFlow == netFlow {
		/* incoming */
		err := h.storage.IncomingTCPPacket(key, payloadLength)
		if err != nil {
			panic(err)
		}
	} else {
		/* outgoing */
		err := h.storage.OutgoingTCPPacket(key, payloadLength)
		if err != nil {
			panic(err)
		}
	}
}

type Assembler struct {
	assembler *tcpassembly.Assembler
}

func NewAssembler(streamPool *tcpassembly.StreamPool) *Assembler {
	return &Assembler{tcpassembly.NewAssembler(streamPool)}
}

/* PFF, a lot of abstractions that make things more difficult then they should be! */
func (a *Assembler) AssembleWithTimestamp(netFlow gopacket.Flow, t *layers.TCP,
	timestamp time.Time) {
	a.assembler.AssembleWithTimestamp(netFlow, t, timestamp)
}

func create_process_ended_channel(cmd *exec.Cmd) (cmd_done chan error) {
	cmd_done = make(chan error, 1)
	go func() {
		cmd_done <- cmd.Wait()
	}()
	return cmd_done
}

/* Create a channel and send any CTRL-C signal over it */
func create_ctrl_c_handler() (timeout chan os.Signal) {
	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)
	return ctrlc
}

/* Create a channel that will contain TRUE after T seconds. */
func create_timeout_channel(t time.Duration) (timeout chan bool) {
	timeout = make(chan bool, 1)
	go func() {
		time.Sleep(t * time.Second)
		timeout <- true
	}()
	return timeout
}

func main() {
	flag.Parse()
	log.SetFlags(log.Ltime | log.Lmicroseconds)
	//	log.SetOutput(ioutil.Discard)

	log.Printf("starting capture on interface %q", *iface)

	/* TODO: we need a storage layer per goroutine! */
	// Set up storage layer
	storage, err := NewStorage()
	if err != nil {
		panic(err)
	}

	// Set up assembly
	streamFactory := &httpStreamFactory{bidiStreams: make(map[uint64]*BidiStream),
		storage: storage}
	streamPool := tcpassembly.NewStreamPool(streamFactory)
	assembler := NewAssembler(streamPool)

	// Setup CTRL-C handler channel
	ctrlc := create_ctrl_c_handler()

	// Setup channel that reports all socket kernel events (Mac OS X only)
	netDescSource := NewOSXNetDescSource()
	descriptors := netDescSource.Descriptors()

	// Setup packet capture
	handle, err := pcap.OpenLive(*iface, 1600, true, pcap.BlockForever)
	if err != nil {
		panic(err)
	} else if err := handle.SetBPFFilter("tcp and port 80"); err != nil {
		panic(err)
	}
	log.Println("reading in packets. Press CTRL-C to end and report.")
	packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
	packets := packetSource.Packets()

	// Run the external command
	pid := uint32(0)
	var cmd_done chan error
	var start_time time.Time
	if *launchCmd != "" {
		s := *launchCmd
		//		n := strings.Index(*launchCmd, " ")
		//		fmt.Println("Launching ", s[0:n], " --- ", s[n:])
		//		cmd := exec.Command(s[0:n], strings.Split(s[n:], " "))
		args := strings.Split(s, " ")
		cmd := exec.Command(args[0], args[1:]...)
		start_time = time.Now()
		cmd_stdout, err := cmd.StdoutPipe()
		if err != nil {
			log.Fatal(err)
		}
		err = cmd.Start()
		if err != nil {
			panic(err)
		}
		go io.Copy(os.Stdout, cmd_stdout)
		//		cmd.Stdout = os.Stdout
		//		cmd.Stderr = os.Stderr
		// Create the channel that listens for the end of the command.
		cmd_done = create_process_ended_channel(cmd)
		pid = uint32(cmd.Process.Pid)
	}

	var timeout chan bool
	for {
		select {
		case netDesc := <-descriptors:
			if netDesc.Pid == pid {
				fmt.Println("event received ", netDesc)
			}
		case packet := <-packets:
			if *logAllPackets {
				log.Println(packet)
			}

			if storage.PacketInScope(packet) {
				streamFactory.LogPacketSize(packet)
				netFlow := packet.NetworkLayer().NetworkFlow()
				tcp := packet.TransportLayer().(*layers.TCP)

				assembler.AssembleWithTimestamp(netFlow, tcp,
					packet.Metadata().Timestamp)
			}
		case err := <-cmd_done:
			if err != nil {
				log.Printf("process done with error = %v\n", err)
			}
			log.Println("Process took: ", time.Now().Sub(start_time))

			log.Println("Waiting for the remaining responses to arrive.")

			/* Wait for a couple of seconds, just enough to get the events
			   handled by the main function. */
			timeout = create_timeout_channel(2)

		case <-ctrlc:
			/* Don't wait. */
			timeout = create_timeout_channel(0)

		case <-timeout:
			reporting, err := NewReporting()
			if err != nil {
				panic(err)
			}

			if err = reporting.Report(); err != nil {
				log.Panic(err)
			}

			os.Exit(0)
		}
	}
}
<|endoftext|>


<commit_before>package bnblog

import (
	"encoding/base64"
	"fmt"
	"net/http"
	"strings"
	"text/template"
	"time"

	"github.com/codegangsta/martini"
	"github.com/russross/blackfriday"

	"appengine"
	"appengine/datastore"
)

var PostTemplate = template.Must(template.ParseFiles("public2/pagetempl.html"))
var HomeTemplate = template.Must(template.ParseFiles("public2/hometempl.html"))

type Post struct {
	Author  string
	Content string `datastore:",noindex"`
	Date    time.Time
	Slug    string
	Title   string
}

type PostFormatted struct {
	Author  string
	Content string `datastore:",noindex"`
	Date    string
	Slug    string
	Title   string
}

func init() {
	m := martini.Classic()
	m.Get("/post/:name", ReadPost)
	m.Get("/raw/:name", ReadRawPost)
	m.Post("/admin/new", PublishPost)
	m.Get("/admin/", Admin)
	m.Get("/", ListPosts)
	m.Get("/all", ListPosts)
	m.Get("/rss.xml", GetRSS)
	m.Get("/sitemap.xml", GetSitemap)
	m.Get("/admin/run_gc", Run_GC)
	m.Get("/admin/backup.tar", Producebackup)
	m.Get("/admin/remove/:name", RemovePost)
	m.Post("/admin/uploadfile", UploadFile)

	m.Get("/lessons/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/lessons/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/errors/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/errors/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/posts/errors/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/posts/errors/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/asset/:tag", ReadFile)

	m.Use(func(res http.ResponseWriter, req *http.Request) {
		res.Header().Add("Cache-Control", "public")
		res.Header().Add("X-Served-By", GimmeDC(res, req))
		res.Header().Add("X-Served-For", req.Header.Get("CF-RAY"))
	})

	m.Use(martini.Static("public2"))

	http.Handle("/", m)
}

func MigrateOldURLS(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	http.Redirect(rw, req, fmt.Sprintf("https://blog.benjojo.co.uk/post/%s-%s-%s-%s.md", params["year"], params["month"], params["day"], params["title"]), http.StatusMovedPermanently)
}

func ReadPost(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	// c := appengine.NewContext(r)
	// key := datastore.NewIncompleteKey(c, "Greeting", PostKey(c))
	// _, err := datastore.Put(c, key, &g)

	c := appengine.NewContext(req)
	k := datastore.NewKey(c, "Post", params["name"], 0, nil)
	post := Post{}
	err := datastore.Get(c, k, &post)

	if err != nil {
		if fmt.Sprint(err) == "datastore: no such entity" {
			http.Error(rw, "This blog post cannot be found, Please check your URL", http.StatusNotFound)
			return
		}

		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	postd, _ := base64.StdEncoding.DecodeString(post.Content)
	// post.Content = strings.Replace(string(postd), "\n", "\r\n\r\n", -1)
	post.Content = string(postd)
	output := blackfriday.MarkdownCommon([]byte(post.Content))
	lines := strings.Split(string(postd), "\n")
	layoutData := struct {
		Title   string
		Content string
		Date    string
	}{
		Title:   lines[0],
		Content: string(output),
		Date:    Post.Date.Format("Jan 2 2006"),
	}

	err = PostTemplate.Execute(rw, layoutData)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	}
}

func ReadRawPost(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	c := appengine.NewContext(req)
	k := datastore.NewKey(c, "Post", params["name"], 0, nil)
	post := Post{}
	err := datastore.Get(c, k, &post)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	postd, _ := base64.StdEncoding.DecodeString(post.Content)
	rw.Write(postd)
}

func ListPosts(rw http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)
	q := datastore.NewQuery("Post").Order("-Date").Limit(100)
	posts := make([]Post, 0, 100)

	if _, err := q.GetAll(c, &posts); err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
	FormattedPosts := make([]PostFormatted, 0)

	for _, v := range posts {
		if !strings.HasPrefix(v.Slug, "DRAFT-") {
			newpost := PostFormatted{
				Author:  v.Author,
				Content: v.Content,
				Date:    v.Date.Format("2006-01-02 15:04:05"),
				Slug:    v.Slug,
				Title:   v.Title,
			}
			FormattedPosts = append(FormattedPosts, newpost)
		}
	}

	layoutData := struct {
		Posts []PostFormatted
	}{
		Posts: FormattedPosts,
	}

	err := HomeTemplate.Execute(rw, layoutData)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	}

}
<commit_msg>+ Deploy bug, I love google app engine!<commit_after>package bnblog

import (
	"encoding/base64"
	"fmt"
	"net/http"
	"strings"
	"text/template"
	"time"

	"github.com/codegangsta/martini"
	"github.com/russross/blackfriday"

	"appengine"
	"appengine/datastore"
)

var PostTemplate = template.Must(template.ParseFiles("public2/pagetempl.html"))
var HomeTemplate = template.Must(template.ParseFiles("public2/hometempl.html"))

type Post struct {
	Author  string
	Content string `datastore:",noindex"`
	Date    time.Time
	Slug    string
	Title   string
}

type PostFormatted struct {
	Author  string
	Content string `datastore:",noindex"`
	Date    string
	Slug    string
	Title   string
}

func init() {
	m := martini.Classic()
	m.Get("/post/:name", ReadPost)
	m.Get("/raw/:name", ReadRawPost)
	m.Post("/admin/new", PublishPost)
	m.Get("/admin/", Admin)
	m.Get("/", ListPosts)
	m.Get("/all", ListPosts)
	m.Get("/rss.xml", GetRSS)
	m.Get("/sitemap.xml", GetSitemap)
	m.Get("/admin/run_gc", Run_GC)
	m.Get("/admin/backup.tar", Producebackup)
	m.Get("/admin/remove/:name", RemovePost)
	m.Post("/admin/uploadfile", UploadFile)

	m.Get("/lessons/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/lessons/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/errors/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/errors/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/posts/errors/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/posts/errors/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/asset/:tag", ReadFile)

	m.Use(func(res http.ResponseWriter, req *http.Request) {
		res.Header().Add("Cache-Control", "public")
		res.Header().Add("X-Served-By", GimmeDC(res, req))
		res.Header().Add("X-Served-For", req.Header.Get("CF-RAY"))
	})

	m.Use(martini.Static("public2"))

	http.Handle("/", m)
}

func MigrateOldURLS(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	http.Redirect(rw, req, fmt.Sprintf("https://blog.benjojo.co.uk/post/%s-%s-%s-%s.md", params["year"], params["month"], params["day"], params["title"]), http.StatusMovedPermanently)
}

func ReadPost(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	// c := appengine.NewContext(r)
	// key := datastore.NewIncompleteKey(c, "Greeting", PostKey(c))
	// _, err := datastore.Put(c, key, &g)

	c := appengine.NewContext(req)
	k := datastore.NewKey(c, "Post", params["name"], 0, nil)
	post := Post{}
	err := datastore.Get(c, k, &post)

	if err != nil {
		if fmt.Sprint(err) == "datastore: no such entity" {
			http.Error(rw, "This blog post cannot be found, Please check your URL", http.StatusNotFound)
			return
		}

		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	postd, _ := base64.StdEncoding.DecodeString(post.Content)
	// post.Content = strings.Replace(string(postd), "\n", "\r\n\r\n", -1)
	post.Content = string(postd)
	output := blackfriday.MarkdownCommon([]byte(post.Content))
	lines := strings.Split(string(postd), "\n")
	layoutData := struct {
		Title   string
		Content string
		Date    string
	}{
		Title:   lines[0],
		Content: string(output),
		Date:    post.Date.Format("Jan 2 2006"),
	}

	err = PostTemplate.Execute(rw, layoutData)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	}
}

func ReadRawPost(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	c := appengine.NewContext(req)
	k := datastore.NewKey(c, "Post", params["name"], 0, nil)
	post := Post{}
	err := datastore.Get(c, k, &post)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	postd, _ := base64.StdEncoding.DecodeString(post.Content)
	rw.Write(postd)
}

func ListPosts(rw http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)
	q := datastore.NewQuery("Post").Order("-Date").Limit(100)
	posts := make([]Post, 0, 100)

	if _, err := q.GetAll(c, &posts); err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
	FormattedPosts := make([]PostFormatted, 0)

	for _, v := range posts {
		if !strings.HasPrefix(v.Slug, "DRAFT-") {
			newpost := PostFormatted{
				Author:  v.Author,
				Content: v.Content,
				Date:    v.Date.Format("2006-01-02 15:04:05"),
				Slug:    v.Slug,
				Title:   v.Title,
			}
			FormattedPosts = append(FormattedPosts, newpost)
		}
	}

	layoutData := struct {
		Posts []PostFormatted
	}{
		Posts: FormattedPosts,
	}

	err := HomeTemplate.Execute(rw, layoutData)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/nelsam/hel/mocks"
	"github.com/nelsam/hel/packages"
	"github.com/nelsam/hel/types"
	"github.com/spf13/cobra"
)

var (
	cmd           *cobra.Command
	goimportsPath string
)

func init() {
	output, err := exec.Command("which", "goimports").Output()
	if err != nil {
		fmt.Println("Could not locate goimports: ", err.Error())
		fmt.Println("If goimports is not installed, please install it somewhere in your path.  " +
			"See https://godoc.org/golang.org/x/tools/cmd/goimports.")
		os.Exit(1)
	}
	goimportsPath = strings.TrimSpace(string(output))

	cmd = &cobra.Command{
		Use:   "hel",
		Short: "A mock generator for Go",
		Long: "A simple mock generator.  The origin of the name is the Norse goddess, Hel, " +
			"who guards over the souls of those unworthy to enter Valhalla.  You can probably " +
			"guess how much I like mocks.",
		Run: func(cmd *cobra.Command, args []string) {
			if len(args) > 0 {
				fmt.Println("Invalid usage.\n")
				err := cmd.Help()
				if err != nil {
					panic(err)
				}
				os.Exit(1)
			}
			packagePatterns, err := cmd.Flags().GetStringSlice("package")
			if err != nil {
				panic(err)
			}
			typePatterns, err := cmd.Flags().GetStringSlice("type")
			if err != nil {
				panic(err)
			}
			outputName, err := cmd.Flags().GetString("output")
			if err != nil {
				panic(err)
			}
			chanSize, err := cmd.Flags().GetInt("chan-size")
			if err != nil {
				panic(err)
			}
			fmt.Printf("Generating mocks for packages %v", packagePatterns)
			var packageList []packages.Package
			progress(func() {
				packageList = packages.Load(packagePatterns...)
			})
			fmt.Print("\n")
			fmt.Printf("Generating mocks for types %v", typePatterns)
			var typeList []types.Types
			progress(func() {
				typeList = types.Load(packageList...) //.Filter(typePatterns...)
			})
			fmt.Print("\n")
			fmt.Printf("Generating mocks in %s", outputName)
			progress(func() {
				for _, types := range typeList {
					mockPath, err := makeMocks(types, outputName, chanSize)
					if err != nil {
						panic(err)
					}
					if err = exec.Command(goimportsPath, "-w", mockPath).Run(); err != nil {
						panic(err)
					}
				}
			})
			fmt.Print("\n")
		},
	}
	cmd.Flags().StringSliceP("package", "p", []string{"."}, "The package(s) to generate mocks for.")
	cmd.Flags().StringSliceP("type", "t", []string{}, "The type(s) to generate mocks for.  If no types "+
		"are passed in, all exported interface types will be generated.")
	cmd.Flags().StringP("output", "o", "helheim_test.go", "The file to write generated mocks to.  Since hel does "+
		"not generate exported types, this file will be saved directly in all packages with generated mocks.  "+
		"Also note that, since the types are not exported, you will want the file to end in '_test.go'.")
	cmd.Flags().IntP("chan-size", "s", 100, "The size of channels used for method calls.")
}

func makeMocks(types types.Types, fileName string, chanSize int) (filePath string, err error) {
	mocks, err := mocks.Generate(types)
	if err != nil {
		return "", err
	}
	if len(mocks) == 0 {
		return "", nil
	}
	filePath = filepath.Join(types.Dir(), fileName)
	f, err := os.Create(filePath)
	if err != nil {
		return "", err
	}
	defer f.Close()
	return filePath, mocks.Output(types.TestPackage(), chanSize, f)
}

func progress(f func()) {
	stop, done := make(chan struct{}), make(chan struct{})
	defer func() {
		close(stop)
		<-done
	}()
	go showProgress(stop, done)
	f()
}

func showProgress(stop <-chan struct{}, done chan<- struct{}) {
	defer close(done)
	ticker := time.NewTicker(time.Second / 2)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			fmt.Print(".")
		case <-stop:
			return
		}
	}
}

func main() {
	cmd.Execute()
}
<commit_msg>Support cleaner types from latest cleanup in main<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"github.com/nelsam/hel/mocks"
	"github.com/nelsam/hel/packages"
	"github.com/nelsam/hel/types"
	"github.com/spf13/cobra"
)

var (
	cmd           *cobra.Command
	goimportsPath string
)

func init() {
	output, err := exec.Command("which", "goimports").Output()
	if err != nil {
		fmt.Println("Could not locate goimports: ", err.Error())
		fmt.Println("If goimports is not installed, please install it somewhere in your path.  " +
			"See https://godoc.org/golang.org/x/tools/cmd/goimports.")
		os.Exit(1)
	}
	goimportsPath = strings.TrimSpace(string(output))

	cmd = &cobra.Command{
		Use:   "hel",
		Short: "A mock generator for Go",
		Long: "A simple mock generator.  The origin of the name is the Norse goddess, Hel, " +
			"who guards over the souls of those unworthy to enter Valhalla.  You can probably " +
			"guess how much I like mocks.",
		Run: func(cmd *cobra.Command, args []string) {
			if len(args) > 0 {
				fmt.Println("Invalid usage.\n")
				err := cmd.Help()
				if err != nil {
					panic(err)
				}
				os.Exit(1)
			}
			packagePatterns, err := cmd.Flags().GetStringSlice("package")
			if err != nil {
				panic(err)
			}
			typePatterns, err := cmd.Flags().GetStringSlice("type")
			if err != nil {
				panic(err)
			}
			outputName, err := cmd.Flags().GetString("output")
			if err != nil {
				panic(err)
			}
			chanSize, err := cmd.Flags().GetInt("chan-size")
			if err != nil {
				panic(err)
			}
			fmt.Printf("Loading directories matching pattern"+pluralize(packagePatterns, "", "s")+" %v", packagePatterns)
			var dirList []packages.Dir
			progress(func() {
				dirList = packages.Load(packagePatterns...)
			})
			fmt.Print("\n")
   fmt.Println("Found directories:")
   for _, dir := range dirList {
    fmt.Println("  "+dir.Path())
   }
   fmt.Print("\n")

			fmt.Printf("Loading interface types in matching directories")
			var typeList []types.Types
			progress(func() {
				godirs := make([]types.GoDir, 0, len(dirList))
				for _, dir := range dirList {
					godirs = append(godirs, dir)
				}
				typeList = types.Load(godirs...) //.Filter(typePatterns...)
    _ = typePatterns
			})
			fmt.Print("\n\n")

			fmt.Printf("Generating mocks in output file %s", outputName)
			progress(func() {
				for _, types := range typeList {
					mockPath, err := makeMocks(types, outputName, chanSize)
					if err != nil {
						panic(err)
					}
					if err = exec.Command(goimportsPath, "-w", mockPath).Run(); err != nil {
						panic(err)
					}
				}
			})
			fmt.Print("\n")
		},
	}
	cmd.Flags().StringSliceP("package", "p", []string{"."}, "The package(s) to generate mocks for.")
	cmd.Flags().StringSliceP("type", "t", []string{}, "The type(s) to generate mocks for.  If no types "+
		"are passed in, all exported interface types will be generated.")
	cmd.Flags().StringP("output", "o", "helheim_test.go", "The file to write generated mocks to.  Since hel does "+
		"not generate exported types, this file will be saved directly in all packages with generated mocks.  "+
		"Also note that, since the types are not exported, you will want the file to end in '_test.go'.")
	cmd.Flags().IntP("chan-size", "s", 100, "The size of channels used for method calls.")
}

func makeMocks(types types.Types, fileName string, chanSize int) (filePath string, err error) {
	mocks, err := mocks.Generate(types)
	if err != nil {
		return "", err
	}
	if len(mocks) == 0 {
		return "", nil
	}
	filePath = filepath.Join(types.Dir(), fileName)
	f, err := os.Create(filePath)
	if err != nil {
		return "", err
	}
	defer f.Close()
	return filePath, mocks.Output(types.TestPackage(), chanSize, f)
}

func progress(f func()) {
	stop, done := make(chan struct{}), make(chan struct{})
	defer func() {
		close(stop)
		<-done
	}()
	go showProgress(stop, done)
	f()
}

func showProgress(stop <-chan struct{}, done chan<- struct{}) {
	defer close(done)
	ticker := time.NewTicker(time.Second / 2)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			fmt.Print(".")
		case <-stop:
			return
		}
	}
}

type lengther interface {
	Len() int
}

func pluralize(values interface{}, singular, plural string) string {
	length := findLength(values)
	if length == 1 {
		return singular
	}
	return plural
}

func findLength(values interface{}) int {
	if lengther, ok := values.(lengther); ok {
		return lengther.Len()
	}
	return reflect.ValueOf(values).Len()
}

func main() {
	cmd.Execute()
}
<|endoftext|>


<commit_before>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"bufio"
	"crypto/tls"
	"crypto/x509"
	"flag"
	"fmt"
	"github.com/nmeum/marvin/irc"
	"github.com/nmeum/marvin/modules"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/user"
	"path/filepath"
	"strings"
	"time"
)

const (
	appName  = "marvin"
	appUsage = "[options] CHANNEL..."
)

var (
	nick = flag.String("n", "marvin", "nickname")
	name = flag.String("r", "Marvin Bot", "realname")
	host = flag.String("h", "irc.hackint.eu", "host")
	cert = flag.String("c", "", "certificate")
	verb = flag.Bool("v", false, "verbose output")
	port = flag.Int("p", 6697, "port")
)

func main() {
	flag.Parse()
	logger := log.New(os.Stderr, "ERROR: ", 0)
	if flag.NArg() < 1 {
		logger.Fatalf("USAGE: %s %s", appName, appUsage)
	}

	conn, err := connect("tcp", fmt.Sprintf("%s:%d", *host, *port))
	if err != nil {
		logger.Fatal(err)
	}
	defer conn.Close()

	errChan := make(chan error)
	go func() {
		for err := range errChan {
			logger.Println(err)
		}
	}()

	for {
		ircBot, err := setup(conn, flag.Args())
		if err != nil {
			logger.Println(err)
		}

		reader := bufio.NewReader(conn)
		for {
			line, err := reader.ReadString('\n')
			if err != nil {
				break
			}

			line = strings.Trim(line, "\n")
			line = strings.Trim(line, "\r")

			if *verb {
				fmt.Println(line)
			}

			ircBot.Handle(line, errChan)
		}

		conn = reconnect(conn)
	}
}

func setup(conn net.Conn, channels []string) (client *irc.Client, err error) {
	client = irc.NewClient(&conn)
	client.CmdHook("ping", func(c *irc.Client, m irc.Message) error {
		return c.Write("PONG %s", m.Data)
	})

	client.CmdHook("001", func(c *irc.Client, m irc.Message) error {
		time.Sleep(3 * time.Second)
		return c.Write("JOIN %s", strings.Join(channels, ","))
	})

	client.Write("USER %s %s * :%s", *nick, *host, *name)
	client.Write("NICK %s", *nick)

	if err = initializeModules(client); err != nil {
		return
	}

	return
}

func initializeModules(c *irc.Client) error {
	config := os.Getenv("XDG_CONFIG_HOME")
	if len(config) <= 0 {
		user, err := user.Current()
		if err != nil {
			return err
		}

		config = filepath.Join(user.HomeDir, ".config")
	}

	moduleSet := modules.NewModuleSet(c, filepath.Join(config, appName))
	for _, fn := range moduleInits {
		fn(moduleSet)
	}

	return moduleSet.LoadAll()
}

func connect(network, address string) (conn net.Conn, err error) {
	if len(*cert) >= 1 {
		certFile, err := ioutil.ReadFile(*cert)
		if err != nil {
			return nil, err
		}

		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(certFile)

		config := &tls.Config{RootCAs: caCertPool}
		return tls.Dial(network, address, config)
	}

	return net.Dial(network, address)
}

func reconnect(c net.Conn) (conn net.Conn) {
	addr := c.RemoteAddr()
	c.Close()

	var err error
	for i := 1; ; i++ {
		conn, err = connect(addr.Network(), addr.String())
		if err == nil {
			break
		}

		time.Sleep(time.Duration(i*3) * time.Second)
	}

	return
}
<commit_msg>Print ReadString error (if any)<commit_after>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"bufio"
	"crypto/tls"
	"crypto/x509"
	"flag"
	"fmt"
	"github.com/nmeum/marvin/irc"
	"github.com/nmeum/marvin/modules"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/user"
	"path/filepath"
	"strings"
	"time"
)

const (
	appName  = "marvin"
	appUsage = "[options] CHANNEL..."
)

var (
	nick = flag.String("n", "marvin", "nickname")
	name = flag.String("r", "Marvin Bot", "realname")
	host = flag.String("h", "irc.hackint.eu", "host")
	cert = flag.String("c", "", "certificate")
	verb = flag.Bool("v", false, "verbose output")
	port = flag.Int("p", 6697, "port")
)

func main() {
	flag.Parse()
	logger := log.New(os.Stderr, "ERROR: ", 0)
	if flag.NArg() < 1 {
		logger.Fatalf("USAGE: %s %s", appName, appUsage)
	}

	conn, err := connect("tcp", fmt.Sprintf("%s:%d", *host, *port))
	if err != nil {
		logger.Fatal(err)
	}
	defer conn.Close()

	errChan := make(chan error)
	go func() {
		for err := range errChan {
			logger.Println(err)
		}
	}()

	for {
		ircBot, err := setup(conn, flag.Args())
		if err != nil {
			logger.Println(err)
		}

		reader := bufio.NewReader(conn)
		for {
			line, err := reader.ReadString('\n')
			if err != nil {
				logger.Println(err)
				break
			}

			line = strings.Trim(line, "\n")
			line = strings.Trim(line, "\r")

			if *verb {
				fmt.Println(line)
			}

			ircBot.Handle(line, errChan)
		}

		conn = reconnect(conn)
	}
}

func setup(conn net.Conn, channels []string) (client *irc.Client, err error) {
	client = irc.NewClient(&conn)
	client.CmdHook("ping", func(c *irc.Client, m irc.Message) error {
		return c.Write("PONG %s", m.Data)
	})

	client.CmdHook("001", func(c *irc.Client, m irc.Message) error {
		time.Sleep(3 * time.Second)
		return c.Write("JOIN %s", strings.Join(channels, ","))
	})

	client.Write("USER %s %s * :%s", *nick, *host, *name)
	client.Write("NICK %s", *nick)

	if err = initializeModules(client); err != nil {
		return
	}

	return
}

func initializeModules(c *irc.Client) error {
	config := os.Getenv("XDG_CONFIG_HOME")
	if len(config) <= 0 {
		user, err := user.Current()
		if err != nil {
			return err
		}

		config = filepath.Join(user.HomeDir, ".config")
	}

	moduleSet := modules.NewModuleSet(c, filepath.Join(config, appName))
	for _, fn := range moduleInits {
		fn(moduleSet)
	}

	return moduleSet.LoadAll()
}

func connect(network, address string) (conn net.Conn, err error) {
	if len(*cert) >= 1 {
		certFile, err := ioutil.ReadFile(*cert)
		if err != nil {
			return nil, err
		}

		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(certFile)

		config := &tls.Config{RootCAs: caCertPool}
		return tls.Dial(network, address, config)
	}

	return net.Dial(network, address)
}

func reconnect(c net.Conn) (conn net.Conn) {
	addr := c.RemoteAddr()
	c.Close()

	var err error
	for i := 1; ; i++ {
		conn, err = connect(addr.Network(), addr.String())
		if err == nil {
			break
		}

		time.Sleep(time.Duration(i*3) * time.Second)
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
  "crypto"
  "encoding/base64"
  "fmt"
  "io"
  "io/ioutil"
  "math/big"
  "os"
  "path/filepath"

  "github.com/gibheer/pki"
)

var (
  EmptyByteArray = make([]byte, 0)
)

func main() {
  if len(os.Args) == 1 {
    crash_with_help(1, "No module selected!")
  }
  switch os.Args[1] {
  case "create-private":   create_private_key()
  case "create-public":    create_public_key()
  case "sign-input":       sign_input()
  case "verify-signature": verify_input()
  case "create-cert-sign": create_sign_request()
  case "create-cert":      create_cert()
  case "help":             print_modules()
  case "--help":             print_modules()
//  case "info":             info_on_file()
  default: crash_with_help(1, "Command not supported!")
  }
}

// create a private key
func create_private_key() {
  fs := NewFlags("create-private")
  fs.AddOutput()
  fs.AddPrivateKeyGenerationFlags()
  err := fs.Parse(program_args())
  if err != nil { os.Exit(2) }

  var pk pki.Pemmer
  switch fs.Flags.PrivateKeyGenerationFlags.Type {
    case "ecdsa": pk, err = pki.NewPrivateKeyEcdsa(fs.Flags.PrivateKeyGenerationFlags.Curve)
    case "rsa":   pk, err = pki.NewPrivateKeyRsa(fs.Flags.PrivateKeyGenerationFlags.Size)
  }
  if err != nil { os.Exit(2) }
  marsh_pem, err := pk.MarshalPem()
  if err != nil { os.Exit(2) }
  _, err = marsh_pem.WriteTo(fs.Flags.Output)
  if err != nil { os.Exit(2) }
}

// create a public key derived from a private key
func create_public_key() {
  fs := NewFlags("create-public")
  fs.AddPrivateKey()
  fs.AddOutput()
  err := fs.Parse(program_args())
  if err != nil { os.Exit(2) }

  var pub_key pki.Pemmer
  pub_key = fs.Flags.PrivateKey.Public()
  marsh_pem, err := pub_key.MarshalPem()
  if err != nil { os.Exit(2) }
  _, err = marsh_pem.WriteTo(fs.Flags.Output)
  if err != nil { os.Exit(2) }
}

// sign a message using he private key
func sign_input() {
  fs := NewFlags("sign-input")
  fs.AddPrivateKey()
  fs.AddInput()
  fs.AddOutput()
  err := fs.Parse(program_args())
  if err != nil { os.Exit(2) }

  message, err := ioutil.ReadAll(fs.Flags.Input)
  if err != nil { crash_with_help(2, "Error reading input: %s", err) }
  signature, err := fs.Flags.PrivateKey.Sign(message, crypto.SHA256)
  if err != nil { crash_with_help(2, "Could not compute signature: %s", err) }
  _, err = io.WriteString(fs.Flags.Output, base64.StdEncoding.EncodeToString(signature))
  if err != nil { crash_with_help(2, "Could not write to output: %s", err) }

  // if we print to stderr, send a final line break to make the output nice
  if fs.Flags.Output == os.Stdout {
    // we can ignore the result, as either Stdout did work or not
    _, _ = io.WriteString(fs.Flags.Output, "\n")
  }
}

// verify a message using a signature and a public key
func verify_input() {
  fs := NewFlags("sign-input")
  fs.AddPublicKey()
  fs.AddInput()
  fs.AddOutput()
  fs.AddSignature()
  err := fs.Parse(program_args())
  if err != nil { os.Exit(2) }

  signature := fs.Flags.Signature
  message, err := ioutil.ReadAll(fs.Flags.Input)
  if err != nil { crash_with_help(2, "Error reading input: %s", err) }
  valid, err := fs.Flags.PublicKey.Verify(message, signature, crypto.SHA256)
  if err != nil { crash_with_help(2, "Could not verify message with signature: %s", err) }
  if valid {
    fmt.Println("valid")
    os.Exit(0)
  }
  fmt.Println("invalid")
  os.Exit(1)
}

// create a certificate sign request
func create_sign_request() {
  fs := NewFlags("create-cert-sign")
  fs.AddPrivateKey()
  fs.AddOutput()
  fs.AddCertificateFields()
  fs.Parse(program_args())

  csr, err := fs.Flags.CertificateData.ToCertificateRequest(fs.Flags.PrivateKey)
  if err != nil { crash_with_help(2, "Could not create certificate sign request: %s", err) }
  pem_block, err := csr.MarshalPem()
  if err != nil { crash_with_help(2, "Could not covnert to pem: %s", err) }
  _, err = pem_block.WriteTo(fs.Flags.Output)
  if err != nil { crash_with_help(2, "Encoding didn't work: %s", err) }
}

func create_cert() {
  fs := NewFlags("create-cert")
  fs.AddPrivateKey()
  fs.AddCSR()
  fs.AddOutput()
  fs.Parse(program_args())

  // TODO implement flags for all certificate options
  cert_opts := pki.CertificateOptions{}
  cert_opts.SerialNumber = big.NewInt(1)
  cert, err := fs.Flags.CertificateSignRequest.ToCertificate(
    fs.Flags.PrivateKey,
    cert_opts,
    nil,
  )
  if err != nil { crash_with_help(2, "Error generating certificate: %s", err) }
  pem_block, err := cert.MarshalPem()
  if err != nil { crash_with_help(2, "Error converting to pem: %s", err) }
  _, err = pem_block.WriteTo(fs.Flags.Output)
  if err != nil { crash_with_help(2, "Output didn't work: %s", err) }
}

// print the module help
func print_modules() {
  fmt.Printf(`Usage: %s command args
where 'command' is one of:
    create-private    create a new private key
    create-public     create a public key from a private one
    sign-input        sign a message with a private key
    verify-signature  verify a signature
    create-cert-sign  create a new certificate sign request
    create-cert       sign a certificate request
    help              show this help
    info              get info on a file
`, filepath.Base(os.Args[0]))
  fmt.Println()
}

// crash and provide a helpful message
func crash_with_help(code int, message string, args ...interface{}) {
  fmt.Fprintf(os.Stderr, message + "\n", args...)
  print_modules()
  os.Exit(code)
}

// return the arguments to the program
func program_args() []string {
  return os.Args[2:]
}
<commit_msg>fixes small typo<commit_after>package main

import (
  "crypto"
  "encoding/base64"
  "fmt"
  "io"
  "io/ioutil"
  "math/big"
  "os"
  "path/filepath"

  "github.com/gibheer/pki"
)

var (
  EmptyByteArray = make([]byte, 0)
)

func main() {
  if len(os.Args) == 1 {
    crash_with_help(1, "No module selected!")
  }
  switch os.Args[1] {
  case "create-private":   create_private_key()
  case "create-public":    create_public_key()
  case "sign-input":       sign_input()
  case "verify-signature": verify_input()
  case "create-cert-sign": create_sign_request()
  case "create-cert":      create_cert()
  case "help":             print_modules()
  case "--help":           print_modules()
//  case "info":             info_on_file()
  default: crash_with_help(1, "Command not supported!")
  }
}

// create a private key
func create_private_key() {
  fs := NewFlags("create-private")
  fs.AddOutput()
  fs.AddPrivateKeyGenerationFlags()
  err := fs.Parse(program_args())
  if err != nil { os.Exit(2) }

  var pk pki.Pemmer
  switch fs.Flags.PrivateKeyGenerationFlags.Type {
    case "ecdsa": pk, err = pki.NewPrivateKeyEcdsa(fs.Flags.PrivateKeyGenerationFlags.Curve)
    case "rsa":   pk, err = pki.NewPrivateKeyRsa(fs.Flags.PrivateKeyGenerationFlags.Size)
  }
  if err != nil { os.Exit(2) }
  marsh_pem, err := pk.MarshalPem()
  if err != nil { os.Exit(2) }
  _, err = marsh_pem.WriteTo(fs.Flags.Output)
  if err != nil { os.Exit(2) }
}

// create a public key derived from a private key
func create_public_key() {
  fs := NewFlags("create-public")
  fs.AddPrivateKey()
  fs.AddOutput()
  err := fs.Parse(program_args())
  if err != nil { os.Exit(2) }

  var pub_key pki.Pemmer
  pub_key = fs.Flags.PrivateKey.Public()
  marsh_pem, err := pub_key.MarshalPem()
  if err != nil { os.Exit(2) }
  _, err = marsh_pem.WriteTo(fs.Flags.Output)
  if err != nil { os.Exit(2) }
}

// sign a message using he private key
func sign_input() {
  fs := NewFlags("sign-input")
  fs.AddPrivateKey()
  fs.AddInput()
  fs.AddOutput()
  err := fs.Parse(program_args())
  if err != nil { os.Exit(2) }

  message, err := ioutil.ReadAll(fs.Flags.Input)
  if err != nil { crash_with_help(2, "Error reading input: %s", err) }
  signature, err := fs.Flags.PrivateKey.Sign(message, crypto.SHA256)
  if err != nil { crash_with_help(2, "Could not compute signature: %s", err) }
  _, err = io.WriteString(fs.Flags.Output, base64.StdEncoding.EncodeToString(signature))
  if err != nil { crash_with_help(2, "Could not write to output: %s", err) }

  // if we print to stderr, send a final line break to make the output nice
  if fs.Flags.Output == os.Stdout {
    // we can ignore the result, as either Stdout did work or not
    _, _ = io.WriteString(fs.Flags.Output, "\n")
  }
}

// verify a message using a signature and a public key
func verify_input() {
  fs := NewFlags("sign-input")
  fs.AddPublicKey()
  fs.AddInput()
  fs.AddOutput()
  fs.AddSignature()
  err := fs.Parse(program_args())
  if err != nil { os.Exit(2) }

  signature := fs.Flags.Signature
  message, err := ioutil.ReadAll(fs.Flags.Input)
  if err != nil { crash_with_help(2, "Error reading input: %s", err) }
  valid, err := fs.Flags.PublicKey.Verify(message, signature, crypto.SHA256)
  if err != nil { crash_with_help(2, "Could not verify message with signature: %s", err) }
  if valid {
    fmt.Println("valid")
    os.Exit(0)
  }
  fmt.Println("invalid")
  os.Exit(1)
}

// create a certificate sign request
func create_sign_request() {
  fs := NewFlags("create-cert-sign")
  fs.AddPrivateKey()
  fs.AddOutput()
  fs.AddCertificateFields()
  fs.Parse(program_args())

  csr, err := fs.Flags.CertificateData.ToCertificateRequest(fs.Flags.PrivateKey)
  if err != nil { crash_with_help(2, "Could not create certificate sign request: %s", err) }
  pem_block, err := csr.MarshalPem()
  if err != nil { crash_with_help(2, "Could not covnert to pem: %s", err) }
  _, err = pem_block.WriteTo(fs.Flags.Output)
  if err != nil { crash_with_help(2, "Encoding didn't work: %s", err) }
}

func create_cert() {
  fs := NewFlags("create-cert")
  fs.AddPrivateKey()
  fs.AddCSR()
  fs.AddOutput()
  fs.Parse(program_args())

  // TODO implement flags for all certificate options
  cert_opts := pki.CertificateOptions{}
  cert_opts.SerialNumber = big.NewInt(1)
  cert, err := fs.Flags.CertificateSignRequest.ToCertificate(
    fs.Flags.PrivateKey,
    cert_opts,
    nil,
  )
  if err != nil { crash_with_help(2, "Error generating certificate: %s", err) }
  pem_block, err := cert.MarshalPem()
  if err != nil { crash_with_help(2, "Error converting to pem: %s", err) }
  _, err = pem_block.WriteTo(fs.Flags.Output)
  if err != nil { crash_with_help(2, "Output didn't work: %s", err) }
}

// print the module help
func print_modules() {
  fmt.Printf(`Usage: %s command args
where 'command' is one of:
    create-private    create a new private key
    create-public     create a public key from a private one
    sign-input        sign a message with a private key
    verify-signature  verify a signature
    create-cert-sign  create a new certificate sign request
    create-cert       sign a certificate request
    help              show this help
    info              get info on a file
`, filepath.Base(os.Args[0]))
  fmt.Println()
}

// crash and provide a helpful message
func crash_with_help(code int, message string, args ...interface{}) {
  fmt.Fprintf(os.Stderr, message + "\n", args...)
  print_modules()
  os.Exit(code)
}

// return the arguments to the program
func program_args() []string {
  return os.Args[2:]
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/nbari/violetear"
	"log"
	"net/http"
)

var version, githash string

func catchAll(w http.ResponseWriter, r *http.Request) {
	html := `

		<meta http-equiv="refresh" content="3" />

change me

	%s`
	fmt.Fprintf(w, html, version)
}

func main() {
	router := violetear.New()
	router.LogRequests = true

	router.HandleFunc("*", catchAll)

	log.Fatal(http.ListenAndServe(":8000", router))
}
<commit_msg>	modified:   main.go<commit_after>package main

import (
	"fmt"
	"github.com/nbari/violetear"
	"log"
	"net/http"
)

var version, githash string

func catchAll(w http.ResponseWriter, r *http.Request) {
	html := `

		<meta http-equiv="refresh" content="3" />

time

	%s`
	fmt.Fprintf(w, html, version)
}

func main() {
	router := violetear.New()
	router.LogRequests = true

	router.HandleFunc("*", catchAll)

	log.Fatal(http.ListenAndServe(":8000", router))
}
<|endoftext|>