<commit_before>package main

import (
	"fmt"
	"math/rand"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/honeycombio/libhoney-go"
	flag "github.com/jessevdk/go-flags"

	"github.com/honeycombio/honeytail/parsers/htjson"
	"github.com/honeycombio/honeytail/parsers/mongodb"
	"github.com/honeycombio/honeytail/parsers/mysql"
	"github.com/honeycombio/honeytail/parsers/nginx"
	"github.com/honeycombio/honeytail/tail"
)

// BuildID is set by Travis CI
var BuildID string

// internal version identifier
var version string

var validParsers = []string{
	"nginx",
	"mongo",
	"json",
	"mysql",
}

// GlobalOptions has all the top level CLI flags that honeytail supports
type GlobalOptions struct {
	APIHost string `hidden:"true" long:"api_host" description:"Host for the Honeycomb API" default:"https://api.honeycomb.io/"`

	SampleRate     uint `short:"r" long:"samplerate" description:"Only send 1 / N log lines" default:"1"`
	NumSenders     uint `short:"P" long:"poolsize" description:"Number of concurrent connections to open to Honeycomb" default:"10"`
	Debug          bool `long:"debug" description:"Print debugging output"`
	StatusInterval uint `long:"status_interval" description:"how frequently, in seconds, to print out summary info" default:"60"`
	BackOff        bool `long:"backoff" description:"When rate limited by the API, back off and retry sending failed events. Otherwise failed events are dropped."`

	ScrubFields  []string `long:"scrub_field" description:"for the field listed, apply a one-way hash to the field content. May be specified multiple times"`
	DropFields   []string `long:"drop_field" description:"do not send the field to Honeycomb. May be specified multiple times"`
	AddFields    []string `long:"add_field" description:"add the field to every event. Field should be key=val. May be specified multiple times"`
	RequestShape []string `long:"request_shape" description:"identify a field that contains an HTTP request of the form 'METHOD /path HTTP/1.x'. Break apart that field into subfields that contain components. May be specified multiple times. Defaults to 'request' when using the nginx parser"`
	ShapePrefix  string   `long:"shape_prefix" description:"prefix to use on fields generated from request_shape to prevent field collision"`

	Reqs  RequiredOptions `group:"Required Options"`
	Modes OtherModes      `group:"Other Modes"`

	Tail tail.TailOptions `group:"Tail Options" namespace:"tail"`

	Nginx nginx.Options   `group:"Nginx Parser Options" namespace:"nginx"`
	JSON  htjson.Options  `group:"JSON Parser Options" namespace:"json"`
	MySQL mysql.Options   `group:"MySQL Parser Options" namespace:"mysql"`
	Mongo mongodb.Options `group:"MongoDB Parser Options" namespace:"mongo"`
}

type RequiredOptions struct {
	ParserName string   `short:"p" long:"parser" description:"Parser module to use. Use --list to list available options."`
	WriteKey   string   `short:"k" long:"writekey" description:"Team write key"`
	LogFiles   []string `short:"f" long:"file" description:"Log file(s) to parse. Use '-' for STDIN, use this flag multiple times to tail multiple files, or use a glob (/path/to/foo-*.log)"`
	Dataset    string   `short:"d" long:"dataset" description:"Name of the dataset"`
}

type OtherModes struct {
	Help        bool `short:"h" long:"help" description:"Show this help message"`
	ListParsers bool `short:"l" long:"list" description:"List available parsers"`
	Version     bool `short:"V" long:"version" description:"Show version"`

	WriteManPage bool `hidden:"true" long:"write-man-page" description:"Write out a man page"`
}

func main() {
	var options GlobalOptions
	flagParser := flag.NewParser(&options, flag.PrintErrors)
	flagParser.Usage = "-p <parser> -k <writekey> -f </path/to/logfile> -d <mydata> [optional arguments]"
	if extraArgs, err := flagParser.Parse(); err != nil || len(extraArgs) != 0 {
		fmt.Println("Error: failed to parse the command line.")
		if err != nil {
			fmt.Printf("\t%s\n", err)
		} else {
			fmt.Printf("\tUnexpected extra arguments: %s\n", strings.Join(extraArgs, " "))
		}
		usage()
		os.Exit(1)
	}
	rand.Seed(time.Now().UnixNano())

	if options.Debug {
		logrus.SetLevel(logrus.DebugLevel)
	}

	setVersion()
	handleOtherModes(flagParser, options)
	addParserDefaultOptions(&options)
	sanityCheckOptions(&options)

	run(options)
}

// setVersion sets the internal version ID and updates libhoney's user-agent
func setVersion() {
	if BuildID == "" {
		version = "dev"
	} else {
		version = BuildID
	}
	libhoney.UserAgentAddition = fmt.Sprintf("honeytail/%s", version)
}

// handleOtherModes takse care of all flags that say we should just do something
// and exit rather than actually parsing logs
func handleOtherModes(fp *flag.Parser, options GlobalOptions) {
	if options.Modes.Version {
		fmt.Println("Honeytail version", version)
		os.Exit(0)
	}
	if options.Modes.Help {
		fp.WriteHelp(os.Stdout)
		fmt.Println("")
		os.Exit(0)
	}
	if options.Modes.WriteManPage {
		fp.WriteManPage(os.Stdout)
		os.Exit(0)
	}

	if options.Modes.ListParsers {
		fmt.Println("Available parsers:", strings.Join(validParsers, ", "))
		os.Exit(0)
	}
}

func addParserDefaultOptions(options *GlobalOptions) {
	switch {
	case options.Reqs.ParserName == "nginx":
		// automatically normalize the request when using the nginx parser
		options.RequestShape = append(options.RequestShape, "request")
	}
}

func sanityCheckOptions(options *GlobalOptions) {
	switch {
	case options.Reqs.ParserName == "":
		fmt.Println("parser required.")
		usage()
		os.Exit(1)
	case options.Reqs.WriteKey == "" || options.Reqs.WriteKey == "NULL":
		fmt.Println("write key required.")
		usage()
		os.Exit(1)
	case len(options.Reqs.LogFiles) == 0:
		fmt.Println("log file name or '-' required.")
		usage()
		os.Exit(1)
	case options.Reqs.Dataset == "":
		fmt.Println("dataset name required.")
		usage()
		os.Exit(1)
	case options.Tail.ReadFrom == "end" && options.Tail.Stop:
		fmt.Println("Reading from the end and stopping when we get there. Zero lines to process. Ok, all done! ;)")
		usage()
		os.Exit(1)
	case len(options.Reqs.LogFiles) > 1 && options.Tail.StateFile != "":
		fmt.Println("Statefile can not be set when tailing from multiple files.")
		usage()
		os.Exit(1)
	case options.Tail.StateFile != "":
		files, err := filepath.Glob(options.Reqs.LogFiles[0])
		if err != nil {
			fmt.Printf("Trying to glob log file %s failed: %+v\n",
				options.Reqs.LogFiles[0], err)
			usage()
			os.Exit(1)
		}
		if len(files) > 1 {
			fmt.Println("Statefile can not be set when tailing from multiple files.")
			usage()
			os.Exit(1)
		}
	}
}

func usage() {
	fmt.Print(`
Usage: honeytail -p <parser> -k <writekey> -f </path/to/logfile> -d <mydata> [optional arguments]

For even more detail on required and optional parameters, run
honeytail --help
`)
}
<commit_msg>adding code to validate the writekey before even trying to parse log … (#10)<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/honeycombio/libhoney-go"
	flag "github.com/jessevdk/go-flags"

	"github.com/honeycombio/honeytail/parsers/htjson"
	"github.com/honeycombio/honeytail/parsers/mongodb"
	"github.com/honeycombio/honeytail/parsers/mysql"
	"github.com/honeycombio/honeytail/parsers/nginx"
	"github.com/honeycombio/honeytail/tail"
)

// BuildID is set by Travis CI
var BuildID string

// internal version identifier
var version string

var validParsers = []string{
	"nginx",
	"mongo",
	"json",
	"mysql",
}

// GlobalOptions has all the top level CLI flags that honeytail supports
type GlobalOptions struct {
	APIHost string `hidden:"true" long:"api_host" description:"Host for the Honeycomb API" default:"https://api.honeycomb.io/"`

	SampleRate     uint `short:"r" long:"samplerate" description:"Only send 1 / N log lines" default:"1"`
	NumSenders     uint `short:"P" long:"poolsize" description:"Number of concurrent connections to open to Honeycomb" default:"10"`
	Debug          bool `long:"debug" description:"Print debugging output"`
	StatusInterval uint `long:"status_interval" description:"how frequently, in seconds, to print out summary info" default:"60"`
	BackOff        bool `long:"backoff" description:"When rate limited by the API, back off and retry sending failed events. Otherwise failed events are dropped."`

	ScrubFields  []string `long:"scrub_field" description:"for the field listed, apply a one-way hash to the field content. May be specified multiple times"`
	DropFields   []string `long:"drop_field" description:"do not send the field to Honeycomb. May be specified multiple times"`
	AddFields    []string `long:"add_field" description:"add the field to every event. Field should be key=val. May be specified multiple times"`
	RequestShape []string `long:"request_shape" description:"identify a field that contains an HTTP request of the form 'METHOD /path HTTP/1.x'. Break apart that field into subfields that contain components. May be specified multiple times. Defaults to 'request' when using the nginx parser"`
	ShapePrefix  string   `long:"shape_prefix" description:"prefix to use on fields generated from request_shape to prevent field collision"`

	Reqs  RequiredOptions `group:"Required Options"`
	Modes OtherModes      `group:"Other Modes"`

	Tail tail.TailOptions `group:"Tail Options" namespace:"tail"`

	Nginx nginx.Options   `group:"Nginx Parser Options" namespace:"nginx"`
	JSON  htjson.Options  `group:"JSON Parser Options" namespace:"json"`
	MySQL mysql.Options   `group:"MySQL Parser Options" namespace:"mysql"`
	Mongo mongodb.Options `group:"MongoDB Parser Options" namespace:"mongo"`
}

type RequiredOptions struct {
	ParserName string   `short:"p" long:"parser" description:"Parser module to use. Use --list to list available options."`
	WriteKey   string   `short:"k" long:"writekey" description:"Team write key"`
	LogFiles   []string `short:"f" long:"file" description:"Log file(s) to parse. Use '-' for STDIN, use this flag multiple times to tail multiple files, or use a glob (/path/to/foo-*.log)"`
	Dataset    string   `short:"d" long:"dataset" description:"Name of the dataset"`
}

type OtherModes struct {
	Help        bool `short:"h" long:"help" description:"Show this help message"`
	ListParsers bool `short:"l" long:"list" description:"List available parsers"`
	Version     bool `short:"V" long:"version" description:"Show version"`

	WriteManPage bool `hidden:"true" long:"write-man-page" description:"Write out a man page"`
}

func main() {
	var options GlobalOptions
	flagParser := flag.NewParser(&options, flag.PrintErrors)
	flagParser.Usage = "-p <parser> -k <writekey> -f </path/to/logfile> -d <mydata> [optional arguments]"
	if extraArgs, err := flagParser.Parse(); err != nil || len(extraArgs) != 0 {
		fmt.Println("Error: failed to parse the command line.")
		if err != nil {
			fmt.Printf("\t%s\n", err)
		} else {
			fmt.Printf("\tUnexpected extra arguments: %s\n", strings.Join(extraArgs, " "))
		}
		usage()
		os.Exit(1)
	}
	rand.Seed(time.Now().UnixNano())

	if options.Debug {
		logrus.SetLevel(logrus.DebugLevel)
	}

	setVersion()
	handleOtherModes(flagParser, options)
	addParserDefaultOptions(&options)
	sanityCheckOptions(&options)
	verifyWritekey(options)
	run(options)
}

// setVersion sets the internal version ID and updates libhoney's user-agent
func setVersion() {
	if BuildID == "" {
		version = "dev"
	} else {
		version = BuildID
	}
	libhoney.UserAgentAddition = fmt.Sprintf("honeytail/%s", version)
}

// handleOtherModes takse care of all flags that say we should just do something
// and exit rather than actually parsing logs
func handleOtherModes(fp *flag.Parser, options GlobalOptions) {
	if options.Modes.Version {
		fmt.Println("Honeytail version", version)
		os.Exit(0)
	}
	if options.Modes.Help {
		fp.WriteHelp(os.Stdout)
		fmt.Println("")
		os.Exit(0)
	}
	if options.Modes.WriteManPage {
		fp.WriteManPage(os.Stdout)
		os.Exit(0)
	}

	if options.Modes.ListParsers {
		fmt.Println("Available parsers:", strings.Join(validParsers, ", "))
		os.Exit(0)
	}
}

func addParserDefaultOptions(options *GlobalOptions) {
	switch {
	case options.Reqs.ParserName == "nginx":
		// automatically normalize the request when using the nginx parser
		options.RequestShape = append(options.RequestShape, "request")
	}
}

func sanityCheckOptions(options *GlobalOptions) {
	switch {
	case options.Reqs.ParserName == "":
		fmt.Println("parser required.")
		usage()
		os.Exit(1)
	case options.Reqs.WriteKey == "" || options.Reqs.WriteKey == "NULL":
		fmt.Println("write key required.")
		usage()
		os.Exit(1)
	case len(options.Reqs.LogFiles) == 0:
		fmt.Println("log file name or '-' required.")
		usage()
		os.Exit(1)
	case options.Reqs.Dataset == "":
		fmt.Println("dataset name required.")
		usage()
		os.Exit(1)
	case options.Tail.ReadFrom == "end" && options.Tail.Stop:
		fmt.Println("Reading from the end and stopping when we get there. Zero lines to process. Ok, all done! ;)")
		usage()
		os.Exit(1)
	case len(options.Reqs.LogFiles) > 1 && options.Tail.StateFile != "":
		fmt.Println("Statefile can not be set when tailing from multiple files.")
		usage()
		os.Exit(1)
	case options.Tail.StateFile != "":
		files, err := filepath.Glob(options.Reqs.LogFiles[0])
		if err != nil {
			fmt.Printf("Trying to glob log file %s failed: %+v\n",
				options.Reqs.LogFiles[0], err)
			usage()
			os.Exit(1)
		}
		if len(files) > 1 {
			fmt.Println("Statefile can not be set when tailing from multiple files.")
			usage()
			os.Exit(1)
		}
	}
}

func usage() {
	fmt.Print(`
Usage: honeytail -p <parser> -k <writekey> -f </path/to/logfile> -d <mydata> [optional arguments]

For even more detail on required and optional parameters, run
honeytail --help
`)
}

// verifyWritekey calls out to api to validate the writekey, so we can exit
// immediately instead of happily sending events that are all rejected.
func verifyWritekey(options GlobalOptions) {
	url := fmt.Sprintf("%s/1/team_slug", options.APIHost)
	req, err := http.NewRequest("GET", url, nil)
	req.Header.Set("User-Agent", libhoney.UserAgentAddition)
	req.Header.Add("X-Honeycomb-Team", options.Reqs.WriteKey)
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("Failed to validate your writekey:")
		fmt.Println("\t", err)
		fmt.Println("Sorry! Please try again.")
		os.Exit(1)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		body, _ := ioutil.ReadAll(resp.Body)
		fmt.Println("Failed to validate your writekey:")
		fmt.Println("\t", string(body))
		fmt.Println("Sorry! Please try again.")
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"log"
	"os"
	"path/filepath"
)

var prefLandscape, prefPortrait, copySmallImages, targetDeDup, targetValidateClean *bool
var ignoreWidth, ignoreHeight *int

const smallWidth = 150
const smallHeight = 150

// Validate if an image matches the specified rules
func (img *ImgFileInfo) Validate() error {
	if !*copySmallImages && ((img.ImgConf.Width <= smallWidth) || (img.ImgConf.Height <= smallHeight)) {
		return newImgError(fmt.Sprintf("Image %dx%d or smaller", smallWidth, smallHeight), img)
	}
	if (*ignoreHeight != 0) && (img.ImgConf.Height != *ignoreHeight) {
		return newImgError(fmt.Sprintf("height %d != expected %d", img.ImgConf.Height, *ignoreHeight), img)
	}
	if (*ignoreWidth != 0) && (img.ImgConf.Width != *ignoreWidth) {
		return newImgError(fmt.Sprintf("width %d != expected %d", img.ImgConf.Width, *ignoreWidth), img)
	}
	if *prefLandscape || *prefPortrait {
		if *prefLandscape && !*prefPortrait && (img.ImgConf.Width < img.ImgConf.Height) {
			return newImgError(fmt.Sprintf("%dx%d not landscape", img.ImgConf.Width, img.ImgConf.Height), img)
		}
		if !*prefLandscape && *prefPortrait && (img.ImgConf.Height > img.ImgConf.Width) {
			return newImgError(fmt.Sprintf("%dx%d not landscape", img.ImgConf.Width, img.ImgConf.Height), img)
		}
	}
	return nil
}

// RemoveDuplicateImages removes all duplicate images from a target directory. It does not recurse
func RemoveDuplicateImages(fileInfo []*ImgFileInfo, removefiles bool) {
	cleancount := 0
	for of := range fileInfo {
		for cf := range fileInfo {
			if of == cf {
				continue
			}
			if fileInfo[of].Equals(fileInfo[cf]) {
				if fileInfo[of].doClean || fileInfo[cf].doClean {
					// Already determined which-one to clean
					continue
				}
				if fileInfo[cf].ModTime.After(fileInfo[of].ModTime) {
					log.Printf("Remove:\n     %s\n  won:\n     %s", fileInfo[of], fileInfo[cf])
					fileInfo[of].doClean = true
					cleancount++
				} else {
					log.Printf("Remove:\n     %s\n  won:\n     %s", fileInfo[cf], fileInfo[of])
					fileInfo[cf].doClean = true
					cleancount++
				}
			}
		}
	}
	if cleancount == 0 {
		return
	}
	if removefiles {
		log.Printf("Removing files:")
	} else {
		log.Printf("Found duplicate files: (not removing)")
	}
	for _, fi := range fileInfo {
		if fi.doClean {
			log.Printf("    %s", fi)
			if removefiles {
				if err := os.Remove(fi.FileName); err != nil {
					log.Printf("Could not remove %s: %s", fi, err)
				}
			}
		}
	}
}

func main() {
	targetDir := flag.String("target", filepath.Join(os.Getenv("USERPROFILE"), "/Pictures/Spotlight"), "the target directory")
	prefLandscape = flag.Bool("landscape", true, "only copy landscape images (width > height)")
	prefPortrait = flag.Bool("portrait", false, "only copy portrait images (height > width)")
	ignoreWidth = flag.Int("width", 0, "only copy files with this width. 0 = ignore width")
	ignoreHeight = flag.Int("height", 0, "only copy files with this height. 0 = ignore height")
	copySmallImages = flag.Bool("copysmall", false, "copy small images (<= 150)")
	targetDeDup = flag.Bool("targetdedup", false, "remove all duplicate images in target directory")
	targetValidateClean = flag.Bool("targetvalidateremove", false, "validate all files in target directory and remove them if they don't match (DANGAROUS)")
	logFile := flag.String("logfile", "", "file to log to")
	flag.Parse()
	if *logFile != "" {
		if f, err := os.OpenFile(*logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666); err == nil {
			defer f.Close()
			log.SetOutput(f)
		}
	}

	// Construct the spotlight source directory
	spotdir := filepath.Join(os.Getenv("USERPROFILE"), "/AppData/Local/Packages/Microsoft.Windows.ContentDeliveryManager_cw5n1h2txyewy/LocalState/Assets/")

	// Log options with which this instance was started
	log.Printf("================================================\n")
	log.Printf("== Starting Windows Spotlight background sync ==\n")
	log.Printf("Target dir: %s\n", *targetDir)
	log.Printf("Source dir: %s\n", spotdir)
	log.Printf("Log file  : %s\n", *logFile)
	log.Printf("Landscape        : %t\n", *prefLandscape)
	log.Printf("Portrait         : %t\n", *prefPortrait)
	log.Printf("Requested Width  : %d\n", *ignoreWidth)
	log.Printf("Requested Height : %d\n", *ignoreHeight)
	log.Printf("Copy small images: %t\n", *copySmallImages)
	log.Printf("Target dedup     : %t\n", *targetDeDup)
	log.Printf("Target validate  : %t\n", *targetValidateClean)

	// Ensure the target directory exists
	os.MkdirAll(*targetDir, 0777)

	files := GetDirImages(spotdir)
	for _, f := range files {
		if err := f.Validate(); err != nil {
			log.Printf("Skipped: %s", err)
			continue
		}
		log.Printf("Copy %s to %s", f, *targetDir)
		if cerr := CopyFile(f.FileName, filepath.Join(*targetDir, filepath.Base(f.FileName)+"."+f.Format)); cerr != nil {
			log.Printf("ERROR: Could not copy %s to %s", f, filepath.Join(*targetDir, filepath.Base(f.FileName)+"."+f.Format))
		}
	}

	log.Printf("Analyzing target folder...")
	rmfiles := GetDirImages(*targetDir)
	RemoveDuplicateImages(rmfiles, *targetDeDup)
	for _, f := range rmfiles {
		if err := f.Validate(); err != nil {
			if *targetValidateClean {
				log.Printf("Image in targetdir not matching requested validation, removing: %s", f)
				if err := os.Remove(f.FileName); err != nil {
					log.Printf("Could not remove %s: %s", f, err)
				}
			} else {
				log.Printf("WARN: Image in targetdir not matching requested validation: %s", f)
			}
		}
	}
}
<commit_msg>Renamed width/height flags to avoid confusion<commit_after>package main

import (
	"flag"
	"fmt"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"log"
	"os"
	"path/filepath"
)

var prefLandscape, prefPortrait, copySmallImages, targetDeDup, targetValidateClean *bool
var matchWidth, matchHeight *int

const smallWidth = 150
const smallHeight = 150

// Validate if an image matches the specified rules
func (img *ImgFileInfo) Validate() error {
	if !*copySmallImages && ((img.ImgConf.Width <= smallWidth) || (img.ImgConf.Height <= smallHeight)) {
		return newImgError(fmt.Sprintf("Image %dx%d or smaller", smallWidth, smallHeight), img)
	}
	if (*matchHeight != 0) && (img.ImgConf.Height != *matchHeight) {
		return newImgError(fmt.Sprintf("height %d != expected %d", img.ImgConf.Height, *matchHeight), img)
	}
	if (*matchWidth != 0) && (img.ImgConf.Width != *matchWidth) {
		return newImgError(fmt.Sprintf("width %d != expected %d", img.ImgConf.Width, *matchWidth), img)
	}
	if *prefLandscape || *prefPortrait {
		if *prefLandscape && !*prefPortrait && (img.ImgConf.Width < img.ImgConf.Height) {
			return newImgError(fmt.Sprintf("%dx%d not landscape", img.ImgConf.Width, img.ImgConf.Height), img)
		}
		if !*prefLandscape && *prefPortrait && (img.ImgConf.Height > img.ImgConf.Width) {
			return newImgError(fmt.Sprintf("%dx%d not landscape", img.ImgConf.Width, img.ImgConf.Height), img)
		}
	}
	return nil
}

// RemoveDuplicateImages removes all duplicate images from a target directory. It does not recurse
func RemoveDuplicateImages(fileInfo []*ImgFileInfo, removefiles bool) {
	cleancount := 0
	for of := range fileInfo {
		for cf := range fileInfo {
			if of == cf {
				continue
			}
			if fileInfo[of].Equals(fileInfo[cf]) {
				if fileInfo[of].doClean || fileInfo[cf].doClean {
					// Already determined which-one to clean
					continue
				}
				if fileInfo[cf].ModTime.After(fileInfo[of].ModTime) {
					log.Printf("Remove:\n     %s\n  won:\n     %s", fileInfo[of], fileInfo[cf])
					fileInfo[of].doClean = true
					cleancount++
				} else {
					log.Printf("Remove:\n     %s\n  won:\n     %s", fileInfo[cf], fileInfo[of])
					fileInfo[cf].doClean = true
					cleancount++
				}
			}
		}
	}
	if cleancount == 0 {
		return
	}
	if removefiles {
		log.Printf("Removing files:")
	} else {
		log.Printf("Found duplicate files: (not removing)")
	}
	for _, fi := range fileInfo {
		if fi.doClean {
			log.Printf("    %s", fi)
			if removefiles {
				if err := os.Remove(fi.FileName); err != nil {
					log.Printf("Could not remove %s: %s", fi, err)
				}
			}
		}
	}
}

func main() {
	targetDir := flag.String("target", filepath.Join(os.Getenv("USERPROFILE"), "/Pictures/Spotlight"), "the target directory")
	prefLandscape = flag.Bool("landscape", true, "only copy landscape images (width > height)")
	prefPortrait = flag.Bool("portrait", false, "only copy portrait images (height > width)")
	matchWidth = flag.Int("width", 0, "only copy files with this width. 0 = ignore width")
	matchHeight = flag.Int("height", 0, "only copy files with this height. 0 = ignore height")
	copySmallImages = flag.Bool("copysmall", false, "copy small images (<= 150)")
	targetDeDup = flag.Bool("targetdedup", false, "remove all duplicate images in target directory")
	targetValidateClean = flag.Bool("targetvalidateremove", false, "validate all files in target directory and remove them if they don't match (DANGAROUS)")
	logFile := flag.String("logfile", "", "file to log to")
	flag.Parse()
	if *logFile != "" {
		if f, err := os.OpenFile(*logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666); err == nil {
			defer f.Close()
			log.SetOutput(f)
		}
	}

	// Construct the spotlight source directory
	spotdir := filepath.Join(os.Getenv("USERPROFILE"), "/AppData/Local/Packages/Microsoft.Windows.ContentDeliveryManager_cw5n1h2txyewy/LocalState/Assets/")

	// Log options with which this instance was started
	log.Printf("================================================\n")
	log.Printf("== Starting Windows Spotlight background sync ==\n")
	log.Printf("Target dir: %s\n", *targetDir)
	log.Printf("Source dir: %s\n", spotdir)
	log.Printf("Log file  : %s\n", *logFile)
	log.Printf("Landscape        : %t\n", *prefLandscape)
	log.Printf("Portrait         : %t\n", *prefPortrait)
	log.Printf("Requested Width  : %d\n", *matchWidth)
	log.Printf("Requested Height : %d\n", *matchHeight)
	log.Printf("Copy small images: %t\n", *copySmallImages)
	log.Printf("Target dedup     : %t\n", *targetDeDup)
	log.Printf("Target validate  : %t\n", *targetValidateClean)

	// Ensure the target directory exists
	os.MkdirAll(*targetDir, 0777)

	files := GetDirImages(spotdir)
	for _, f := range files {
		if err := f.Validate(); err != nil {
			log.Printf("Skipped: %s", err)
			continue
		}
		log.Printf("Copy %s to %s", f, *targetDir)
		if cerr := CopyFile(f.FileName, filepath.Join(*targetDir, filepath.Base(f.FileName)+"."+f.Format)); cerr != nil {
			log.Printf("ERROR: Could not copy %s to %s", f, filepath.Join(*targetDir, filepath.Base(f.FileName)+"."+f.Format))
		}
	}

	log.Printf("Analyzing target folder...")
	rmfiles := GetDirImages(*targetDir)
	RemoveDuplicateImages(rmfiles, *targetDeDup)
	for _, f := range rmfiles {
		if err := f.Validate(); err != nil {
			if *targetValidateClean {
				log.Printf("Image in targetdir not matching requested validation, removing: %s", f)
				if err := os.Remove(f.FileName); err != nil {
					log.Printf("Could not remove %s: %s", f, err)
				}
			} else {
				log.Printf("WARN: Image in targetdir not matching requested validation: %s", f)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/gob"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/gorilla/feeds"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
	"github.com/qbit/dnews/src"
)

var store = sessions.NewCookieStore([]byte("something-very-secret"))
var templ, err = template.New("dnews").Funcs(funcMap).ParseGlob("templates/*.html")

var funcMap = template.FuncMap{
	"formatDate": dnews.FormatDate,
	"printHTML": func(b []byte) template.HTML {
		return template.HTML(string(b))
	},
}

func init() {
	gob.Register(&dnews.User{})
}

func main() {
	r := mux.NewRouter()
	fs := http.FileServer(http.Dir("public"))
	http.Handle("/public/", http.StripPrefix("/public/", fs))
	r.HandleFunc("/feeds", func(w http.ResponseWriter, r *http.Request) {
		err = templ.ExecuteTemplate(w, "feed.html", nil)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	})
	r.HandleFunc("/feed/{type}", func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		feedType := vars["type"]
		fmt.Println(feedType)
		now := time.Now()
		feed := &feeds.Feed{
			Title:       "Daemon.News",
			Link:        &feeds.Link{Href: "https://daemon.news"},
			Description: "*BSD News and Advocacy",
			Author:      &feeds.Author{Name: "The Daemon News Team", Email: "daemons@daemon.news"},
			Created:     now,
			Copyright:   "This work is copyright © Daemon.News",
		}

		// TODO populate items

		switch feedType {
		case "atom":
			atom, err := feed.ToAtom()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			fmt.Fprintf(w, atom)
		case "rss":
			rss, err := feed.ToRss()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			fmt.Fprintf(w, rss)
		default:
			http.Error(w, "Invalid feed type!", http.StatusInternalServerError)
			return
		}

	})
	r.HandleFunc("/article/raw/{id:[0-9]+}", func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		id, err := strconv.Atoi(vars["id"])
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		article, err := dnews.GetRawArticle(id)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		fmt.Fprintf(w, "%s", article.Body)
	})
	r.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) {
		session, err := store.Get(r, "session-name")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		user := r.FormValue("user")
		passwd := r.FormValue("passwd")

		if user == "" && passwd == "" {
			err = templ.ExecuteTemplate(w, "login.html", nil)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		} else {
			// do auth
			u, err := dnews.Auth(user, passwd)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			if u.Authed {
				session.Values["user"] = u
				session.Save(r, w)
				http.Redirect(w, r, "/", http.StatusFound)
			} else {
				log.Printf("Invalid user: %s", user)
				err = templ.ExecuteTemplate(w, "login.html", struct {
					Error string
				}{
					"Invalid User!",
				})
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
			}
		}
	})
	r.HandleFunc("/logout", func(w http.ResponseWriter, r *http.Request) {
		session, err := store.Get(r, "session-name")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		session.Options = &sessions.Options{
			MaxAge: -1,
		}
		session.Save(r, w)
		http.Redirect(w, r, "/", http.StatusFound)

	})
	r.HandleFunc("/favicon.ico", func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "NO!", http.StatusNotFound)
		return
	})
	r.HandleFunc("/advocacy", func(w http.ResponseWriter, r *http.Request) {
		session, err := store.Get(r, "session-name")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		val := session.Values["user"]
		if _, ok := val.(*dnews.User); !ok {
			val = &dnews.User{}
			session.Values["user"] = &val
			session.Save(r, w)
		}

		data := struct {
			User interface{}
		}{
			&val,
		}

		err = templ.ExecuteTemplate(w, "advocacy.html", data)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	})
	r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		session, err := store.Get(r, "session-name")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		val := session.Values["user"]
		if _, ok := val.(*dnews.User); !ok {
			val = &dnews.User{}
			session.Values["user"] = &val
			session.Save(r, w)
		}

		a, err := dnews.GetNArticles(10)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		data := struct {
			Articles *dnews.Articles
			// we did a type check above, but it would be nice to
			// be able to use the actual type when sending to the
			// templating stuffs :(
			User interface{}
		}{
			&a,
			&val,
		}

		err = templ.ExecuteTemplate(w, "index.html", data)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	})

	http.Handle("/", r)
	http.ListenAndServe(":8080", nil)
}
<commit_msg>abstract out responses and template rendering<commit_after>package main

import (
	"encoding/gob"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/gorilla/feeds"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
	"github.com/qbit/dnews/src"
)

var store = sessions.NewCookieStore([]byte("something-very-secret"))
var templ, err = template.New("dnews").Funcs(funcMap).ParseGlob("templates/*.html")

type response struct {
	Error    string
	User     interface{}
	Articles *dnews.Articles
	Article  *dnews.Article
}

var funcMap = template.FuncMap{
	"formatDate": dnews.FormatDate,
	"printHTML": func(b []byte) template.HTML {
		return template.HTML(string(b))
	},
}

func init() {
	gob.Register(&dnews.User{})
}

func renderTemplate(w http.ResponseWriter, d *response, t string) {
	err = templ.ExecuteTemplate(w, t, d)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func grabUser(w http.ResponseWriter, r *http.Request) (*response, error) {
	session, err := store.Get(r, "session-name")
	if err != nil {
		return nil, err
	}

	uVal := session.Values["user"]
	if _, ok := uVal.(*dnews.User); !ok {
		uVal = &dnews.User{}
		session.Values["user"] = &uVal
		session.Save(r, w)
	}

	var data = response{}
	data.User = &uVal
	return &data, nil
}

func main() {
	r := mux.NewRouter()
	fs := http.FileServer(http.Dir("public"))
	http.Handle("/public/", http.StripPrefix("/public/", fs))
	r.HandleFunc("/feeds", func(w http.ResponseWriter, r *http.Request) {
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		renderTemplate(w, data, "feeds.html")
	})
	r.HandleFunc("/ml", func(w http.ResponseWriter, r *http.Request) {
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		renderTemplate(w, data, "ml.html")
	})
	r.HandleFunc("/search", func(w http.ResponseWriter, r *http.Request) {
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		renderTemplate(w, data, "search_results.html")
	})

	r.HandleFunc("/feed/{type}", func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		feedType := vars["type"]
		now := time.Now()
		feed := &feeds.Feed{
			Title:       "Daemon.News",
			Link:        &feeds.Link{Href: "https://daemon.news"},
			Description: "*BSD News and Advocacy",
			Author:      &feeds.Author{Name: "The Daemon News Team", Email: "daemons@daemon.news"},
			Created:     now,
			Copyright:   "This work is copyright © Daemon.News",
		}

		a, err := dnews.GetNArticles(10)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		// TODO populate items
		feed.Items = []*feeds.Item{}

		for _, article := range a {
			f := feeds.Item{}
			f.Title = article.Title
			f.Link = &feeds.Link{Href: fmt.Sprintf("http://daemon.news/article/%d", article.ID)}
			f.Author = &feeds.Author{Name: article.Author.FName, Email: article.Author.Email}
			f.Created = article.Date

			feed.Items = append(feed.Items, &f)
		}

		switch feedType {
		case "atom":
			atom, err := feed.ToAtom()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			fmt.Fprintf(w, atom)
		case "rss":
			rss, err := feed.ToRss()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			fmt.Fprintf(w, rss)
		default:
			http.Error(w, "Invalid feed type!", http.StatusInternalServerError)
			return
		}

	})
	r.HandleFunc("/article/{id:[0-9]+}", func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		id, err := strconv.Atoi(vars["id"])
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		article, err := dnews.GetArticle(id)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		data.Article = article
		renderTemplate(w, data, "article.html")

	})
	r.HandleFunc("/article/raw/{id:[0-9]+}", func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		id, err := strconv.Atoi(vars["id"])
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		article, err := dnews.GetRawArticle(id)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		fmt.Fprintf(w, "%s", article.Body)
	})
	r.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) {
		session, err := store.Get(r, "session-name")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		user := r.FormValue("user")
		passwd := r.FormValue("passwd")

		if user == "" && passwd == "" {
			err = templ.ExecuteTemplate(w, "login.html", nil)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		} else {
			// do auth
			u, err := dnews.Auth(user, passwd)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			if u.Authed {
				session.Values["user"] = u
				session.Save(r, w)
				http.Redirect(w, r, "/", http.StatusFound)
			} else {
				log.Printf("Invalid user: %s", user)
				err = templ.ExecuteTemplate(w, "login.html", struct {
					Error string
				}{
					"Invalid User!",
				})
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
			}
		}
	})
	r.HandleFunc("/logout", func(w http.ResponseWriter, r *http.Request) {
		session, err := store.Get(r, "session-name")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		session.Options = &sessions.Options{
			MaxAge: -1,
		}
		session.Save(r, w)
		http.Redirect(w, r, "/", http.StatusFound)

	})
	r.HandleFunc("/advocacy", func(w http.ResponseWriter, r *http.Request) {
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		renderTemplate(w, data, "advocacy.html")
	})
	r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		session, err := store.Get(r, "session-name")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		val := session.Values["user"]
		if _, ok := val.(*dnews.User); !ok {
			val = &dnews.User{}
			session.Values["user"] = &val
			session.Save(r, w)
		}

		a, err := dnews.GetNArticles(10)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		data := struct {
			Articles *dnews.Articles
			// we did a type check above, but it would be nice to
			// be able to use the actual type when sending to the
			// templating stuffs :(
			User interface{}
		}{
			&a,
			&val,
		}

		err = templ.ExecuteTemplate(w, "index.html", data)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	})

	http.Handle("/", r)
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"os"
	"strings"
	"time"
)

const staticVersion = "v0.3.0+"

var version string

func setupFlag() {
	viper.SetConfigName("config")
	viper.AddConfigPath("/etc/xdg/vv")
	viper.AddConfigPath("$HOME/.config/vv")
	pflag.String("mpd.host", "localhost", "mpd server hostname to connect")
	pflag.String("mpd.port", "6600", "mpd server TCP port to connect")
	pflag.String("mpd.music_directory", "", "set music_directory in mpd.conf value to search album cover image")
	pflag.String("server.port", "8080", "this app serving TCP port")
	pflag.BoolP("debug", "d", false, "use local assets if exists")
	pflag.Parse()
	viper.BindPFlags(pflag.CommandLine)
}

func getMusicDirectory(confpath string) (string, error) {
	f, err := os.Open(confpath)
	if err != nil {
		return "", err
	}
	defer f.Close()
	sc := bufio.NewScanner(f)
	for i := 1; sc.Scan(); i++ {
		if err := sc.Err(); err != nil {
			return "", err
		}
		l := sc.Text()
		if strings.HasPrefix(l, "music_directory") {
			q := strings.TrimSpace(strings.TrimPrefix(l, "music_directory"))
			if strings.HasPrefix(q, "\"") && strings.HasSuffix(q, "\"") {
				return strings.Trim(q, "\""), nil
			}
		}
	}
	return "", nil
}

//go:generate go-bindata assets
func main() {
	setupFlag()
	err := viper.ReadInConfig()
	if err != nil {
		if _, notfound := err.(viper.ConfigFileNotFoundError); !notfound {
			fmt.Printf("faied to load config file: %s\n", err)
			os.Exit(1)
		}
	}
	musicDirectory := viper.GetString("mpd.music_directory")
	if len(musicDirectory) == 0 && viper.GetString("mpd.host") == "localhost" {
		dir, err := getMusicDirectory("/etc/mpd.conf")
		if err == nil {
			musicDirectory = dir
		}
	}
	addr := viper.GetString("mpd.host") + ":" + viper.GetString("mpd.port")
	music, err := Dial("tcp", addr, "", musicDirectory)
	defer music.Close()
	if err != nil {
		fmt.Printf("faied to connect/initialize mpd: %s\n", err)
		os.Exit(1)
	}
	s := Server{
		Music:          music,
		MusicDirectory: musicDirectory,
		Port:           viper.GetString("server.port"),
		StartTime:      time.Now().UTC(),
		debug:          viper.GetBool("debug"),
	}
	s.Serve()
}
<commit_msg>v0.3.1+<commit_after>package main

import (
	"bufio"
	"fmt"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"os"
	"strings"
	"time"
)

const staticVersion = "v0.3.1+"

var version string

func setupFlag() {
	viper.SetConfigName("config")
	viper.AddConfigPath("/etc/xdg/vv")
	viper.AddConfigPath("$HOME/.config/vv")
	pflag.String("mpd.host", "localhost", "mpd server hostname to connect")
	pflag.String("mpd.port", "6600", "mpd server TCP port to connect")
	pflag.String("mpd.music_directory", "", "set music_directory in mpd.conf value to search album cover image")
	pflag.String("server.port", "8080", "this app serving TCP port")
	pflag.BoolP("debug", "d", false, "use local assets if exists")
	pflag.Parse()
	viper.BindPFlags(pflag.CommandLine)
}

func getMusicDirectory(confpath string) (string, error) {
	f, err := os.Open(confpath)
	if err != nil {
		return "", err
	}
	defer f.Close()
	sc := bufio.NewScanner(f)
	for i := 1; sc.Scan(); i++ {
		if err := sc.Err(); err != nil {
			return "", err
		}
		l := sc.Text()
		if strings.HasPrefix(l, "music_directory") {
			q := strings.TrimSpace(strings.TrimPrefix(l, "music_directory"))
			if strings.HasPrefix(q, "\"") && strings.HasSuffix(q, "\"") {
				return strings.Trim(q, "\""), nil
			}
		}
	}
	return "", nil
}

//go:generate go-bindata assets
func main() {
	setupFlag()
	err := viper.ReadInConfig()
	if err != nil {
		if _, notfound := err.(viper.ConfigFileNotFoundError); !notfound {
			fmt.Printf("faied to load config file: %s\n", err)
			os.Exit(1)
		}
	}
	musicDirectory := viper.GetString("mpd.music_directory")
	if len(musicDirectory) == 0 && viper.GetString("mpd.host") == "localhost" {
		dir, err := getMusicDirectory("/etc/mpd.conf")
		if err == nil {
			musicDirectory = dir
		}
	}
	addr := viper.GetString("mpd.host") + ":" + viper.GetString("mpd.port")
	music, err := Dial("tcp", addr, "", musicDirectory)
	defer music.Close()
	if err != nil {
		fmt.Printf("faied to connect/initialize mpd: %s\n", err)
		os.Exit(1)
	}
	s := Server{
		Music:          music,
		MusicDirectory: musicDirectory,
		Port:           viper.GetString("server.port"),
		StartTime:      time.Now().UTC(),
		debug:          viper.GetBool("debug"),
	}
	s.Serve()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"

	_ "expvar"
	_ "net/http/pprof"

	"github.com/dt/go-metrics-reporting"
	"github.com/foursquare/gohfile"
)

type SettingDefs struct {
	port  int
	debug bool
	mlock bool

	configJsonUrl string

	cachePath string

	graphite       string
	graphitePrefix string
}

var Settings SettingDefs

func readSettings() []string {
	s := SettingDefs{}
	flag.IntVar(&s.port, "port", 9999, "listen port")

	flag.BoolVar(&s.debug, "debug", false, "print debug output")

	flag.BoolVar(&s.mlock, "mlock", true, "mlock mapped files in memory (only applies to cmd-line specified files).")

	flag.StringVar(&s.configJsonUrl, "config-json", "", "URL of collection configuration json")

	flag.StringVar(&s.cachePath, "cache", os.TempDir(), "local path to write files fetched (*not* cleaned up automatically)")

	flag.StringVar(&s.graphite, "graphite", "", "graphite server to report to")
	flag.StringVar(&s.graphitePrefix, "graphite-prefix", "", "prefix to prepend to reported metrics")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr,
			`
Usage: %s [options] col1=path1 col2=path2 ...
`, os.Args[0])
		flag.PrintDefaults()
	}

	flag.Parse()
	Settings = s

	if (len(flag.Args()) > 1) == (Settings.configJsonUrl != "") {
		log.Println("Collections must be specified OR URL to configuration json.")
		flag.Usage()
		os.Exit(-1)
	}

	return flag.Args()
}

func getCollectionConfig(args []string) []*hfile.CollectionConfig {
	var configs []*hfile.CollectionConfig
	var err error

	if Settings.configJsonUrl != "" {
		if len(args) > 0 {
			log.Fatalln("Only one of command-line collection specs or json config may be used.")
		}
		configs, err = ConfigsFromJsonUrl(Settings.configJsonUrl)
		if err != nil {
			log.Fatalln(err)
		}
	} else {
		configs = make([]*hfile.CollectionConfig, len(args))
		for i, pair := range args {
			parts := strings.SplitN(pair, "=", 2)
			if len(parts) != 2 {
				log.Fatal("collections must be specified in the form 'name=path'")
			}
			configs[i] = &hfile.CollectionConfig{parts[0], parts[1], parts[1], Settings.mlock, Settings.debug}
		}
	}

	return configs
}

func main() {
	args := readSettings()

	report.NewRecorder().
		EnableGCInfoCollection().
		ReportTo(Settings.graphite, Settings.graphitePrefix).
		SetAsDefault()

	configs := getCollectionConfig(args)

	log.Printf("Loading collections (debug %v)...\n", Settings.debug)
	cs, err := hfile.LoadCollections(configs, Settings.cachePath)
	if err != nil {
		log.Fatal(err)
	}

	name, err := os.Hostname()
	if err != nil {
		name = "localhost"
	}
	log.Printf("Serving on http://%s:%d/ \n", name, Settings.port)

	http.Handle("/rpc/HFileService", NewHttpRpcHandler(cs))
	http.Handle("/", &DebugHandler{cs})

	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", Settings.port), nil))
}
<commit_msg>update to match upstream changes in reporter<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"

	_ "expvar"
	_ "net/http/pprof"

	"github.com/dt/go-metrics-reporting"
	"github.com/foursquare/gohfile"
)

type SettingDefs struct {
	port  int
	debug bool
	mlock bool

	configJsonUrl string

	cachePath string

	graphite       string
	graphitePrefix string
}

var Settings SettingDefs

func readSettings() []string {
	s := SettingDefs{}
	flag.IntVar(&s.port, "port", 9999, "listen port")

	flag.BoolVar(&s.debug, "debug", false, "print debug output")

	flag.BoolVar(&s.mlock, "mlock", true, "mlock mapped files in memory (only applies to cmd-line specified files).")

	flag.StringVar(&s.configJsonUrl, "config-json", "", "URL of collection configuration json")

	flag.StringVar(&s.cachePath, "cache", os.TempDir(), "local path to write files fetched (*not* cleaned up automatically)")

	flag.StringVar(&s.graphite, "graphite", "", "graphite server to report to")
	flag.StringVar(&s.graphitePrefix, "graphite-prefix", "", "prefix to prepend to reported metrics")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr,
			`
Usage: %s [options] col1=path1 col2=path2 ...
`, os.Args[0])
		flag.PrintDefaults()
	}

	flag.Parse()
	Settings = s

	if (len(flag.Args()) > 1) == (Settings.configJsonUrl != "") {
		log.Println("Collections must be specified OR URL to configuration json.")
		flag.Usage()
		os.Exit(-1)
	}

	return flag.Args()
}

func getCollectionConfig(args []string) []*hfile.CollectionConfig {
	var configs []*hfile.CollectionConfig
	var err error

	if Settings.configJsonUrl != "" {
		if len(args) > 0 {
			log.Fatalln("Only one of command-line collection specs or json config may be used.")
		}
		configs, err = ConfigsFromJsonUrl(Settings.configJsonUrl)
		if err != nil {
			log.Fatalln(err)
		}
	} else {
		configs = make([]*hfile.CollectionConfig, len(args))
		for i, pair := range args {
			parts := strings.SplitN(pair, "=", 2)
			if len(parts) != 2 {
				log.Fatal("collections must be specified in the form 'name=path'")
			}
			configs[i] = &hfile.CollectionConfig{parts[0], parts[1], parts[1], Settings.mlock, Settings.debug}
		}
	}

	return configs
}

func main() {
	args := readSettings()

	report.NewRecorder().
		EnableGCInfoCollection().
		ReportToServer(Settings.graphite, Settings.graphitePrefix).
		SetAsDefault()

	configs := getCollectionConfig(args)

	log.Printf("Loading collections (debug %v)...\n", Settings.debug)
	cs, err := hfile.LoadCollections(configs, Settings.cachePath)
	if err != nil {
		log.Fatal(err)
	}

	name, err := os.Hostname()
	if err != nil {
		name = "localhost"
	}
	log.Printf("Serving on http://%s:%d/ \n", name, Settings.port)

	http.Handle("/rpc/HFileService", NewHttpRpcHandler(cs))
	http.Handle("/", &DebugHandler{cs})

	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", Settings.port), nil))
}
<|endoftext|>


<commit_before>// csvsplit: Split a .csv into multiple files.
// https://github.com/JeffPaine/csvsplit
package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
)

var (
	flagRecords = flag.Int("records", 0, "The number of records per file")
	flagInput   = flag.String("input", "", "Filename of the input file to split (if blank, uses stdin)")
	flagOutput  = flag.String("output", "", "filename / path of the file output (optional)")
	flagHeaders = flag.Int("headers", 0, "Number of header lines in the input file (will be repeated in each output file")
)

func main() {
	flag.Parse()

	// Sanity check flags
	if *flagRecords < 1 || *flagHeaders < 0 || *flagHeaders >= *flagRecords {
		flag.Usage()
		os.Exit(1)
	}

	// Get input from a given file or stdin
	var r *csv.Reader
	if *flagInput != "" {
		f, err := os.Open(*flagInput)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		r = csv.NewReader(f)
	} else {
		r = csv.NewReader(os.Stdin)
	}

	var recs [][]string
	count := 1
	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}

		recs = append(recs, record)
		if len(recs) == *flagRecords {
			save(&recs, count)
			// Reset records to include just the header lines (if any)
			recs = recs[:*flagHeaders]
			count++
		}
	}
	if len(recs) > 0 {
		save(&recs, count)
	}
}

// save saves the given *[][]string of csv data to a .csv file. Files are named
// sequentially in the form of 001.csv, 002.csv, etc.
func save(recs *[][]string, c int) {
	name := fmt.Sprintf("%v%03d%v", *flagOutput, c, ".csv")

	// Make sure we don't overwrite existing files
	if _, err := os.Stat(name); err == nil {
		log.Fatal("File exists: ", name)
	}

	// If a path is specified, make sure that path exists
	if filepath.Dir(*flagOutput) != "." {
		_, err := os.Stat(filepath.Dir(*flagOutput))
		if err != nil {
			log.Fatalf("no such directory '%v'", *flagOutput)
		}
	}

	f, err := os.Create(name)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	w := csv.NewWriter(f)
	w.WriteAll(*recs)
}
<commit_msg>minor formatting improvements<commit_after>// csvsplit: Split a .csv into multiple files.
// https://github.com/JeffPaine/csvsplit
package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
)

var (
	flagRecords = flag.Int("records", 0, "The number of records per file")
	flagInput   = flag.String("input", "", "Filename of the input file to split (if blank, uses stdin)")
	flagOutput  = flag.String("output", "", "filename / path of the file output (optional)")
	flagHeaders = flag.Int("headers", 0, "Number of header lines in the input file (will be repeated in each output file")
)

func main() {
	flag.Parse()

	// Sanity check flags
	if *flagRecords < 1 || *flagHeaders < 0 || *flagHeaders >= *flagRecords {
		flag.Usage()
		os.Exit(1)
	}

	// Get input from a given file or stdin
	var r *csv.Reader
	if *flagInput != "" {
		f, err := os.Open(*flagInput)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		r = csv.NewReader(f)
	} else {
		r = csv.NewReader(os.Stdin)
	}

	var recs [][]string
	count := 1
	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}

		recs = append(recs, record)
		if len(recs) == *flagRecords {
			save(&recs, count)
			// Reset records to include just the header lines (if any)
			recs = recs[:*flagHeaders]
			count++
		}
	}
	if len(recs) > 0 {
		save(&recs, count)
	}
}

// save() saves the given *[][]string of csv data to a .csv file. Files are named
// sequentially in the form of 001.csv, 002.csv, etc.
func save(recs *[][]string, c int) {
	name := fmt.Sprintf("%v%03d%v", *flagOutput, c, ".csv")

	// Make sure we don't overwrite existing files
	if _, err := os.Stat(name); err == nil {
		log.Fatal("file exists: ", name)
	}

	// If a directory is specified, make sure that directory exists
	if filepath.Dir(*flagOutput) != "." {
		_, err := os.Stat(filepath.Dir(*flagOutput))
		if err != nil {
			log.Fatalf("no such directory '%v'", *flagOutput)
		}
	}

	f, err := os.Create(name)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	w := csv.NewWriter(f)
	w.WriteAll(*recs)
}
<|endoftext|>


<commit_before>package weatherupdate

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"golang.org/x/net/context"

	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/urlfetch"
)

const (
	weddingYear  = 2017
	weddingMonth = time.September
	weddingDay   = 23
	weddingHour  = 15

	twilioNumber = "+18457648204"
	passphrase   = "wedding weather"

	weatherHistoryKey = "history1"
)

var (
	weatherAPI       = "http://api.openweathermap.org/data/2.5/forecast?zip=22304&units=imperial&APPID=" + os.Getenv("WEATHER_API_KEY")
	twilioSendSMSAPI = "https://api.twilio.com/2010-04-01/Accounts/" + os.Getenv("TWILIO_ACCOUNT_SID") + "/Messages"
)

func init() {
	http.HandleFunc("/checkTask", checkTaskHandler)
	http.HandleFunc("/checkNow", checkNowHandler)
}

type weatherResponse struct {
	List []struct {
		Dt   int64 `json:"dt"`
		Main struct {
			Temp     float32 `json:"temp"`
			Humidity int     `json:"humidity"`
		} `json:"main"`
		Weather []struct {
			ID          int    `json:"id"`
			Main        string `json:"main"`
			Description string `json:"description"`
			Icon        string `json:"icon"`
		} `json:"weather"`
	} `json:"list"`
}

func checkTaskHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)

	results, err := fetchWeather(ctx, weatherAPI)
	if err != nil {
		http.Error(w, "could not fetch weather data", http.StatusInternalServerError)
		log.Errorf(ctx, "could not fetch weather data: %v", err)
		return
	}

	var history = new(weatherHistory)
	history.load(ctx)

	if history.changed(*results) {
		msgBody := messageBody(*results)
		history.update(ctx, *results)

		if err := sendMessage(ctx, msgBody, "+18456496408"); err != nil {
			http.Error(w, "could not send sms", http.StatusInternalServerError)
			log.Errorf(ctx, "could not send sms: %v", err)
			return
		}
	}
}

const smsAllowContent = `<?xml version="1.0" encoding="UTF-8"?>
	<Response>
		<Message>
			%s
		</Message>
	</Response>`

const smsDenyContent = `<?xml version="1.0" encoding="UTF-8"?>
	<Response>
		<Message>
			<Media>%s</Media>
		</Message>
	</Response>`

func checkNowHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	w.Header().Set("Content-Type", "text/xml")

	body := r.PostFormValue("Body")
	if !strings.EqualFold(body, passphrase) {
		media := "http://www.icge.co.uk/languagesciencesblog/wp-content/uploads/2014/04/you_shall_not_pass1.jpg"
		fmt.Fprintf(w, smsDenyContent, media)
		return
	}

	results, err := fetchWeather(ctx, weatherAPI)
	if err != nil {
		http.Error(w, "could not fetch weather data", http.StatusInternalServerError)
		log.Errorf(ctx, "could not fetch weather data: %v", err)
		return
	}

	reply := messageBody(*results)
	fmt.Fprintf(w, smsAllowContent, reply)
}

func fetchWeather(ctx context.Context, url string) (*weatherResponse, error) {
	resp, err := urlfetch.Client(ctx).Get(url)
	if err != nil {
		return nil, fmt.Errorf("could not get weather api data: %v", err)
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf(resp.Status)
	}

	var results = new(weatherResponse)

	defer resp.Body.Close()
	if err = json.NewDecoder(resp.Body).Decode(&results); err != nil {
		return nil, fmt.Errorf("count not decode weather response: %v", err)
	}
	return results, nil
}

func sendMessage(ctx context.Context, body string, toNumber string) error {
	vals := url.Values{}
	vals.Set("To", toNumber)
	vals.Set("From", twilioNumber)
	vals.Set("Body", body)

	mb := strings.NewReader(vals.Encode())
	req, err := http.NewRequest("POST", twilioSendSMSAPI, mb)
	if err != nil {
		return fmt.Errorf("could not create sms request: %v", err)
	}
	req.SetBasicAuth(os.Getenv("TWILIO_ACCOUNT_SID"), os.Getenv("TWILIO_AUTH_TOKEN"))
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	_, err = urlfetch.Client(ctx).Do(req)
	if err != nil {
		return fmt.Errorf("could not send sms request: %v", err)
	}
	return nil
}

func messageBody(results weatherResponse) string {
	msg := "Wedding Weather Report\n"
	for _, r := range results.List {
		t := time.Unix(r.Dt, 0)
		if isWeddingDay(t) {
			msg += fmt.Sprintf("Temp: %d degrees\nHumidity: %d%%\n%s: %s",
				int(r.Main.Temp), r.Main.Humidity, r.Weather[0].Main, r.Weather[0].Description)
		}
	}
	return msg
}

func isWeddingDay(d time.Time) bool {
	return d.Year() == weddingYear && d.Month() == weddingMonth &&
		d.Day() == weddingDay && d.Hour() == weddingHour
}
<commit_msg>Updated checkTaskHandler so send message to 2 phone numbers.<commit_after>package weatherupdate

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"golang.org/x/net/context"

	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/urlfetch"
)

const (
	weddingYear  = 2017
	weddingMonth = time.September
	weddingDay   = 23
	weddingHour  = 15

	twilioNumber = "+18457648204"
	passphrase   = "wedding weather"

	weatherHistoryKey = "history1"
)

var (
	weatherAPI       = "http://api.openweathermap.org/data/2.5/forecast?zip=22304&units=imperial&APPID=" + os.Getenv("WEATHER_API_KEY")
	twilioSendSMSAPI = "https://api.twilio.com/2010-04-01/Accounts/" + os.Getenv("TWILIO_ACCOUNT_SID") + "/Messages"
)

func init() {
	http.HandleFunc("/checkTask", checkTaskHandler)
	http.HandleFunc("/checkNow", checkNowHandler)
}

type weatherResponse struct {
	List []struct {
		Dt   int64 `json:"dt"`
		Main struct {
			Temp     float32 `json:"temp"`
			Humidity int     `json:"humidity"`
		} `json:"main"`
		Weather []struct {
			ID          int    `json:"id"`
			Main        string `json:"main"`
			Description string `json:"description"`
			Icon        string `json:"icon"`
		} `json:"weather"`
	} `json:"list"`
}

func checkTaskHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)

	results, err := fetchWeather(ctx, weatherAPI)
	if err != nil {
		http.Error(w, "could not fetch weather data", http.StatusInternalServerError)
		log.Errorf(ctx, "could not fetch weather data: %v", err)
		return
	}

	var history = new(weatherHistory)
	history.load(ctx)

	if history.changed(*results) {
		msgBody := messageBody(*results)
		history.update(ctx, *results)

		receiverNumbers := []string{"+18456496408", "+12405384163"}
		for _, n := range receiverNumbers {
			log.Infof(ctx, "sending to: %s", n)
			if err := sendMessage(ctx, msgBody, n); err != nil {
				http.Error(w, "could not send sms", http.StatusInternalServerError)
				log.Errorf(ctx, "could not send sms: %v", err)
				return
			}
		}
	}
}

const smsAllowContent = `<?xml version="1.0" encoding="UTF-8"?>
	<Response>
		<Message>
			%s
		</Message>
	</Response>`

const smsDenyContent = `<?xml version="1.0" encoding="UTF-8"?>
	<Response>
		<Message>
			<Media>%s</Media>
		</Message>
	</Response>`

func checkNowHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	w.Header().Set("Content-Type", "text/xml")

	body := r.PostFormValue("Body")
	if !strings.EqualFold(body, passphrase) {
		media := "http://www.icge.co.uk/languagesciencesblog/wp-content/uploads/2014/04/you_shall_not_pass1.jpg"
		fmt.Fprintf(w, smsDenyContent, media)
		return
	}

	results, err := fetchWeather(ctx, weatherAPI)
	if err != nil {
		http.Error(w, "could not fetch weather data", http.StatusInternalServerError)
		log.Errorf(ctx, "could not fetch weather data: %v", err)
		return
	}

	reply := messageBody(*results)
	fmt.Fprintf(w, smsAllowContent, reply)
}

func fetchWeather(ctx context.Context, url string) (*weatherResponse, error) {
	resp, err := urlfetch.Client(ctx).Get(url)
	if err != nil {
		return nil, fmt.Errorf("could not get weather api data: %v", err)
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf(resp.Status)
	}

	var results = new(weatherResponse)

	defer resp.Body.Close()
	if err = json.NewDecoder(resp.Body).Decode(&results); err != nil {
		return nil, fmt.Errorf("count not decode weather response: %v", err)
	}
	return results, nil
}

func sendMessage(ctx context.Context, body string, toNumber string) error {
	vals := url.Values{}
	vals.Set("To", toNumber)
	vals.Set("From", twilioNumber)
	vals.Set("Body", body)

	mb := strings.NewReader(vals.Encode())
	req, err := http.NewRequest("POST", twilioSendSMSAPI, mb)
	if err != nil {
		return fmt.Errorf("could not create sms request: %v", err)
	}
	req.SetBasicAuth(os.Getenv("TWILIO_ACCOUNT_SID"), os.Getenv("TWILIO_AUTH_TOKEN"))
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	_, err = urlfetch.Client(ctx).Do(req)
	if err != nil {
		return fmt.Errorf("could not send sms request: %v", err)
	}
	return nil
}

func messageBody(results weatherResponse) string {
	msg := "Wedding Weather Report\n"
	for _, r := range results.List {
		t := time.Unix(r.Dt, 0)
		if isWeddingDay(t) {
			msg += fmt.Sprintf("Temp: %d degrees\nHumidity: %d%%\n%s: %s",
				int(r.Main.Temp), r.Main.Humidity, r.Weather[0].Main, r.Weather[0].Description)
		}
	}
	return msg
}

func isWeddingDay(d time.Time) bool {
	return d.Year() == weddingYear && d.Month() == weddingMonth &&
		d.Day() == weddingDay && d.Hour() == weddingHour
}
<|endoftext|>


<commit_before>/*
 * Aminer (C) 2014, 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/minio/cli"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// LogMessage is a serializable json log message
type LogMessage struct {
	StartTime     time.Time
	Duration      time.Duration
	StatusMessage string // human readable http status message
	ContentLength string // human readable content length

	// HTTP detailed message
	HTTP struct {
		ResponseHeaders http.Header
		Request         *http.Request
	}
}

var session *mgo.Session

func connectToMongo(c *cli.Context) {
	session, err := mgo.Dial(c.GlobalString("server"))
	if err != nil {
		panic(err)
	}
	defer session.Close()

	// Optional. Switch the session to a monotonic behavior.
	session.SetMode(mgo.Monotonic, true)
}

func runPopulateCmd(c *cli.Context) {
	f, err := os.Open(c.Args()[0])
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	connectToMongo(c)
	// make this configurable
	m := session.DB("test").C("downloads")
	scanner := bufio.NewScanner(f)
	scanner.Split(bufio.ScanLines)

	var message LogMessage
	for scanner.Scan() {
		json.Unmarshal([]byte(scanner.Text()), &message)
		err = m.Insert(&message)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func runFindCmd(c *cli.Context) {
	connectToMongo(c)
	// make this configurable
	m := session.DB("test").C("downloads")
	result := LogMessage{}
	iter := m.Find(bson.M{"http.request.method": "GET"}).Iter()
	for iter.Next(&result) {
		if result.HTTP.Request.Method != "GET" {
			continue
		}
		if strings.Contains(result.HTTP.Request.RemoteAddr, "50.204.118.154") {
			continue
		}
		if strings.Contains(result.HTTP.Request.RemoteAddr, "10.134.253.170") {
			continue
		}
		fmt.Print(result.HTTP.Request.Method)
		fmt.Print("    ")
		fmt.Print(result.HTTP.Request.RemoteAddr)
		fmt.Print("    ")
		fmt.Print(result.HTTP.Request.RequestURI)
		fmt.Println("    ")
	}
}

var commands = []cli.Command{
	findCmd,
	populateCmd,
}

var findCmd = cli.Command{
	Name:        "find",
	Description: "find all documents for a map",
	Action:      runFindCmd,
}

var populateCmd = cli.Command{
	Name:        "populate",
	Description: "populate your mongo instance with new data",
	Action:      runPopulateCmd,
}

var flags = []cli.Flag{
	cli.StringFlag{
		Name:  "server",
		Value: "localhost",
		Usage: "IP/HOSTNAME of your mongodb instance",
	},
}

func main() {
	app := cli.NewApp()
	app.Usage = "A miner for your minio access logs"
	app.Version = "0.0.1"
	app.Commands = commands
	app.Flags = flags
	app.Author = "Minio.io"

	app.RunAndExitOnError()
}
<commit_msg>Fix commands<commit_after>/*
 * Aminer (C) 2014, 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/minio/cli"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// LogMessage is a serializable json log message
type LogMessage struct {
	StartTime     time.Time
	Duration      time.Duration
	StatusMessage string // human readable http status message
	ContentLength string // human readable content length

	// HTTP detailed message
	HTTP struct {
		ResponseHeaders http.Header
		Request         *http.Request
	}
}

var db *mgo.Collection

func connectToMongo(c *cli.Context) *mgo.Session {
	session, err := mgo.Dial(c.GlobalString("server"))
	if err != nil {
		panic(err)
	}
	// Optional. Switch the session to a monotonic behavior.
	session.SetMode(mgo.Monotonic, true)
	// make this configurable
	db = session.DB("test").C("downloads")
	return session
}

func runPopulateCmd(c *cli.Context) {
	if len(c.Args()) > 1 || c.Args().First() == "help" {
		cli.ShowCommandHelpAndExit(c, "populate", 1) // last argument is exit code
	}
	s := connectToMongo(c)
	defer s.Close()
	f, err := os.Open(strings.TrimSpace(c.Args().First()))
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	scanner.Split(bufio.ScanLines)

	var message LogMessage
	for scanner.Scan() {
		json.Unmarshal([]byte(scanner.Text()), &message)
		err = db.Insert(&message)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func runFindCmd(c *cli.Context) {
	if len(c.Args()) > 1 || c.Args().First() == "help" {
		cli.ShowCommandHelpAndExit(c, "find", 1) // last argument is exit code
	}
	s := connectToMongo(c)
	defer s.Close()
	switch {
	case strings.ToUpper(c.Args().First()) == "GET":
		result := LogMessage{}
		iter := db.Find(bson.M{"http.request.method": "GET"}).Iter()
		for iter.Next(&result) {
			if strings.Contains(result.HTTP.Request.RemoteAddr, "50.204.118.154") {
				continue
			}
			if strings.Contains(result.HTTP.Request.RemoteAddr, "10.134.253.170") {
				continue
			}
			fmt.Print(result.HTTP.Request.Method)
			fmt.Print("    ")
			fmt.Print(result.HTTP.Request.RemoteAddr)
			fmt.Print("    ")
			fmt.Print(result.HTTP.Request.RequestURI)
			fmt.Println("    ")
		}
	case strings.ToUpper(c.Args().First()) == "PUT":
		result := LogMessage{}
		iter := db.Find(bson.M{"http.request.method": "PUT"}).Iter()
		for iter.Next(&result) {
			if strings.Contains(result.HTTP.Request.RemoteAddr, "50.204.118.154") {
				continue
			}
			if strings.Contains(result.HTTP.Request.RemoteAddr, "10.134.253.170") {
				continue
			}
			fmt.Print(result.HTTP.Request.Method)
			fmt.Print("    ")
			fmt.Print(result.HTTP.Request.RemoteAddr)
			fmt.Print("    ")
			fmt.Print(result.HTTP.Request.RequestURI)
			fmt.Println("    ")
		}
	}
}

var commands = []cli.Command{
	findCmd,
	populateCmd,
}

var findCmd = cli.Command{
	Name:   "find",
	Usage:  "find all documents for a map",
	Action: runFindCmd,
}

var populateCmd = cli.Command{
	Name:   "populate",
	Usage:  "populate your mongo instance with new data",
	Action: runPopulateCmd,
}

var flags = []cli.Flag{
	cli.StringFlag{
		Name:  "server",
		Value: "localhost",
		Usage: "IP/HOSTNAME of your mongodb instance",
	},
}

func main() {
	app := cli.NewApp()
	app.Usage = "A miner for your minio access logs"
	app.Version = "0.0.1"
	app.Commands = commands
	app.Flags = flags
	app.Author = "Minio.io"

	app.RunAndExitOnError()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/gob"
	"fmt"
	"html/template"
	"net/http"
	"time"

	"github.com/DaemonNews/dnews/src"
	"github.com/gorilla/csrf"
	"github.com/gorilla/feeds"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
)

// TODO change this secret
var store = sessions.NewCookieStore([]byte("something-very-secret"))
var templ, err = template.New("dnews").Funcs(funcMap).ParseGlob("templates/*.html")

type response struct {
	Error string
	User  interface{}
	//Articles *dnews.Articles
	//Article  *dnews.Article
	Data interface{}
	CSRF map[string]interface{}
}

var funcMap = template.FuncMap{
	"formatDate": dnews.FormatDate,
	"printByte": func(b []byte) string {
		return string(b)
	},
	"printHTML": func(b []byte) template.HTML {
		return template.HTML(string(b))
	},
}

func init() {
	gob.Register(&dnews.User{})
}

func renderTemplate(w http.ResponseWriter, r *http.Request, d *response, t string) {
	d.CSRF = map[string]interface{}{
		csrf.TemplateTag: csrf.TemplateField(r),
	}
	err = templ.ExecuteTemplate(w, t, d)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func grabUser(w http.ResponseWriter, r *http.Request) (*response, error) {
	session, err := store.Get(r, "session-name")
	if err != nil {
		return nil, err
	}

	uVal := session.Values["user"]
	if _, ok := uVal.(*dnews.User); !ok {
		uVal = &dnews.User{}
		session.Values["user"] = &uVal
		session.Save(r, w)
	}

	var data = response{}
	data.User = &uVal
	return &data, nil
}

func main() {
	router := mux.NewRouter()
	router.PathPrefix("/public/").Handler(http.StripPrefix("/public/", http.FileServer(http.Dir("public"))))
	router.HandleFunc("/feeds", func(w http.ResponseWriter, r *http.Request) {
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		renderTemplate(w, r, data, "feeds.html")
	})
	router.HandleFunc("/ml", func(w http.ResponseWriter, r *http.Request) {
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		renderTemplate(w, r, data, "ml.html")
	})
	router.HandleFunc("/archives", func(w http.ResponseWriter, r *http.Request) {
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		renderTemplate(w, r, data, "archives.html")
	})
	router.HandleFunc("/search", func(w http.ResponseWriter, r *http.Request) {
		query := r.FormValue("search")
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, fmt.Sprintf("Can't get user: %s", err.Error()), http.StatusInternalServerError)
			return
		}
		// TODO sanatize!
		a, err := dnews.SearchArticles(query, 100)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		data.Data = &a

		renderTemplate(w, r, data, "search_results.html")
	})

	router.HandleFunc("/feed/{type}", func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		feedType := vars["type"]
		now := time.Now()
		feed := &feeds.Feed{
			Title:       "Daemon.News",
			Link:        &feeds.Link{Href: "https://daemon.news"},
			Description: "*BSD News and Advocacy",
			Author:      &feeds.Author{Name: "The Daemon News Team", Email: "daemons@daemon.news"},
			Created:     now,
			Copyright:   "This work is copyright © Daemon.News",
		}

		a, err := dnews.GetNArticles(10)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		// TODO populate body?
		feed.Items = []*feeds.Item{}

		for _, article := range a {
			f := feeds.Item{}
			f.Title = article.Title
			f.Link = &feeds.Link{Href: fmt.Sprintf("http://daemon.news/article/%s", article.Slug)}
			f.Author = &feeds.Author{Name: article.Author.FName, Email: article.Author.Email}
			f.Created = article.Date

			feed.Items = append(feed.Items, &f)
		}

		switch feedType {
		case "atom":
			atom, err := feed.ToAtom()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			fmt.Fprintf(w, atom)
		case "rss":
			rss, err := feed.ToRss()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			fmt.Fprintf(w, rss)
		default:
			http.Error(w, "Invalid feed type!", http.StatusInternalServerError)
			return
		}

	})
	router.HandleFunc("/article/{slug:[a-zA-Z0-9-]+}", func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		slug := vars["slug"]

		article, err := dnews.GetArticle(slug)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		data.Data = article
		renderTemplate(w, r, data, "article.html")

	})
	router.HandleFunc("/article/raw/{slug:[a-zA-Z0-9-]+}", func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		slug := vars["slug"]
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		article, err := dnews.GetRawArticle(slug)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		fmt.Fprintf(w, "%s", article.Body)
	})
	router.HandleFunc("/login/post", func(w http.ResponseWriter, r *http.Request) {
		session, err := store.Get(r, "session-name")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		user := r.FormValue("user")
		passwd := r.FormValue("passwd")

		if user == "" && passwd == "" {
			http.Redirect(w, r, "/", http.StatusFound)
		} else {
			u, err := dnews.Auth(user, passwd)

			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			if u.Authed {
				session.Values["user"] = u
				session.Save(r, w)
				http.Redirect(w, r, "/", http.StatusFound)
			}
		}
	})
	router.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) {
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		renderTemplate(w, r, data, "login.html")
	})
	router.HandleFunc("/logout", func(w http.ResponseWriter, r *http.Request) {
		session, err := store.Get(r, "session-name")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		session.Options = &sessions.Options{
			MaxAge: -1,
		}
		session.Save(r, w)
		http.Redirect(w, r, "/", http.StatusFound)

	})
	router.HandleFunc("/advocacy", func(w http.ResponseWriter, r *http.Request) {
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		bugs, err := dnews.GetBugs()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		data.Data = &bugs

		renderTemplate(w, r, data, "advocacy.html")
	})
	router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		a, err := dnews.GetNArticles(10)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		data.Data = &a
		renderTemplate(w, r, data, "index.html")
	})
	http.Handle("/", router)

	// TODO change this secret
	http.ListenAndServe(":8080",
		csrf.Protect([]byte("32-byte-long-auth-key"), csrf.Secure(false))(router))
	//http.ListenAndServe(":8080", nil)

}
<commit_msg>logging<commit_after>package main

import (
	"encoding/gob"
	"fmt"
	"html/template"
	"net/http"
	"os"
	"time"

	"github.com/DaemonNews/dnews/src"
	"github.com/gorilla/csrf"
	"github.com/gorilla/feeds"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
)

// TODO change this secret
var store = sessions.NewCookieStore([]byte("something-very-secret"))
var templ, err = template.New("dnews").Funcs(funcMap).ParseGlob("templates/*.html")

type response struct {
	Error string
	User  interface{}
	//Articles *dnews.Articles
	//Article  *dnews.Article
	Data interface{}
	CSRF map[string]interface{}
}

var funcMap = template.FuncMap{
	"formatDate": dnews.FormatDate,
	"printByte": func(b []byte) string {
		return string(b)
	},
	"printHTML": func(b []byte) template.HTML {
		return template.HTML(string(b))
	},
}

func init() {
	gob.Register(&dnews.User{})
}

func renderTemplate(w http.ResponseWriter, r *http.Request, d *response, t string) {
	d.CSRF = map[string]interface{}{
		csrf.TemplateTag: csrf.TemplateField(r),
	}
	err = templ.ExecuteTemplate(w, t, d)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func grabUser(w http.ResponseWriter, r *http.Request) (*response, error) {
	session, err := store.Get(r, "session-name")
	if err != nil {
		return nil, err
	}

	uVal := session.Values["user"]
	if _, ok := uVal.(*dnews.User); !ok {
		uVal = &dnews.User{}
		session.Values["user"] = &uVal
		session.Save(r, w)
	}

	var data = response{}
	data.User = &uVal
	return &data, nil
}

func main() {
	router := mux.NewRouter()

	router.PathPrefix("/public/").Handler(http.StripPrefix("/public/", http.FileServer(http.Dir("public"))))
	router.HandleFunc("/feeds", func(w http.ResponseWriter, r *http.Request) {
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		renderTemplate(w, r, data, "feeds.html")
	})
	router.HandleFunc("/ml", func(w http.ResponseWriter, r *http.Request) {
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		renderTemplate(w, r, data, "ml.html")
	})
	router.HandleFunc("/archives", func(w http.ResponseWriter, r *http.Request) {
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		renderTemplate(w, r, data, "archives.html")
	})
	router.HandleFunc("/search", func(w http.ResponseWriter, r *http.Request) {
		query := r.FormValue("search")
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, fmt.Sprintf("Can't get user: %s", err.Error()), http.StatusInternalServerError)
			return
		}
		// TODO sanatize!
		a, err := dnews.SearchArticles(query, 100)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		data.Data = &a

		renderTemplate(w, r, data, "search_results.html")
	})

	router.HandleFunc("/feed/{type}", func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		feedType := vars["type"]
		now := time.Now()
		feed := &feeds.Feed{
			Title:       "Daemon.News",
			Link:        &feeds.Link{Href: "https://daemon.news"},
			Description: "*BSD News and Advocacy",
			Author:      &feeds.Author{Name: "The Daemon News Team", Email: "daemons@daemon.news"},
			Created:     now,
			Copyright:   "This work is copyright © Daemon.News",
		}

		a, err := dnews.GetNArticles(10)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		// TODO populate body?
		feed.Items = []*feeds.Item{}

		for _, article := range a {
			f := feeds.Item{}
			f.Title = article.Title
			f.Link = &feeds.Link{Href: fmt.Sprintf("http://daemon.news/article/%s", article.Slug)}
			f.Author = &feeds.Author{Name: article.Author.FName, Email: article.Author.Email}
			f.Created = article.Date

			feed.Items = append(feed.Items, &f)
		}

		switch feedType {
		case "atom":
			atom, err := feed.ToAtom()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			fmt.Fprintf(w, atom)
		case "rss":
			rss, err := feed.ToRss()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			fmt.Fprintf(w, rss)
		default:
			http.Error(w, "Invalid feed type!", http.StatusInternalServerError)
			return
		}

	})
	router.HandleFunc("/article/{slug:[a-zA-Z0-9-]+}", func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		slug := vars["slug"]

		article, err := dnews.GetArticle(slug)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		data.Data = article
		renderTemplate(w, r, data, "article.html")

	})
	router.HandleFunc("/article/raw/{slug:[a-zA-Z0-9-]+}", func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		slug := vars["slug"]
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		article, err := dnews.GetRawArticle(slug)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		fmt.Fprintf(w, "%s", article.Body)
	})
	router.HandleFunc("/login/post", func(w http.ResponseWriter, r *http.Request) {
		session, err := store.Get(r, "session-name")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		user := r.FormValue("user")
		passwd := r.FormValue("passwd")

		if user == "" && passwd == "" {
			http.Redirect(w, r, "/", http.StatusFound)
		} else {
			u, err := dnews.Auth(user, passwd)

			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			if u.Authed {
				session.Values["user"] = u
				session.Save(r, w)
				http.Redirect(w, r, "/", http.StatusFound)
			}
		}
	})
	router.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) {
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		renderTemplate(w, r, data, "login.html")
	})
	router.HandleFunc("/logout", func(w http.ResponseWriter, r *http.Request) {
		session, err := store.Get(r, "session-name")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		session.Options = &sessions.Options{
			MaxAge: -1,
		}
		session.Save(r, w)
		http.Redirect(w, r, "/", http.StatusFound)

	})
	router.HandleFunc("/advocacy", func(w http.ResponseWriter, r *http.Request) {
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		bugs, err := dnews.GetBugs()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		data.Data = &bugs

		renderTemplate(w, r, data, "advocacy.html")
	})
	router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		data, err := grabUser(w, r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		a, err := dnews.GetNArticles(10)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		data.Data = &a
		renderTemplate(w, r, data, "index.html")
	})

	loggedRouter := handlers.LoggingHandler(os.Stdout, router)

	// TODO change this secret
	http.ListenAndServe(":8080", csrf.Protect([]byte("32-byte-long-auth-key"), csrf.Secure(false))(loggedRouter))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/go-kit/kit/log/level"
	"github.com/justwatchcom/elasticsearch_exporter/collector"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/version"
)

func main() {
	var (
		Name                 = "elasticsearch_exporter"
		listenAddress        = flag.String("web.listen-address", ":9108", "Address to listen on for web interface and telemetry.")
		metricsPath          = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
		esURI                = flag.String("es.uri", "http://localhost:9200", "HTTP API address of an Elasticsearch node.")
		esTimeout            = flag.Duration("es.timeout", 5*time.Second, "Timeout for trying to get stats from Elasticsearch.")
		esAllNodes           = flag.Bool("es.all", false, "Export stats for all nodes in the cluster. If used, this flag will override the flag es.node.")
		esNode               = flag.String("es.node", "_local", "Node's name of which metrics should be exposed.")
		esExportIndices      = flag.Bool("es.indices", false, "Export stats for indices in the cluster.")
		esExportShards       = flag.Bool("es.shards", false, "Export stats for shards in the cluster (implies es.indices=true).")
		esExportSnapshots    = flag.Bool("es.snapshots", false, "Export stats for the cluster snapshots.")
		esCA                 = flag.String("es.ca", "", "Path to PEM file that contains trusted CAs for the Elasticsearch connection.")
		esClientPrivateKey   = flag.String("es.client-private-key", "", "Path to PEM file that contains the private key for client auth when connecting to Elasticsearch.")
		esClientCert         = flag.String("es.client-cert", "", "Path to PEM file that contains the corresponding cert for the private key to connect to Elasticsearch.")
		esInsecureSkipVerify = flag.Bool("es.ssl-skip-verify", false, "Skip SSL verification when connecting to Elasticsearch.")
		logLevel             = flag.String("log.level", "info", "Sets the loglevel. Valid levels are debug, info, warn, error")
		logFormat            = flag.String("log.format", "logfmt", "Sets the log format. Valid formats are json and logfmt")
		logOutput            = flag.String("log.output", "stdout", "Sets the log output. Valid outputs are stdout and stderr")
		showVersion          = flag.Bool("version", false, "Show version and exit")
	)
	flag.Parse()

	if *showVersion {
		fmt.Print(version.Print(Name))
		os.Exit(0)
	}

	logger := getLogger(*logLevel, *logOutput, *logFormat)

	esURIEnv, ok := os.LookupEnv("ES_URI")
	if ok {
		*esURI = esURIEnv
	}
	esURL, err := url.Parse(*esURI)
	if err != nil {
		_ = level.Error(logger).Log(
			"msg", "failed to parse es.uri",
			"err", err,
		)
		os.Exit(1)
	}

	// returns nil if not provided and falls back to simple TCP.
	tlsConfig := createTLSConfig(*esCA, *esClientCert, *esClientPrivateKey, *esInsecureSkipVerify)

	httpClient := &http.Client{
		Timeout: *esTimeout,
		Transport: &http.Transport{
			TLSClientConfig: tlsConfig,
		},
	}

	// version metric
	versionMetric := version.NewCollector(Name)
	prometheus.MustRegister(versionMetric)
	prometheus.MustRegister(collector.NewClusterHealth(logger, httpClient, esURL))
	prometheus.MustRegister(collector.NewNodes(logger, httpClient, esURL, *esAllNodes, *esNode))
	if *esExportIndices || *esExportShards {
		prometheus.MustRegister(collector.NewIndices(logger, httpClient, esURL, *esExportShards))
	}
	if *esExportSnapshots {
		prometheus.MustRegister(collector.NewSnapshots(logger, httpClient, esURL))
	}
	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		_, err = w.Write([]byte(`
			<title>Elasticsearch Exporter</title>

Elasticsearch Exporter

<a href="` + *metricsPath + `">Metrics

			`))
		if err != nil {
			_ = level.Error(logger).Log(
				"msg", "failed handling writer",
				"err", err,
			)
		}
	})

	_ = level.Info(logger).Log(
		"msg", "starting elasticsearch_exporter",
		"addr", *listenAddress,
	)

	if err := http.ListenAndServe(*listenAddress, nil); err != nil {
		_ = level.Error(logger).Log(
			"msg", "http server quit",
			"err", err,
		)
	}
}
<commit_msg>Set proxy for http.Transport to autoevaluate from Enviroment<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/go-kit/kit/log/level"
	"github.com/justwatchcom/elasticsearch_exporter/collector"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/version"
)

func main() {
	var (
		Name                 = "elasticsearch_exporter"
		listenAddress        = flag.String("web.listen-address", ":9108", "Address to listen on for web interface and telemetry.")
		metricsPath          = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
		esURI                = flag.String("es.uri", "http://localhost:9200", "HTTP API address of an Elasticsearch node.")
		esTimeout            = flag.Duration("es.timeout", 5*time.Second, "Timeout for trying to get stats from Elasticsearch.")
		esAllNodes           = flag.Bool("es.all", false, "Export stats for all nodes in the cluster. If used, this flag will override the flag es.node.")
		esNode               = flag.String("es.node", "_local", "Node's name of which metrics should be exposed.")
		esExportIndices      = flag.Bool("es.indices", false, "Export stats for indices in the cluster.")
		esExportShards       = flag.Bool("es.shards", false, "Export stats for shards in the cluster (implies es.indices=true).")
		esExportSnapshots    = flag.Bool("es.snapshots", false, "Export stats for the cluster snapshots.")
		esCA                 = flag.String("es.ca", "", "Path to PEM file that contains trusted CAs for the Elasticsearch connection.")
		esClientPrivateKey   = flag.String("es.client-private-key", "", "Path to PEM file that contains the private key for client auth when connecting to Elasticsearch.")
		esClientCert         = flag.String("es.client-cert", "", "Path to PEM file that contains the corresponding cert for the private key to connect to Elasticsearch.")
		esInsecureSkipVerify = flag.Bool("es.ssl-skip-verify", false, "Skip SSL verification when connecting to Elasticsearch.")
		logLevel             = flag.String("log.level", "info", "Sets the loglevel. Valid levels are debug, info, warn, error")
		logFormat            = flag.String("log.format", "logfmt", "Sets the log format. Valid formats are json and logfmt")
		logOutput            = flag.String("log.output", "stdout", "Sets the log output. Valid outputs are stdout and stderr")
		showVersion          = flag.Bool("version", false, "Show version and exit")
	)
	flag.Parse()

	if *showVersion {
		fmt.Print(version.Print(Name))
		os.Exit(0)
	}

	logger := getLogger(*logLevel, *logOutput, *logFormat)

	esURIEnv, ok := os.LookupEnv("ES_URI")
	if ok {
		*esURI = esURIEnv
	}
	esURL, err := url.Parse(*esURI)
	if err != nil {
		_ = level.Error(logger).Log(
			"msg", "failed to parse es.uri",
			"err", err,
		)
		os.Exit(1)
	}

	// returns nil if not provided and falls back to simple TCP.
	tlsConfig := createTLSConfig(*esCA, *esClientCert, *esClientPrivateKey, *esInsecureSkipVerify)

	httpClient := &http.Client{
		Timeout: *esTimeout,
		Transport: &http.Transport{
			TLSClientConfig: tlsConfig,
			Proxy:           http.ProxyFromEnvironment,
		},
	}

	// version metric
	versionMetric := version.NewCollector(Name)
	prometheus.MustRegister(versionMetric)
	prometheus.MustRegister(collector.NewClusterHealth(logger, httpClient, esURL))
	prometheus.MustRegister(collector.NewNodes(logger, httpClient, esURL, *esAllNodes, *esNode))
	if *esExportIndices || *esExportShards {
		prometheus.MustRegister(collector.NewIndices(logger, httpClient, esURL, *esExportShards))
	}
	if *esExportSnapshots {
		prometheus.MustRegister(collector.NewSnapshots(logger, httpClient, esURL))
	}
	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		_, err = w.Write([]byte(`
			<title>Elasticsearch Exporter</title>

Elasticsearch Exporter

<a href="` + *metricsPath + `">Metrics

			`))
		if err != nil {
			_ = level.Error(logger).Log(
				"msg", "failed handling writer",
				"err", err,
			)
		}
	})

	_ = level.Info(logger).Log(
		"msg", "starting elasticsearch_exporter",
		"addr", *listenAddress,
	)

	if err := http.ListenAndServe(*listenAddress, nil); err != nil {
		_ = level.Error(logger).Log(
			"msg", "http server quit",
			"err", err,
		)
	}
}
<|endoftext|>