<commit_before>package distconf

import (
	"sync"

	"time"

	"math"

	log "github.com/Sirupsen/logrus"
)

// Config gets configuration data from the first backing that has it
type Config struct {
	readers []Reader

	varsMutex      sync.Mutex
	registeredVars map[string]configVariable
}

type configVariable interface {
	Update(newValue []byte) error
}

type noopCloser struct {
}

func (n *noopCloser) Close() {
}

// Int object that can be referenced to get integer values from a backing config
func (c *Config) Int(key string, defaultVal int64) *Int {
	s := &intConf{
		defaultVal: defaultVal,
		Int: Int{
			currentVal: defaultVal,
		},
	}
	// Note: in race conditions 's' may not be the thing actually returned
	ret, okCast := c.register(key, s).(*intConf)
	if !okCast {
		log.WithField("key", key).Error("Registering key with multiple types!  FIX ME!!!!")
		return nil
	}
	return &ret.Int
}

// Float object that can be referenced to get float values from a backing config
func (c *Config) Float(key string, defaultVal float64) *Float {
	s := &floatConf{
		defaultVal: defaultVal,
		Float: Float{
			currentVal: math.Float64bits(defaultVal),
		},
	}
	// Note: in race conditions 's' may not be the thing actually returned
	ret, okCast := c.register(key, s).(*floatConf)
	if !okCast {
		log.WithField("key", key).Error("Registering key with multiple types!  FIX ME!!!!")
		return nil
	}
	return &ret.Float
}

// Str object that can be referenced to get string values from a backing config
func (c *Config) Str(key string, defaultVal string) *Str {
	s := &strConf{
		defaultVal: defaultVal,
	}
	s.currentVal.Store(defaultVal)
	// Note: in race conditions 's' may not be the thing actually returned
	ret, okCast := c.register(key, s).(*strConf)
	if !okCast {
		log.WithField("key", key).Error("Registering key with multiple types!  FIX ME!!!!")
		return nil
	}
	return &ret.Str
}

// Bool object that can be referenced to get boolean values from a backing config
func (c *Config) Bool(key string, defaultVal bool) *Bool {
	var defautlAsInt int32
	if defaultVal {
		defautlAsInt = 1
	} else {
		defautlAsInt = 0
	}

	s := &boolConf{
		defaultVal: defautlAsInt,
		Bool: Bool{
			currentVal: defautlAsInt,
		},
	}
	// Note: in race conditions 's' may not be the thing actually returned
	ret, okCast := c.register(key, s).(*boolConf)
	if !okCast {
		log.WithField("key", key).Error("Registering key with multiple types!  FIX ME!!!!")
		return nil
	}
	return &ret.Bool
}

// Duration returns a duration object that calls ParseDuration() on the given key
func (c *Config) Duration(key string, defaultVal time.Duration) *Duration {
	s := &durationConf{
		defaultVal: defaultVal,
		Duration: Duration{
			currentVal: defaultVal.Nanoseconds(),
		},
	}
	// Note: in race conditions 's' may not be the thing actually returned
	ret, okCast := c.register(key, s).(*durationConf)
	if !okCast {
		log.WithField("key", key).Error("Registering key with multiple types!  FIX ME!!!!")
		return nil
	}
	return &ret.Duration
}

// Close this config framework's readers.  Config variable results are undefined after this call.
func (c *Config) Close() {
	c.varsMutex.Lock()
	defer c.varsMutex.Unlock()
	for _, backing := range c.readers {
		backing.Close()
	}
}

func (c *Config) register(key string, configVariable configVariable) configVariable {
	c.varsMutex.Lock()
	defer c.varsMutex.Unlock()
	existing, exists := c.registeredVars[key]
	if exists {
		// Don't log if everything else is the same?
		log.WithField("key", key).Warn("Possible race registering key")
		c.refresh(key, existing)
		return existing
	}
	dynamicOnPath := c.refresh(key, configVariable)
	if dynamicOnPath {
		c.watch(key, configVariable)
	}
	c.registeredVars[key] = configVariable
	return configVariable
}

func (c *Config) refresh(key string, configVar configVariable) bool {
	dynamicReadersOnPath := false
	for _, backing := range c.readers {
		if !dynamicReadersOnPath {
			_, ok := backing.(Dynamic)
			if ok {
				dynamicReadersOnPath = true
			}
		}

		v, e := backing.Get(key)
		if e != nil {
			log.WithField("err", e).Warn("Unable to read from backing")
			continue
		}
		if v != nil {
			e = configVar.Update(v)
			if e != nil {
				log.WithField("err", e).Error("Invalid config bytes for " + key)
			}
			return dynamicReadersOnPath
		}
	}

	e := configVar.Update(nil)
	if e != nil {
		log.WithField("err", e).Warn("Unable to set bytes to nil/clear")
	}

	// If this is false, then the variable is fixed and can never change
	return dynamicReadersOnPath
}

func (c *Config) watch(key string, configVar configVariable) {
	for _, backing := range c.readers {
		d, ok := backing.(Dynamic)
		if ok {
			err := d.Watch(key, c.onBackingChange)
			if err != nil {
				log.WithField("err", err).WithField("key", key).Warn("Unable to watch for config var")
			}
		}
	}
}

func (c *Config) onBackingChange(key string) {
	m, exists := c.registeredVars[key]
	if !exists {
		log.WithField("key", key).Warn("Backing callback on variable that doesn't exist")
		return
	}
	c.refresh(key, m)
}

// Reader can get a []byte value for a config key
type Reader interface {
	Get(key string) ([]byte, error)
	Close()
}

// Writer can modify Config properties
type Writer interface {
	Write(key string, value []byte) error
}

type backingCallbackFunction func(string)

// A Dynamic config can change what it thinks a value is over time.
type Dynamic interface {
	Watch(key string, callback backingCallbackFunction) error
}

// A ReaderWriter can both read and write configuration information
type ReaderWriter interface {
	Reader
	Writer
}
<commit_msg>Fix new -race detected w/ go 1.5<commit_after>package distconf

import (
	"sync"

	"time"

	"math"

	log "github.com/Sirupsen/logrus"
)

// Config gets configuration data from the first backing that has it
type Config struct {
	readers []Reader

	varsMutex      sync.Mutex
	registeredVars map[string]configVariable
}

type configVariable interface {
	Update(newValue []byte) error
}

type noopCloser struct {
}

func (n *noopCloser) Close() {
}

// Int object that can be referenced to get integer values from a backing config
func (c *Config) Int(key string, defaultVal int64) *Int {
	s := &intConf{
		defaultVal: defaultVal,
		Int: Int{
			currentVal: defaultVal,
		},
	}
	// Note: in race conditions 's' may not be the thing actually returned
	ret, okCast := c.register(key, s).(*intConf)
	if !okCast {
		log.WithField("key", key).Error("Registering key with multiple types!  FIX ME!!!!")
		return nil
	}
	return &ret.Int
}

// Float object that can be referenced to get float values from a backing config
func (c *Config) Float(key string, defaultVal float64) *Float {
	s := &floatConf{
		defaultVal: defaultVal,
		Float: Float{
			currentVal: math.Float64bits(defaultVal),
		},
	}
	// Note: in race conditions 's' may not be the thing actually returned
	ret, okCast := c.register(key, s).(*floatConf)
	if !okCast {
		log.WithField("key", key).Error("Registering key with multiple types!  FIX ME!!!!")
		return nil
	}
	return &ret.Float
}

// Str object that can be referenced to get string values from a backing config
func (c *Config) Str(key string, defaultVal string) *Str {
	s := &strConf{
		defaultVal: defaultVal,
	}
	s.currentVal.Store(defaultVal)
	// Note: in race conditions 's' may not be the thing actually returned
	ret, okCast := c.register(key, s).(*strConf)
	if !okCast {
		log.WithField("key", key).Error("Registering key with multiple types!  FIX ME!!!!")
		return nil
	}
	return &ret.Str
}

// Bool object that can be referenced to get boolean values from a backing config
func (c *Config) Bool(key string, defaultVal bool) *Bool {
	var defautlAsInt int32
	if defaultVal {
		defautlAsInt = 1
	} else {
		defautlAsInt = 0
	}

	s := &boolConf{
		defaultVal: defautlAsInt,
		Bool: Bool{
			currentVal: defautlAsInt,
		},
	}
	// Note: in race conditions 's' may not be the thing actually returned
	ret, okCast := c.register(key, s).(*boolConf)
	if !okCast {
		log.WithField("key", key).Error("Registering key with multiple types!  FIX ME!!!!")
		return nil
	}
	return &ret.Bool
}

// Duration returns a duration object that calls ParseDuration() on the given key
func (c *Config) Duration(key string, defaultVal time.Duration) *Duration {
	s := &durationConf{
		defaultVal: defaultVal,
		Duration: Duration{
			currentVal: defaultVal.Nanoseconds(),
		},
	}
	// Note: in race conditions 's' may not be the thing actually returned
	ret, okCast := c.register(key, s).(*durationConf)
	if !okCast {
		log.WithField("key", key).Error("Registering key with multiple types!  FIX ME!!!!")
		return nil
	}
	return &ret.Duration
}

// Close this config framework's readers.  Config variable results are undefined after this call.
func (c *Config) Close() {
	c.varsMutex.Lock()
	defer c.varsMutex.Unlock()
	for _, backing := range c.readers {
		backing.Close()
	}
}

func (c *Config) register(key string, configVariable configVariable) configVariable {
	c.varsMutex.Lock()
	defer c.varsMutex.Unlock()
	existing, exists := c.registeredVars[key]
	if exists {
		// Don't log if everything else is the same?
		log.WithField("key", key).Warn("Possible race registering key")
		c.refresh(key, existing)
		return existing
	}
	dynamicOnPath := c.refresh(key, configVariable)
	if dynamicOnPath {
		c.watch(key, configVariable)
	}
	c.registeredVars[key] = configVariable
	return configVariable
}

func (c *Config) refresh(key string, configVar configVariable) bool {
	dynamicReadersOnPath := false
	for _, backing := range c.readers {
		if !dynamicReadersOnPath {
			_, ok := backing.(Dynamic)
			if ok {
				dynamicReadersOnPath = true
			}
		}

		v, e := backing.Get(key)
		if e != nil {
			log.WithField("err", e).Warn("Unable to read from backing")
			continue
		}
		if v != nil {
			e = configVar.Update(v)
			if e != nil {
				log.WithField("err", e).Error("Invalid config bytes for " + key)
			}
			return dynamicReadersOnPath
		}
	}

	e := configVar.Update(nil)
	if e != nil {
		log.WithField("err", e).Warn("Unable to set bytes to nil/clear")
	}

	// If this is false, then the variable is fixed and can never change
	return dynamicReadersOnPath
}

func (c *Config) watch(key string, configVar configVariable) {
	for _, backing := range c.readers {
		d, ok := backing.(Dynamic)
		if ok {
			err := d.Watch(key, c.onBackingChange)
			if err != nil {
				log.WithField("err", err).WithField("key", key).Warn("Unable to watch for config var")
			}
		}
	}
}

func (c *Config) onBackingChange(key string) {
	c.varsMutex.Lock()
	m, exists := c.registeredVars[key]
	c.varsMutex.Unlock()
	if !exists {
		log.WithField("key", key).Warn("Backing callback on variable that doesn't exist")
		return
	}
	c.refresh(key, m)
}

// Reader can get a []byte value for a config key
type Reader interface {
	Get(key string) ([]byte, error)
	Close()
}

// Writer can modify Config properties
type Writer interface {
	Write(key string, value []byte) error
}

type backingCallbackFunction func(string)

// A Dynamic config can change what it thinks a value is over time.
type Dynamic interface {
	Watch(key string, callback backingCallbackFunction) error
}

// A ReaderWriter can both read and write configuration information
type ReaderWriter interface {
	Reader
	Writer
}
<|endoftext|>


<commit_before>package kitsu

import (
	"strings"
	"time"
)

// AllAnime returns a stream of all anime objects (async).
// Be very careful to only use this function once as each
// call will start a new goroutine requesting the whole data.
func AllAnime() chan *Anime {
	channel := make(chan *Anime)
	url := "anime?page[limit]=20&page[offset]=0"
	ticker := time.NewTicker(500 * time.Millisecond)
	rateLimit := ticker.C

	go func() {
		defer close(channel)
		defer ticker.Stop()

		for {
			page, err := GetAnimePage(url)

			if err != nil {
				panic(err)
			}

			// Feed anime data from current page to the stream
			for _, anime := range page.Data {
				channel <- anime
			}

			nextURL := page.Links.Next

			// Did we reach the end?
			if nextURL == "" {
				break
			}

			// Cut off API base URL
			nextURL = strings.TrimPrefix(nextURL, APIBaseURL)

			// Continue with the next page
			url = nextURL

			// Wait for rate limiter to allow the next request
			<-rateLimit
		}
	}()

	return channel
}
<commit_msg>Added FixImageURL<commit_after>package kitsu

import (
	"strings"
	"time"
)

// AllAnime returns a stream of all anime objects (async).
// Be very careful to only use this function once as each
// call will start a new goroutine requesting the whole data.
func AllAnime() chan *Anime {
	channel := make(chan *Anime)
	url := "anime?page[limit]=20&page[offset]=0"
	ticker := time.NewTicker(500 * time.Millisecond)
	rateLimit := ticker.C

	go func() {
		defer close(channel)
		defer ticker.Stop()

		for {
			page, err := GetAnimePage(url)

			if err != nil {
				panic(err)
			}

			// Feed anime data from current page to the stream
			for _, anime := range page.Data {
				channel <- anime
			}

			nextURL := page.Links.Next

			// Did we reach the end?
			if nextURL == "" {
				break
			}

			// Cut off API base URL
			nextURL = strings.TrimPrefix(nextURL, APIBaseURL)

			// Continue with the next page
			url = nextURL

			// Wait for rate limiter to allow the next request
			<-rateLimit
		}
	}()

	return channel
}

// FixImageURL removes the right-most part, e.g. "?1416336000" from image URLs.
func FixImageURL(url string) string {
	questionMark := strings.IndexByte(url, '?')

	if questionMark == -1 {
		return url
	}

	return url[:questionMark]
}
<|endoftext|>


<commit_before>package hermes

import (
	"fmt"
	"net/http"
	"net/url"
	"runtime"
	"sync"
	"time"

	"strings"

	"github.com/PuerkitoBio/fetchbot"
	"github.com/PuerkitoBio/goquery"
)

var (
	mu sync.Mutex // Protect access to dup

	dup = map[string]bool{} // Duplicates table

	settingLinks = map[string]bool{} // Tracking link settings

	ingestionSet []Document // ingestion data TODO make non global

	badLinks []string // bad links TODO make non global
)

// Crawl function that will take a url string and start firing out some crawling functions
// it will return true/false based on the url root it starts with.
func Crawl(settings Settings, linkSettings CustomSettings, u *url.URL) ([]Document, bool) {
	// Create the muxer
	mux := fetchbot.NewMux()

	// Handle all errors the same
	mux.HandleErrors(fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
	}))

	// Handle GET requests for html responses, to parse the body and enqueue all links as HEAD
	// requests.
	mux.Response().Method("GET").ContentType("text/html").Handler(fetchbot.HandlerFunc(
		func(ctx *fetchbot.Context, res *http.Response, err error) {
			// Process the body to find the links
			doc, err := goquery.NewDocumentFromReader(res.Body)
			if err != nil {
				// find the bad links in the documents
				fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
				return
			}
			// Enqueue all links as HEAD requests
			enqueueLinks(ctx, doc, u, linkSettings)
		}))

	// Handle HEAD requests for html responses coming from the source host - we don't want
	// to crawl links from other hosts.
	mux.Response().Method("HEAD").Host(u.Host).ContentType("text/html").Handler(fetchbot.HandlerFunc(
		func(ctx *fetchbot.Context, res *http.Response, err error) {
			if _, err := ctx.Q.SendStringGet(ctx.Cmd.URL().String()); err != nil {
				fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
			}
		}))

	// Create the Fetcher, handle the logging first, then dispatch to the Muxer
	h := scrapeHandler(mux, linkSettings)
	if settings.StopAtURL != "" || settings.CancelAtURL != "" {
		stopURL := settings.StopAtURL
		if settings.CancelAtURL != "" {
			stopURL = settings.CancelAtURL
		}
		h = stopHandler(stopURL, settings.CancelAtURL != "", scrapeHandler(mux, linkSettings))
	}
	f := fetchbot.New(h)

	// set the fetchbots settings from flag parameters
	f.UserAgent = settings.UserAgent
	f.CrawlDelay = settings.CrawlDelay * time.Second
	f.WorkerIdleTTL = settings.WorkerIdleTTL * time.Second
	f.AutoClose = settings.AutoClose

	// First mem stat print must be right after creating the fetchbot
	if settings.MemStatsInterval > 0 {
		// Print starting stats
		printMemStats(nil)
		// Run at regular intervals
		runMemStats(f, settings.MemStatsInterval)
		// On exit, print ending stats after a GC
		defer func() {
			runtime.GC()
			printMemStats(nil)
		}()
	}

	// Start processing
	q := f.Start()

	// if a stop or cancel is requested after some duration, launch the goroutine
	// that will stop or cancel.
	if settings.StopDuration*time.Minute > 0 || settings.CancelDuration*time.Minute > 0 {
		after := settings.StopDuration * time.Minute
		stopFunc := q.Close
		if settings.CancelDuration != 0 {
			after = settings.CancelDuration * time.Minute
			stopFunc = q.Cancel
		}

		go func() {
			c := time.After(after)
			<-c
			stopFunc()
		}()
	}

	// Enqueue the seed, which is the first entry in the dup map
	dup[linkSettings.RootLink] = true
	_, err := q.SendStringGet(linkSettings.RootLink)
	if err != nil {
		fmt.Printf("[ERR] GET %s - %s\n", linkSettings.RootLink, err)
	}
	q.Block()

	return ingestionSet, true
}

// stopHandler stops the fetcher if the stopurl is reached. Otherwise it dispatches
// the call to the wrapped Handler.
func stopHandler(stopurl string, cancel bool, wrapped fetchbot.Handler) fetchbot.Handler {
	return fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		if ctx.Cmd.URL().String() == stopurl {
			fmt.Printf(">>>>> STOP URL %s\n", ctx.Cmd.URL())
			// generally not a good idea to stop/block from a handler goroutine
			// so do it in a separate goroutine
			go func() {
				if cancel {
					ctx.Q.Cancel()
				} else {
					ctx.Q.Close()
				}
			}()
			return
		}
		wrapped.Handle(ctx, res, err)
	})
}

// scrapeHandler will fire a scraper function on the page if successful response,
// append the scraped document stored for index ingestion
// and dispatches the call to the wrapped Handler.
func scrapeHandler(wrapped fetchbot.Handler, linkSettings CustomSettings) fetchbot.Handler {
	return fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		if err == nil {
			if res.StatusCode == 200 {
				url := ctx.Cmd.URL().String()
				responseDocument, err := Scrape(ctx, linkSettings)
				if err != nil {
					fmt.Printf("[ERR] SCRAPE URL: %s - %s", url, err)
				}
				ingestionSet = append(ingestionSet, responseDocument)
			}
			fmt.Printf("[%d] %s %s - %s\n", res.StatusCode, ctx.Cmd.Method(), ctx.Cmd.URL(), res.Header.Get("Content-Type"))
		}
		wrapped.Handle(ctx, res, err)
	})
}

// enqueueLinks will make sure we are adding links to the queue to be processed for crawling/scraping
// this will pull all the href attributes on pages, check for duplicates and add them to the queue
func enqueueLinks(ctx *fetchbot.Context, doc *goquery.Document, host *url.URL, settings CustomSettings) {
	mu.Lock()

	doc.Find("a[href]").Each(func(i int, s *goquery.Selection) {
		val, exists := s.Attr("href")
		if exists == false {
			fmt.Print("error: address within the document\n")
			return
		}

		// Resolve address
		u, err := ctx.Cmd.URL().Parse(val)
		if err != nil {
			fmt.Printf("error: resolve URL %s - %s\n", u, err)
			return
		}

		// check whether or not the link is an email link
		emailCheck := false
		func(s string, emailCheck *bool) {
			if strings.Contains(s, "mailto:") {
				*emailCheck = true
			}
		}(u.String(), &emailCheck)

		if emailCheck == true {
			fmt.Printf("[ERR] Email link - %s\n", u.String())
			return
		}

		// remove the 'www' from the URL so that we have better duplicate detection
		normalizeLink(u)

		// catch the duplicate urls here before trying to add them to the queue
		if !dup[u.String()] {
			// tld & subdomain
			if settings.TopLevelDomain == true && settings.Subdomain == true {
				rootDomain := getDomain(host.Host)
				current := getDomain(u.Host)

				if rootDomain == current {
					err := addLink(ctx, u)
					if err != nil {
						fmt.Printf("error: enqueue head %s - %s\n", u, err)
						return
					}
				} else {
					fmt.Printf("catch: out of domain scope -- %s != %s\n", u.Host, host)
				}
			}

			// tld check
			if settings.TopLevelDomain == true && settings.Subdomain == false {
				rootTLD := getDomain(host.Host)
				current := getTLD(u.Host)

				if rootTLD == current {
					err := addLink(ctx, u)
					if err != nil {
						fmt.Printf("error: enqueue head %s - %s\n", u, err)
						return
					}
				}
			}

			// subdomain check
			if settings.Subdomain == true && settings.TopLevelDomain == false {
				rootDomain := getDomain(host.Host)
				current := getDomain(u.Host)

				if rootDomain == current {
					err := addLink(ctx, u)
					if err != nil {
						fmt.Printf("error: enqueue head %s - %s\n", u, err)
						return
					}
				} else {
					fmt.Printf("catch: out of domain scope -- %s != %s\n", u.Host, host)
				}
			}
		}
	})
	mu.Unlock()
}

// remove the www from the URL host
func normalizeLink(u *url.URL) {
	s := strings.Split(u.Host, ".")
	if len(s) == 0 {
		fmt.Printf("[ERR] URL doesn't have a TLD: %s\n", u.Host)
	} else if s[0] == "www" {
		u.Host = strings.Join(s[1:], ".")
	}
}

// addLink will add a url to fetchbot's queue and to the global hashmap to audit for duplicates
func addLink(ctx *fetchbot.Context, u *url.URL) error {
	if _, err := ctx.Q.SendStringHead(u.String()); err != nil {
		return err
	}
	dup[u.String()] = true
	return nil
}

// getDomain will parse a url and return the domain with the tld on it (ie. example.com)
func getDomain(u string) (root string) {
	s := strings.Split(u, ".")
	if len(s) == 0 {
		root = u
		return
	}
	last := len(s) - 1
	runnerUp := last - 1
	root = s[runnerUp] + "." + s[last]
	return
}

// getTLD will parse a url type and return the top-level domain (.com, .edu, .gov, etc.)
func getTLD(u string) (tld string) {
	s := strings.Split(u, ".")
	if len(s) == 0 {
		tld = u
		return
	}
	last := len(s) - 1
	tld = s[last]
	return
}
<commit_msg>fix pointer issues with lengths on string parsing<commit_after>package hermes

import (
	"fmt"
	"net/http"
	"net/url"
	"runtime"
	"sync"
	"time"

	"strings"

	"github.com/PuerkitoBio/fetchbot"
	"github.com/PuerkitoBio/goquery"
)

var (
	mu sync.Mutex // Protect access to dup

	dup = map[string]bool{} // Duplicates table

	settingLinks = map[string]bool{} // Tracking link settings

	ingestionSet []Document // ingestion data TODO make non global

	badLinks []string // bad links TODO make non global
)

// Crawl function that will take a url string and start firing out some crawling functions
// it will return true/false based on the url root it starts with.
func Crawl(settings Settings, linkSettings CustomSettings, u *url.URL) ([]Document, bool) {
	// Create the muxer
	mux := fetchbot.NewMux()

	// Handle all errors the same
	mux.HandleErrors(fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
	}))

	// Handle GET requests for html responses, to parse the body and enqueue all links as HEAD
	// requests.
	mux.Response().Method("GET").ContentType("text/html").Handler(fetchbot.HandlerFunc(
		func(ctx *fetchbot.Context, res *http.Response, err error) {
			// Process the body to find the links
			doc, err := goquery.NewDocumentFromReader(res.Body)
			if err != nil {
				// find the bad links in the documents
				fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
				return
			}
			// Enqueue all links as HEAD requests
			enqueueLinks(ctx, doc, u, linkSettings)
		}))

	// Handle HEAD requests for html responses coming from the source host - we don't want
	// to crawl links from other hosts.
	mux.Response().Method("HEAD").Host(u.Host).ContentType("text/html").Handler(fetchbot.HandlerFunc(
		func(ctx *fetchbot.Context, res *http.Response, err error) {
			if _, err := ctx.Q.SendStringGet(ctx.Cmd.URL().String()); err != nil {
				fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
			}
		}))

	// Create the Fetcher, handle the logging first, then dispatch to the Muxer
	h := scrapeHandler(mux, linkSettings)
	if settings.StopAtURL != "" || settings.CancelAtURL != "" {
		stopURL := settings.StopAtURL
		if settings.CancelAtURL != "" {
			stopURL = settings.CancelAtURL
		}
		h = stopHandler(stopURL, settings.CancelAtURL != "", scrapeHandler(mux, linkSettings))
	}
	f := fetchbot.New(h)

	// set the fetchbots settings from flag parameters
	f.UserAgent = settings.UserAgent
	f.CrawlDelay = settings.CrawlDelay * time.Second
	f.WorkerIdleTTL = settings.WorkerIdleTTL * time.Second
	f.AutoClose = settings.AutoClose

	// First mem stat print must be right after creating the fetchbot
	if settings.MemStatsInterval > 0 {
		// Print starting stats
		printMemStats(nil)
		// Run at regular intervals
		runMemStats(f, settings.MemStatsInterval)
		// On exit, print ending stats after a GC
		defer func() {
			runtime.GC()
			printMemStats(nil)
		}()
	}

	// Start processing
	q := f.Start()

	// if a stop or cancel is requested after some duration, launch the goroutine
	// that will stop or cancel.
	if settings.StopDuration*time.Minute > 0 || settings.CancelDuration*time.Minute > 0 {
		after := settings.StopDuration * time.Minute
		stopFunc := q.Close
		if settings.CancelDuration != 0 {
			after = settings.CancelDuration * time.Minute
			stopFunc = q.Cancel
		}

		go func() {
			c := time.After(after)
			<-c
			stopFunc()
		}()
	}

	// Enqueue the seed, which is the first entry in the dup map
	dup[linkSettings.RootLink] = true
	_, err := q.SendStringGet(linkSettings.RootLink)
	if err != nil {
		fmt.Printf("[ERR] GET %s - %s\n", linkSettings.RootLink, err)
	}
	q.Block()

	return ingestionSet, true
}

// stopHandler stops the fetcher if the stopurl is reached. Otherwise it dispatches
// the call to the wrapped Handler.
func stopHandler(stopurl string, cancel bool, wrapped fetchbot.Handler) fetchbot.Handler {
	return fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		if ctx.Cmd.URL().String() == stopurl {
			fmt.Printf(">>>>> STOP URL %s\n", ctx.Cmd.URL())
			// generally not a good idea to stop/block from a handler goroutine
			// so do it in a separate goroutine
			go func() {
				if cancel {
					ctx.Q.Cancel()
				} else {
					ctx.Q.Close()
				}
			}()
			return
		}
		wrapped.Handle(ctx, res, err)
	})
}

// scrapeHandler will fire a scraper function on the page if successful response,
// append the scraped document stored for index ingestion
// and dispatches the call to the wrapped Handler.
func scrapeHandler(wrapped fetchbot.Handler, linkSettings CustomSettings) fetchbot.Handler {
	return fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		if err == nil {
			if res.StatusCode == 200 {
				url := ctx.Cmd.URL().String()
				responseDocument, err := Scrape(ctx, linkSettings)
				if err != nil {
					fmt.Printf("[ERR] SCRAPE URL: %s - %s", url, err)
				}
				ingestionSet = append(ingestionSet, responseDocument)
			}
			fmt.Printf("[%d] %s %s - %s\n", res.StatusCode, ctx.Cmd.Method(), ctx.Cmd.URL(), res.Header.Get("Content-Type"))
		}
		wrapped.Handle(ctx, res, err)
	})
}

// enqueueLinks will make sure we are adding links to the queue to be processed for crawling/scraping
// this will pull all the href attributes on pages, check for duplicates and add them to the queue
func enqueueLinks(ctx *fetchbot.Context, doc *goquery.Document, host *url.URL, settings CustomSettings) {
	mu.Lock()

	doc.Find("a[href]").Each(func(i int, s *goquery.Selection) {
		val, exists := s.Attr("href")
		if exists == false {
			fmt.Print("error: address within the document\n")
			return
		}

		// Resolve address
		u, err := ctx.Cmd.URL().Parse(val)
		if err != nil {
			fmt.Printf("error: resolve URL %s - %s\n", u, err)
			return
		}

		// check whether or not the link is an email link
		emailCheck := false
		func(s string, emailCheck *bool) {
			if strings.Contains(s, "mailto:") {
				*emailCheck = true
			}
		}(u.String(), &emailCheck)

		if emailCheck == true {
			fmt.Printf("[ERR] Email link - %s\n", u.String())
			return
		}

		// remove the 'www' from the URL so that we have better duplicate detection
		normalizeLink(u)

		// catch the duplicate urls here before trying to add them to the queue
		if !dup[u.String()] {
			// tld & subdomain
			if settings.TopLevelDomain == true && settings.Subdomain == true {
				rootDomain := getDomain(host.Host)
				current := getDomain(u.Host)

				if rootDomain == current {
					err := addLink(ctx, u)
					if err != nil {
						fmt.Printf("error: enqueue head %s - %s\n", u, err)
						return
					}
				} else {
					fmt.Printf("catch: out of domain scope -- %s != %s\n", u.Host, host)
				}
			}

			// tld check
			if settings.TopLevelDomain == true && settings.Subdomain == false {
				rootTLD := getDomain(host.Host)
				current := getTLD(u.Host)

				if rootTLD == current {
					err := addLink(ctx, u)
					if err != nil {
						fmt.Printf("error: enqueue head %s - %s\n", u, err)
						return
					}
				}
			}

			// subdomain check
			if settings.Subdomain == true && settings.TopLevelDomain == false {
				rootDomain := getDomain(host.Host)
				current := getDomain(u.Host)

				if rootDomain == current {
					err := addLink(ctx, u)
					if err != nil {
						fmt.Printf("error: enqueue head %s - %s\n", u, err)
						return
					}
				} else {
					fmt.Printf("catch: out of domain scope -- %s != %s\n", u.Host, host)
				}
			}
		}
	})
	mu.Unlock()
}

// remove the www from the URL host
func normalizeLink(u *url.URL) {
	s := strings.Split(u.Host, ".")
	if len(s) == 0 {
		fmt.Printf("[ERR] URL doesn't have a TLD: %s\n", u.Host)
	} else if s[0] == "www" {
		u.Host = strings.Join(s[1:], ".")
	}
}

// addLink will add a url to fetchbot's queue and to the global hashmap to audit for duplicates
func addLink(ctx *fetchbot.Context, u *url.URL) error {
	if _, err := ctx.Q.SendStringHead(u.String()); err != nil {
		return err
	}
	dup[u.String()] = true
	return nil
}

// getDomain will parse a url and return the domain with the tld on it (ie. example.com)
func getDomain(u string) (root string) {
	s := strings.Split(u, ".")
	if len(s) == 0 {
		root = u
		return
	}
	last := len(s) - 1
	runnerUp := len(s) - 2
	root = s[runnerUp] + "." + s[last]
	return
}

// getTLD will parse a url type and return the top-level domain (.com, .edu, .gov, etc.)
func getTLD(u string) (tld string) {
	s := strings.Split(u, ".")
	if len(s) == 0 {
		tld = u
		return
	} else if len(s) > 0 {
		last := len(s) - 1
		tld = s[last]
	}
	tld = u
	return
}
<|endoftext|>


<commit_before>package police

import (
	"fmt"
)

type Crime struct {
	Category        string        `json:"category"`
	PersistentID    string        `json:"persistent_id"`
	Month           string        `json:"month"`
	Location        Location      `json:"location"`
	Context         string        `json:"context"`
	ID              uint64        `json:"id"`
	LocationType    string        `json:"location_type"`
	LocationSubtype string        `json:"location_subtype"`
	Outcome         OutcomeStatus `json:"outcome_status"`
}

type Location struct {
	Latitude  string `json:"latitude"`
	Longitude string `json:"longitude"`
	Street    `json:"street"`
}

type Street struct {
	ID   uint64 `json:"id"`
	Name string `json:"name"`
}

type OutcomeStatus struct {
	Category string `json:"category"`
	Date     string `json:"date"`
}

func (c Client) StreetCrime(lat, long string, date string) ([]Crime, error) {
	url := fmt.Sprintf("crimes-street/all-crime?lat=%s&lng=%s&date=%s", lat, long, date)
	var crimes []Crime
	if err := c.decodeJSONResponse(url, &crimes); err != nil {
		return nil, err
	}
	return crimes, nil
}
<commit_msg>add support for choosing a crime category<commit_after>package police

import (
	"fmt"
)

type Crime struct {
	Category        string        `json:"category"`
	PersistentID    string        `json:"persistent_id"`
	Month           string        `json:"month"`
	Location        Location      `json:"location"`
	Context         string        `json:"context"`
	ID              uint64        `json:"id"`
	LocationType    string        `json:"location_type"`
	LocationSubtype string        `json:"location_subtype"`
	Outcome         OutcomeStatus `json:"outcome_status"`
}

type Location struct {
	Latitude  string `json:"latitude"`
	Longitude string `json:"longitude"`
	Street    `json:"street"`
}

type Street struct {
	ID   uint64 `json:"id"`
	Name string `json:"name"`
}

type OutcomeStatus struct {
	Category string `json:"category"`
	Date     string `json:"date"`
}

func (c Client) StreetCrime(lat, long float64, date string, category string) ([]Crime, error) {
	if category == "" {
		category = "all-crime"
	}
	url := fmt.Sprintf("crimes-street/%s?lat=%d&lng=%d&date=%s", category, lat, long, date)
	var crimes []Crime
	if err := c.decodeJSONResponse(url, &crimes); err != nil {
		return nil, err
	}
	return crimes, nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// +build ignore

package main

import (
	"archive/tar"
	"archive/zip"
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
)

var (
	versionRe = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
	goarch    string
	goos      string
	noupgrade bool
)

const minGoVersion = 1.3

func main() {
	log.SetOutput(os.Stdout)
	log.SetFlags(0)

	if os.Getenv("GOPATH") == "" {
		cwd, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		gopath := filepath.Clean(filepath.Join(cwd, "../../../../"))
		log.Println("GOPATH is", gopath)
		os.Setenv("GOPATH", gopath)
	}
	os.Setenv("PATH", fmt.Sprintf("%s%cbin%c%s", os.Getenv("GOPATH"), os.PathSeparator, os.PathListSeparator, os.Getenv("PATH")))

	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
	flag.BoolVar(&noupgrade, "no-upgrade", false, "Disable upgrade functionality")
	flag.Parse()

	checkRequiredGoVersion()

	if check() != nil {
		setup()
	}

	if flag.NArg() == 0 {
		install("./cmd/...")
		return
	}

	switch flag.Arg(0) {
	case "install":
		pkg := "./cmd/..."
		if flag.NArg() > 2 {
			pkg = flag.Arg(1)
		}
		install(pkg)

	case "build":
		pkg := "./cmd/syncthing"
		if flag.NArg() > 2 {
			pkg = flag.Arg(1)
		}
		var tags []string
		if noupgrade {
			tags = []string{"noupgrade"}
		}
		build(pkg, tags)

	case "test":
		pkg := "./..."
		if flag.NArg() > 2 {
			pkg = flag.Arg(1)
		}
		test(pkg)

	case "assets":
		assets()

	case "xdr":
		xdr()

	case "translate":
		translate()

	case "transifex":
		transifex()

	case "deps":
		deps()

	case "tar":
		buildTar()

	case "zip":
		buildZip()

	case "clean":
		clean()

	default:
		log.Fatalf("Unknown command %q", flag.Arg(0))
	}
}

func check() error {
	_, err := exec.LookPath("godep")
	return err
}

func checkRequiredGoVersion() {
	ver := run("go", "version")
	re := regexp.MustCompile(`go version go(\d+\.\d+)`)
	if m := re.FindSubmatch(ver); len(m) == 2 {
		vs := string(m[1])
		// This is a standard go build. Verify that it's new enough.
		f, err := strconv.ParseFloat(vs, 64)
		if err != nil {
			log.Printf("*** Could parse Go version out of %q.\n*** This isn't known to work, proceed on your own risk.", vs)
			return
		}
		if f < minGoVersion {
			log.Fatalf("*** Go version %.01f is less than required %.01f.\n*** This is known not to work, not proceeding.", f, minGoVersion)
		}
	} else {
		log.Printf("*** Unknown Go version %q.\n*** This isn't known to work, proceed on your own risk.", ver)
	}
}

func setup() {
	runPrint("go", "get", "-v", "code.google.com/p/go.tools/cmd/cover")
	runPrint("go", "get", "-v", "code.google.com/p/go.tools/cmd/vet")
	runPrint("go", "get", "-v", "code.google.com/p/go.net/html")
	runPrint("go", "get", "-v", "github.com/tools/godep")
}

func test(pkg string) {
	runPrint("godep", "go", "test", pkg)
}

func install(pkg string) {
	os.Setenv("GOBIN", "./bin")
	setBuildEnv()
	runPrint("godep", "go", "install", "-ldflags", ldflags(), pkg)
}

func build(pkg string, tags []string) {
	rmr("syncthing", "syncthing.exe")
	args := []string{"go", "build", "-ldflags", ldflags()}
	if len(tags) > 0 {
		args = append(args, "-tags", strings.Join(tags, ","))
	}
	args = append(args, pkg)
	setBuildEnv()
	runPrint("godep", args...)
}

func buildTar() {
	name := archiveName()
	var tags []string
	if noupgrade {
		tags = []string{"noupgrade"}
		name += "-noupgrade"
	}
	build("./cmd/syncthing", tags)
	filename := name + ".tar.gz"
	tarGz(filename, []archiveFile{
		{"README.md", name + "/README.txt"},
		{"LICENSE", name + "/LICENSE.txt"},
		{"CONTRIBUTORS", name + "/CONTRIBUTORS.txt"},
		{"syncthing", name + "/syncthing"},
	})
	log.Println(filename)
}

func buildZip() {
	name := archiveName()
	var tags []string
	if noupgrade {
		tags = []string{"noupgrade"}
		name += "-noupgrade"
	}
	build("./cmd/syncthing", tags)
	filename := name + ".zip"
	zipFile(filename, []archiveFile{
		{"README.md", name + "/README.txt"},
		{"LICENSE", name + "/LICENSE.txt"},
		{"CONTRIBUTORS", name + "/CONTRIBUTORS.txt"},
		{"syncthing.exe", name + "/syncthing.exe"},
	})
	log.Println(filename)
}

func setBuildEnv() {
	os.Setenv("GOOS", goos)
	if strings.HasPrefix(goarch, "arm") {
		os.Setenv("GOARCH", "arm")
		os.Setenv("GOARM", goarch[4:])
	} else {
		os.Setenv("GOARCH", goarch)
	}
	if goarch == "386" {
		os.Setenv("GO386", "387")
	}
}

func assets() {
	runPipe("auto/gui.files.go", "godep", "go", "run", "cmd/genassets/main.go", "gui")
}

func xdr() {
	for _, f := range []string{"discover/packets", "files/leveldb", "protocol/message"} {
		runPipe(f+"_xdr.go", "go", "run", "./Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go", "--", f+".go")
	}
}

func translate() {
	os.Chdir("gui")
	runPipe("lang-en-new.json", "go", "run", "../cmd/translate/main.go", "lang-en.json", "index.html")
	os.Remove("lang-en.json")
	err := os.Rename("lang-en-new.json", "lang-en.json")
	if err != nil {
		log.Fatal(err)
	}
	os.Chdir("..")
}

func transifex() {
	os.Chdir("gui")
	runPrint("go", "run", "../cmd/transifexdl/main.go")
	os.Chdir("..")
	assets()
}

func deps() {
	rmr("Godeps")
	runPrint("godep", "save", "./cmd/...")
}

func clean() {
	rmr("bin", "Godeps/_workspace/pkg", "Godeps/_workspace/bin")
	rmr(filepath.Join(os.Getenv("GOPATH"), fmt.Sprintf("pkg/%s_%s/github.com/syncthing", goos, goarch)))
}

func ldflags() string {
	var b bytes.Buffer
	b.WriteString("-w")
	b.WriteString(fmt.Sprintf(" -X main.Version %s", version()))
	b.WriteString(fmt.Sprintf(" -X main.BuildStamp %d", buildStamp()))
	b.WriteString(fmt.Sprintf(" -X main.BuildUser %s", buildUser()))
	b.WriteString(fmt.Sprintf(" -X main.BuildHost %s", buildHost()))
	b.WriteString(fmt.Sprintf(" -X main.BuildEnv %s", buildEnvironment()))
	if strings.HasPrefix(goarch, "arm") {
		b.WriteString(fmt.Sprintf(" -X main.GoArchExtra %s", goarch[3:]))
	}
	return b.String()
}

func rmr(paths ...string) {
	for _, path := range paths {
		log.Println("rm -r", path)
		os.RemoveAll(path)
	}
}

func version() string {
	v := run("git", "describe", "--always", "--dirty")
	v = versionRe.ReplaceAllFunc(v, func(s []byte) []byte {
		s[0] = '+'
		return s
	})
	return string(v)
}

func buildStamp() int64 {
	bs := run("git", "show", "-s", "--format=%ct")
	s, _ := strconv.ParseInt(string(bs), 10, 64)
	return s
}

func buildUser() string {
	u, err := user.Current()
	if err != nil {
		return "unknown-user"
	}
	return strings.Replace(u.Username, " ", "-", -1)
}

func buildHost() string {
	h, err := os.Hostname()
	if err != nil {
		return "unknown-host"
	}
	return h
}

func buildEnvironment() string {
	if v := os.Getenv("ENVIRONMENT"); len(v) > 0 {
		return v
	}
	return "default"
}

func buildArch() string {
	os := goos
	if os == "darwin" {
		os = "macosx"
	}
	return fmt.Sprintf("%s-%s", os, goarch)
}

func archiveName() string {
	return fmt.Sprintf("syncthing-%s-%s", buildArch(), version())
}

func run(cmd string, args ...string) []byte {
	ecmd := exec.Command(cmd, args...)
	bs, err := ecmd.CombinedOutput()
	if err != nil {
		log.Println(cmd, strings.Join(args, " "))
		log.Println(string(bs))
		log.Fatal(err)
	}
	return bytes.TrimSpace(bs)
}

func runPrint(cmd string, args ...string) {
	log.Println(cmd, strings.Join(args, " "))
	ecmd := exec.Command(cmd, args...)
	ecmd.Stdout = os.Stdout
	ecmd.Stderr = os.Stderr
	err := ecmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func runPipe(file, cmd string, args ...string) {
	log.Println(cmd, strings.Join(args, " "), ">", file)
	fd, err := os.Create(file)
	if err != nil {
		log.Fatal(err)
	}
	ecmd := exec.Command(cmd, args...)
	ecmd.Stdout = fd
	ecmd.Stderr = os.Stderr
	err = ecmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

type archiveFile struct {
	src string
	dst string
}

func tarGz(out string, files []archiveFile) {
	fd, err := os.Create(out)
	if err != nil {
		log.Fatal(err)
	}

	gw := gzip.NewWriter(fd)
	tw := tar.NewWriter(gw)

	for _, f := range files {
		sf, err := os.Open(f.src)
		if err != nil {
			log.Fatal(err)
		}

		info, err := sf.Stat()
		if err != nil {
			log.Fatal(err)
		}
		h := &tar.Header{
			Name:    f.dst,
			Size:    info.Size(),
			Mode:    int64(info.Mode()),
			ModTime: info.ModTime(),
		}

		err = tw.WriteHeader(h)
		if err != nil {
			log.Fatal(err)
		}
		_, err = io.Copy(tw, sf)
		if err != nil {
			log.Fatal(err)
		}
		sf.Close()
	}

	err = tw.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = gw.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = fd.Close()
	if err != nil {
		log.Fatal(err)
	}
}

func zipFile(out string, files []archiveFile) {
	fd, err := os.Create(out)
	if err != nil {
		log.Fatal(err)
	}

	zw := zip.NewWriter(fd)

	for _, f := range files {
		sf, err := os.Open(f.src)
		if err != nil {
			log.Fatal(err)
		}

		info, err := sf.Stat()
		if err != nil {
			log.Fatal(err)
		}

		fh, err := zip.FileInfoHeader(info)
		if err != nil {
			log.Fatal(err)
		}
		fh.Name = f.dst
		fh.Method = zip.Deflate

		if strings.HasSuffix(f.dst, ".txt") {
			// Text file. Read it and convert line endings.
			bs, err := ioutil.ReadAll(sf)
			if err != nil {
				log.Fatal(err)
			}
			bs = bytes.Replace(bs, []byte{'\n'}, []byte{'\n', '\r'}, -1)
			fh.UncompressedSize = uint32(len(bs))
			fh.UncompressedSize64 = uint64(len(bs))

			of, err := zw.CreateHeader(fh)
			if err != nil {
				log.Fatal(err)
			}
			of.Write(bs)
		} else {
			// Binary file. Copy verbatim.
			of, err := zw.CreateHeader(fh)
			if err != nil {
				log.Fatal(err)
			}
			_, err = io.Copy(of, sf)
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	err = zw.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = fd.Close()
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Fix build.go on Windows<commit_after>// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// +build ignore

package main

import (
	"archive/tar"
	"archive/zip"
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
)

var (
	versionRe = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
	goarch    string
	goos      string
	noupgrade bool
)

const minGoVersion = 1.3

func main() {
	log.SetOutput(os.Stdout)
	log.SetFlags(0)

	if os.Getenv("GOPATH") == "" {
		cwd, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		gopath := filepath.Clean(filepath.Join(cwd, "../../../../"))
		log.Println("GOPATH is", gopath)
		os.Setenv("GOPATH", gopath)
	}
	os.Setenv("PATH", fmt.Sprintf("%s%cbin%c%s", os.Getenv("GOPATH"), os.PathSeparator, os.PathListSeparator, os.Getenv("PATH")))

	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
	flag.BoolVar(&noupgrade, "no-upgrade", false, "Disable upgrade functionality")
	flag.Parse()

	checkRequiredGoVersion()

	if check() != nil {
		setup()
	}

	if flag.NArg() == 0 {
		install("./cmd/...")
		return
	}

	switch flag.Arg(0) {
	case "install":
		pkg := "./cmd/..."
		if flag.NArg() > 2 {
			pkg = flag.Arg(1)
		}
		install(pkg)

	case "build":
		pkg := "./cmd/syncthing"
		if flag.NArg() > 2 {
			pkg = flag.Arg(1)
		}
		var tags []string
		if noupgrade {
			tags = []string{"noupgrade"}
		}
		build(pkg, tags)

	case "test":
		pkg := "./..."
		if flag.NArg() > 2 {
			pkg = flag.Arg(1)
		}
		test(pkg)

	case "assets":
		assets()

	case "xdr":
		xdr()

	case "translate":
		translate()

	case "transifex":
		transifex()

	case "deps":
		deps()

	case "tar":
		buildTar()

	case "zip":
		buildZip()

	case "clean":
		clean()

	default:
		log.Fatalf("Unknown command %q", flag.Arg(0))
	}
}

func check() error {
	_, err := exec.LookPath("godep")
	return err
}

func checkRequiredGoVersion() {
	ver := run("go", "version")
	re := regexp.MustCompile(`go version go(\d+\.\d+)`)
	if m := re.FindSubmatch(ver); len(m) == 2 {
		vs := string(m[1])
		// This is a standard go build. Verify that it's new enough.
		f, err := strconv.ParseFloat(vs, 64)
		if err != nil {
			log.Printf("*** Could parse Go version out of %q.\n*** This isn't known to work, proceed on your own risk.", vs)
			return
		}
		if f < minGoVersion {
			log.Fatalf("*** Go version %.01f is less than required %.01f.\n*** This is known not to work, not proceeding.", f, minGoVersion)
		}
	} else {
		log.Printf("*** Unknown Go version %q.\n*** This isn't known to work, proceed on your own risk.", ver)
	}
}

func setup() {
	runPrint("go", "get", "-v", "code.google.com/p/go.tools/cmd/cover")
	runPrint("go", "get", "-v", "code.google.com/p/go.tools/cmd/vet")
	runPrint("go", "get", "-v", "code.google.com/p/go.net/html")
	runPrint("go", "get", "-v", "github.com/tools/godep")
}

func test(pkg string) {
	runPrint("godep", "go", "test", pkg)
}

func install(pkg string) {
	os.Setenv("GOBIN", "./bin")
	setBuildEnv()
	runPrint("godep", "go", "install", "-ldflags", ldflags(), pkg)
}

func build(pkg string, tags []string) {
	rmr("syncthing", "syncthing.exe")
	args := []string{"go", "build", "-ldflags", ldflags()}
	if len(tags) > 0 {
		args = append(args, "-tags", strings.Join(tags, ","))
	}
	args = append(args, pkg)
	setBuildEnv()
	runPrint("godep", args...)
}

func buildTar() {
	name := archiveName()
	var tags []string
	if noupgrade {
		tags = []string{"noupgrade"}
		name += "-noupgrade"
	}
	build("./cmd/syncthing", tags)
	filename := name + ".tar.gz"
	tarGz(filename, []archiveFile{
		{"README.md", name + "/README.txt"},
		{"LICENSE", name + "/LICENSE.txt"},
		{"CONTRIBUTORS", name + "/CONTRIBUTORS.txt"},
		{"syncthing", name + "/syncthing"},
	})
	log.Println(filename)
}

func buildZip() {
	name := archiveName()
	var tags []string
	if noupgrade {
		tags = []string{"noupgrade"}
		name += "-noupgrade"
	}
	build("./cmd/syncthing", tags)
	filename := name + ".zip"
	zipFile(filename, []archiveFile{
		{"README.md", name + "/README.txt"},
		{"LICENSE", name + "/LICENSE.txt"},
		{"CONTRIBUTORS", name + "/CONTRIBUTORS.txt"},
		{"syncthing.exe", name + "/syncthing.exe"},
	})
	log.Println(filename)
}

func setBuildEnv() {
	os.Setenv("GOOS", goos)
	if strings.HasPrefix(goarch, "arm") {
		os.Setenv("GOARCH", "arm")
		os.Setenv("GOARM", goarch[4:])
	} else {
		os.Setenv("GOARCH", goarch)
	}
	if goarch == "386" {
		os.Setenv("GO386", "387")
	}
}

func assets() {
	runPipe("auto/gui.files.go", "godep", "go", "run", "cmd/genassets/main.go", "gui")
}

func xdr() {
	for _, f := range []string{"discover/packets", "files/leveldb", "protocol/message"} {
		runPipe(f+"_xdr.go", "go", "run", "./Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go", "--", f+".go")
	}
}

func translate() {
	os.Chdir("gui")
	runPipe("lang-en-new.json", "go", "run", "../cmd/translate/main.go", "lang-en.json", "index.html")
	os.Remove("lang-en.json")
	err := os.Rename("lang-en-new.json", "lang-en.json")
	if err != nil {
		log.Fatal(err)
	}
	os.Chdir("..")
}

func transifex() {
	os.Chdir("gui")
	runPrint("go", "run", "../cmd/transifexdl/main.go")
	os.Chdir("..")
	assets()
}

func deps() {
	rmr("Godeps")
	runPrint("godep", "save", "./cmd/...")
}

func clean() {
	rmr("bin", "Godeps/_workspace/pkg", "Godeps/_workspace/bin")
	rmr(filepath.Join(os.Getenv("GOPATH"), fmt.Sprintf("pkg/%s_%s/github.com/syncthing", goos, goarch)))
}

func ldflags() string {
	var b bytes.Buffer
	b.WriteString("-w")
	b.WriteString(fmt.Sprintf(" -X main.Version %s", version()))
	b.WriteString(fmt.Sprintf(" -X main.BuildStamp %d", buildStamp()))
	b.WriteString(fmt.Sprintf(" -X main.BuildUser %s", buildUser()))
	b.WriteString(fmt.Sprintf(" -X main.BuildHost %s", buildHost()))
	b.WriteString(fmt.Sprintf(" -X main.BuildEnv %s", buildEnvironment()))
	if strings.HasPrefix(goarch, "arm") {
		b.WriteString(fmt.Sprintf(" -X main.GoArchExtra %s", goarch[3:]))
	}
	return b.String()
}

func rmr(paths ...string) {
	for _, path := range paths {
		log.Println("rm -r", path)
		os.RemoveAll(path)
	}
}

func version() string {
	v := run("git", "describe", "--always", "--dirty")
	v = versionRe.ReplaceAllFunc(v, func(s []byte) []byte {
		s[0] = '+'
		return s
	})
	return string(v)
}

func buildStamp() int64 {
	bs := run("git", "show", "-s", "--format=%ct")
	s, _ := strconv.ParseInt(string(bs), 10, 64)
	return s
}

func buildUser() string {
	u, err := user.Current()
	if err != nil {
		return "unknown-user"
	}
	return strings.Replace(u.Username, " ", "-", -1)
}

func buildHost() string {
	h, err := os.Hostname()
	if err != nil {
		return "unknown-host"
	}
	return h
}

func buildEnvironment() string {
	if v := os.Getenv("ENVIRONMENT"); len(v) > 0 {
		return v
	}
	return "default"
}

func buildArch() string {
	os := goos
	if os == "darwin" {
		os = "macosx"
	}
	return fmt.Sprintf("%s-%s", os, goarch)
}

func archiveName() string {
	return fmt.Sprintf("syncthing-%s-%s", buildArch(), version())
}

func run(cmd string, args ...string) []byte {
	ecmd := exec.Command(cmd, args...)
	bs, err := ecmd.CombinedOutput()
	if err != nil {
		log.Println(cmd, strings.Join(args, " "))
		log.Println(string(bs))
		log.Fatal(err)
	}
	return bytes.TrimSpace(bs)
}

func runPrint(cmd string, args ...string) {
	log.Println(cmd, strings.Join(args, " "))
	ecmd := exec.Command(cmd, args...)
	ecmd.Stdout = os.Stdout
	ecmd.Stderr = os.Stderr
	err := ecmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func runPipe(file, cmd string, args ...string) {
	log.Println(cmd, strings.Join(args, " "), ">", file)
	fd, err := os.Create(file)
	if err != nil {
		log.Fatal(err)
	}
	ecmd := exec.Command(cmd, args...)
	ecmd.Stdout = fd
	ecmd.Stderr = os.Stderr
	err = ecmd.Run()
	if err != nil {
		log.Fatal(err)
	}
	fd.Close()
}

type archiveFile struct {
	src string
	dst string
}

func tarGz(out string, files []archiveFile) {
	fd, err := os.Create(out)
	if err != nil {
		log.Fatal(err)
	}

	gw := gzip.NewWriter(fd)
	tw := tar.NewWriter(gw)

	for _, f := range files {
		sf, err := os.Open(f.src)
		if err != nil {
			log.Fatal(err)
		}

		info, err := sf.Stat()
		if err != nil {
			log.Fatal(err)
		}
		h := &tar.Header{
			Name:    f.dst,
			Size:    info.Size(),
			Mode:    int64(info.Mode()),
			ModTime: info.ModTime(),
		}

		err = tw.WriteHeader(h)
		if err != nil {
			log.Fatal(err)
		}
		_, err = io.Copy(tw, sf)
		if err != nil {
			log.Fatal(err)
		}
		sf.Close()
	}

	err = tw.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = gw.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = fd.Close()
	if err != nil {
		log.Fatal(err)
	}
}

func zipFile(out string, files []archiveFile) {
	fd, err := os.Create(out)
	if err != nil {
		log.Fatal(err)
	}

	zw := zip.NewWriter(fd)

	for _, f := range files {
		sf, err := os.Open(f.src)
		if err != nil {
			log.Fatal(err)
		}

		info, err := sf.Stat()
		if err != nil {
			log.Fatal(err)
		}

		fh, err := zip.FileInfoHeader(info)
		if err != nil {
			log.Fatal(err)
		}
		fh.Name = f.dst
		fh.Method = zip.Deflate

		if strings.HasSuffix(f.dst, ".txt") {
			// Text file. Read it and convert line endings.
			bs, err := ioutil.ReadAll(sf)
			if err != nil {
				log.Fatal(err)
			}
			bs = bytes.Replace(bs, []byte{'\n'}, []byte{'\n', '\r'}, -1)
			fh.UncompressedSize = uint32(len(bs))
			fh.UncompressedSize64 = uint64(len(bs))

			of, err := zw.CreateHeader(fh)
			if err != nil {
				log.Fatal(err)
			}
			of.Write(bs)
		} else {
			// Binary file. Copy verbatim.
			of, err := zw.CreateHeader(fh)
			if err != nil {
				log.Fatal(err)
			}
			_, err = io.Copy(of, sf)
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	err = zw.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = fd.Close()
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
 "fmt"
 "math")

 func main() {
     var y float64 = 8
 x := math.Log2(y)
    fmt.Printf("Hello, Vebjørn\n")
    fmt.Println(x)
    fmt.Println("Jeg bare skriver det samme som dere skrev, men har ikke mye peiling på hva det betyr")
    fmt.Println("JJ is back»)
    fmt.Println("Lønner seg å sette seg ned med tutorialen :)\nEr ganske likt java\nFunksjonene er bare fra standardbiblioteket")
  fmt.Println("Håper du kan lære meg litt av dette Vebbis!")
<<<<<<< HEAD
    fmt.Println("Men da fikk vi det til da")
=======
  fmt.println("Kim was here v2")
  fmt.println("Hei,hei!")
>>>>>>> a4c2d2728cf6c87f636d5c395410d076d57e46fe

 }
<commit_msg>Endring Jørgen T<commit_after>package main

import (
 "fmt"
 "math")

 func main() {
     var y float64 = 8
 x := math.Log2(y)
    fmt.Printf("Hello, Vebjørn\n")
    fmt.Println(x)
    fmt.Println("Jeg bare skriver det samme som dere skrev, men har ikke mye peiling på hva det betyr")
    fmt.Println("JJ is back»)
    fmt.Println("Lønner seg å sette seg ned med tutorialen :)\nEr ganske likt java\nFunksjonene er bare fra standardbiblioteket")
  fmt.Println("Håper du kan lære meg litt av dette Vebbis!")
<<<<<<< HEAD
    fmt.Println("Men da fikk vi det til da")
=======
  fmt.println("Kim was here v2")
  fmt.println("Hei,hei!")
  fmt.println("Nå da?")
>>>>>>> a4c2d2728cf6c87f636d5c395410d076d57e46fe

 }
<|endoftext|>


<commit_before>package main

import (
	set "github.com/deckarep/golang-set"
	"github.com/howeyc/fsnotify"
	"github.com/tillberg/ansi-log"
	"github.com/tillberg/autorestart"
	"github.com/tillberg/bismuth"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

const moduleDirtyIdle = 0
const moduleDirtyQueued = 1
const moduleBuilding = 2
const moduleBuildingButDirty = 3
const moduleReady = 4

var srcRoot = filepath.Join(os.Getenv("GOPATH"), "src")
var goSrcRoot = filepath.Join(os.Getenv("GOROOT"), "src")
var dirtyModuleQueue = make(chan string, 3000)
var moduleState = make(map[string]int)
var moduleStateMutex sync.RWMutex

var pathDiscoveryChan = make(chan string)
var moduleTriggerChan = make(chan string)

var pathSkipSet = set.NewSetFromSlice([]interface{}{".git", ".hg", "node_modules"})
var rebuildExts = set.NewSetFromSlice([]interface{}{".go"})
var importsMap = make(map[string]set.Set)
var importsMapMutex sync.RWMutex

var watcher *fsnotify.Watcher

var builders chan *builder

type builder struct {
	ctx *bismuth.ExecContext
}

func newBuilder() *builder {
	me := &builder{}
	me.ctx = bismuth.NewExecContext()
	me.ctx.Connect()
	return me
}

func isStandardLibraryPackage(moduleName string) bool {
	stdLibCacheMutex.RLock()
	isInStdLib, ok := stdLibCache[moduleName]
	stdLibCacheMutex.RUnlock()
	if !ok {
		if moduleName == "C" {
			isInStdLib = true
		} else {
			// This is not a perfect method. Also, if GOROOT isn't defined, this will be checking some random paths
			stat, err := os.Stat(filepath.Join(goSrcRoot, moduleName))
			isInStdLib = err == nil && stat.IsDir()
		}
		stdLibCacheMutex.Lock()
		stdLibCache[moduleName] = isInStdLib
		stdLibCacheMutex.Unlock()
	}
	return isInStdLib
}

func parseModuleImports(moduleName string) set.Set {
	fileSet := token.NewFileSet()
	absPath := filepath.Join(srcRoot, moduleName)
	pkgMap, err := parser.ParseDir(fileSet, absPath, nil, parser.ImportsOnly)
	if err != nil {
		log.Printf("@(error:Error parsing import of module) %s@(error::) %s\n", moduleName, err)
		return nil
	}
	moduleDepNames := set.NewSet()
	for _, pkg := range pkgMap {
		for filename, file := range pkg.Files {
			if strings.HasSuffix(filename, "_test.go") {
				continue
			}
			for _, importObj := range file.Imports {
				moduleDepName := importObj.Path.Value
				// importObj.Path.Value is the parsed literal, including quote marks. Let's kludgily
				// remove the quotes:
				if len(moduleDepName) > 2 {
					moduleDepName = moduleDepName[1 : len(moduleDepName)-1]
					if moduleDepName != moduleName && !isStandardLibraryPackage(moduleDepName) {
						moduleDepNames.Add(moduleDepName)
					}
				}
			}
		}
	}
	return moduleDepNames
}

func getImportsForModule(moduleName string, forceRegen bool) set.Set {
	var moduleNames set.Set
	if !forceRegen {
		importsMapMutex.RLock()
		moduleNames = importsMap[moduleName]
		importsMapMutex.RUnlock()
	}
	if moduleNames == nil {
		moduleNames = parseModuleImports(moduleName)
		if moduleNames == nil {
			return nil
		}
		importsMapMutex.Lock()
		importsMap[moduleName] = moduleNames
		importsMapMutex.Unlock()
	}
	return moduleNames
}

var stdLibCache = make(map[string]bool)
var stdLibCacheMutex sync.RWMutex

func moduleHasUpToDateDependencies(moduleName string, forceRegen bool) bool {
	moduleDepNames := getImportsForModule(moduleName, forceRegen)
	if moduleDepNames == nil {
		return false
	}
	// log.Printf("module %s depends on %s\n", moduleName, moduleNames.ToSlice())
	for moduleDepName := range moduleDepNames.Iter() {
		moduleDepNameStr := moduleDepName.(string)
		moduleStateMutex.RLock()
		isReady := moduleState[moduleDepNameStr] == moduleReady
		moduleStateMutex.RUnlock()
		if !isReady {
			log.Printf("@(dim:%s is missing dep %s)\n", moduleName, moduleDepNameStr)
			return false
		}
		if !moduleHasUpToDateDependencies(moduleDepNameStr, false) {
			log.Printf("  (required by %s)\n", moduleName)
			return false
		}
	}
	return true
}

func triggerDependenciesOfModule(moduleName string) {
	triggers := []string{}
	importsMapMutex.RLock()
	for otherModuleName, moduleDepNames := range importsMap {
		// log.Println(moduleName, moduleDepNames)
		if moduleDepNames.Contains(moduleName) {
			triggers = append(triggers, otherModuleName)
		}
	}
	importsMapMutex.RUnlock()
	for _, trigger := range triggers {
		// log.Println("trigger", trigger)
		moduleTriggerChan <- trigger
	}
}

func printStateSummary() {
	moduleStateMutex.RLock()
	counts := make(map[int]int)
	for name, state := range moduleState {
		counts[state] = counts[state] + 1
		_ = name
		// if state == moduleDirtyIdle {
		// 	log.Printf("@(dim:  idle:) %s\n", name)
		// }
	}
	moduleStateMutex.RUnlock()
	log.Printf("@(dim:idle:) %d@(dim:, queued:) %d@(dim:, building:) %d@(dim:, building+dirty:) %d@(dim:, ready:) %d\n", counts[0], counts[1], counts[2], counts[3], counts[4])
}

func (b *builder) buildModule(moduleName string) {
	defer func() {
		builders <- b
	}()
	abort := func() {
		moduleStateMutex.Lock()
		currState := moduleState[moduleName]
		moduleState[moduleName] = moduleDirtyIdle
		moduleStateMutex.Unlock()
		if currState == moduleBuildingButDirty {
			moduleTriggerChan <- moduleName
		}
	}
	moduleStateMutex.Lock()
	moduleState[moduleName] = moduleBuilding
	moduleStateMutex.Unlock()
	if !moduleHasUpToDateDependencies(moduleName, true) {
		// If this module is missing any up-to-date dependecies, send
		// it to the end of the queue after a brief pause
		log.Printf("@(dim:Not building) %s @(dim:yet, dependencies not ready.)\n", moduleName)
		// go func() {
		// 	time.Sleep(1000 * time.Millisecond)
		// 	dirtyModuleQueue <- moduleName
		// }()
		abort()
		return
	}
	ctx := b.ctx
	log.Printf("@(dim:Building) %s@(dim:...)\n", moduleName)
	absPath := filepath.Join(srcRoot, moduleName)
	retCode, err := ctx.QuoteCwd("go-install", absPath, "go", "install")
	if retCode != 0 {
		log.Printf("@(error:Failed to build) %s@(error:.)\n", moduleName)
		abort()
		return
	}
	if err != nil {
		log.Printf("@(error:Failed to install %s@(error:: %s)\n", moduleName, err)
		abort()
		return
	}
	log.Printf("@(green:Successfully built) %s@(green:.)\n", moduleName)
	moduleStateMutex.Lock()
	currState := moduleState[moduleName]
	if currState == moduleBuildingButDirty {
		moduleState[moduleName] = moduleDirtyIdle
		moduleStateMutex.Unlock()
		moduleTriggerChan <- moduleName
	} else {
		moduleState[moduleName] = moduleReady
		moduleStateMutex.Unlock()
	}
	// printStateSummary()
	go triggerDependenciesOfModule(moduleName)
}

func processBuildQueue() {
	for {
		moduleName := <-dirtyModuleQueue
		b := <-builders
		go b.buildModule(moduleName)
	}
}

func processModuleTriggers() {
	neverChan := make(<-chan time.Time)
	for {
		timeoutChan := neverChan
		moduleNames := set.NewSet()
		for {
			select {
			case moduleName := <-moduleTriggerChan:
				moduleNames.Add(moduleName)
				// Debounce additional triggers for this module. This is particularly
				// important for filesystem events, as they often come in pairs.
				timeoutChan = time.After(10 * time.Millisecond)
				continue
			case <-timeoutChan:
				goto breakFor
			}
		}
	breakFor:
		for ifaceModuleName := range moduleNames.Iter() {
			moduleName := ifaceModuleName.(string)
			if strings.Contains(moduleName, "/test") {
				// Skip modules that look like test data
				continue
			}
			moduleStateMutex.Lock()
			currState := moduleState[moduleName]
			if currState == moduleBuilding {
				log.Printf("@(dim:Marking in-progress build of) %s @(dim:dirty.)\n", moduleName)
				moduleState[moduleName] = moduleBuildingButDirty
				moduleStateMutex.Unlock()
			} else if currState == moduleDirtyIdle || currState == moduleReady {
				log.Printf("@(dim:Queueing rebuild of) %s@(dim:.)\n", moduleName)
				moduleState[moduleName] = moduleDirtyQueued
				moduleStateMutex.Unlock()
				dirtyModuleQueue <- moduleName
			} else {
				moduleStateMutex.Unlock()
			}
		}
	}
}

func processPathTriggers() {
	for {
		// var verb string
		var path string
		var moduleName string
		select {
		case err := <-watcher.Error:
			log.Printf("Watcher error: %s\n", err)
			continue
		case ev := <-watcher.Event:
			path = ev.Name
			// verb = "change in"
			var err error
			moduleName, err = filepath.Rel(srcRoot, filepath.Dir(path))
			if err != nil {
				log.Bail(err)
			}
		case path = <-pathDiscoveryChan:
			// verb = "discovery of"
			moduleName = filepath.Dir(path)
		}
		if !rebuildExts.Contains(filepath.Ext(path)) {
			continue
		}
		// log.Printf("@(dim:Triggering module) %s @(dim:due to %s) %s\n", moduleName, verb, path)
		moduleTriggerChan <- moduleName
	}
}

func watchRecursive(relPath string) {
	absPath := filepath.Join(srcRoot, relPath)
	base := filepath.Base(absPath)
	if pathSkipSet.Contains(base) {
		return
	}
	stat, err := os.Stat(absPath)
	if err != nil {
		log.Printf("Error calling stat on %s: %s\n", absPath, err)
		return
	}
	if stat.IsDir() {
		watcher.Watch(absPath)
		entries, err := ioutil.ReadDir(absPath)
		if err != nil {
			log.Printf("Error reading directory %s: %s\n", absPath, err)
			return
		}
		for _, entry := range entries {
			watchRecursive(filepath.Join(relPath, entry.Name()))
		}
	} else {
		pathDiscoveryChan <- relPath
	}
}

func main() {
	log.EnableMultilineMode()
	log.EnableColorTemplate()
	log.SetFlags(0)
	log.SetPrefix("@(dim){isodate micros} ")
	log.AddAnsiColorCode("error", 31)
	log.AddAnsiColorCode("warn", 35)
	autorestart.CleanUpChildZombies()

	var err error
	watcher, err = fsnotify.NewWatcher()
	if err != nil {
		log.Bail(err)
	}
	go watchRecursive("")
	go processPathTriggers()
	go processModuleTriggers()
	go autorestart.RestartOnChange()
	numBuilders := runtime.NumCPU()
	builders = make(chan *builder, numBuilders)
	for i := 0; i < numBuilders; i++ {
		builders <- newBuilder()
	}
	processBuildQueue()
}
<commit_msg>Add watchers to newly-created directories, and build stuff in them<commit_after>package main

import (
	set "github.com/deckarep/golang-set"
	"github.com/howeyc/fsnotify"
	"github.com/tillberg/ansi-log"
	"github.com/tillberg/autorestart"
	"github.com/tillberg/bismuth"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

const moduleDirtyIdle = 0
const moduleDirtyQueued = 1
const moduleBuilding = 2
const moduleBuildingButDirty = 3
const moduleReady = 4

var srcRoot = filepath.Join(os.Getenv("GOPATH"), "src")
var goSrcRoot = filepath.Join(os.Getenv("GOROOT"), "src")
var dirtyModuleQueue = make(chan string, 3000)
var moduleState = make(map[string]int)
var moduleStateMutex sync.RWMutex

var pathDiscoveryChan = make(chan string)
var moduleTriggerChan = make(chan string)

var pathSkipSet = set.NewSetFromSlice([]interface{}{".git", ".hg", "node_modules"})
var rebuildExts = set.NewSetFromSlice([]interface{}{".go"})
var importsMap = make(map[string]set.Set)
var importsMapMutex sync.RWMutex

var watcher *fsnotify.Watcher

var builders chan *builder

type builder struct {
	ctx *bismuth.ExecContext
}

func newBuilder() *builder {
	me := &builder{}
	me.ctx = bismuth.NewExecContext()
	me.ctx.Connect()
	return me
}

func isStandardLibraryPackage(moduleName string) bool {
	stdLibCacheMutex.RLock()
	isInStdLib, ok := stdLibCache[moduleName]
	stdLibCacheMutex.RUnlock()
	if !ok {
		if moduleName == "C" {
			isInStdLib = true
		} else {
			// This is not a perfect method. Also, if GOROOT isn't defined, this will be checking some random paths
			stat, err := os.Stat(filepath.Join(goSrcRoot, moduleName))
			isInStdLib = err == nil && stat.IsDir()
		}
		stdLibCacheMutex.Lock()
		stdLibCache[moduleName] = isInStdLib
		stdLibCacheMutex.Unlock()
	}
	return isInStdLib
}

func parseModuleImports(moduleName string) set.Set {
	fileSet := token.NewFileSet()
	absPath := filepath.Join(srcRoot, moduleName)
	pkgMap, err := parser.ParseDir(fileSet, absPath, nil, parser.ImportsOnly)
	if err != nil {
		log.Printf("@(error:Error parsing import of module) %s@(error::) %s\n", moduleName, err)
		return nil
	}
	moduleDepNames := set.NewSet()
	for _, pkg := range pkgMap {
		for filename, file := range pkg.Files {
			if strings.HasSuffix(filename, "_test.go") {
				continue
			}
			for _, importObj := range file.Imports {
				moduleDepName := importObj.Path.Value
				// importObj.Path.Value is the parsed literal, including quote marks. Let's kludgily
				// remove the quotes:
				if len(moduleDepName) > 2 {
					moduleDepName = moduleDepName[1 : len(moduleDepName)-1]
					if moduleDepName != moduleName && !isStandardLibraryPackage(moduleDepName) {
						moduleDepNames.Add(moduleDepName)
					}
				}
			}
		}
	}
	return moduleDepNames
}

func getImportsForModule(moduleName string, forceRegen bool) set.Set {
	var moduleNames set.Set
	if !forceRegen {
		importsMapMutex.RLock()
		moduleNames = importsMap[moduleName]
		importsMapMutex.RUnlock()
	}
	if moduleNames == nil {
		moduleNames = parseModuleImports(moduleName)
		if moduleNames == nil {
			return nil
		}
		importsMapMutex.Lock()
		importsMap[moduleName] = moduleNames
		importsMapMutex.Unlock()
	}
	return moduleNames
}

var stdLibCache = make(map[string]bool)
var stdLibCacheMutex sync.RWMutex

func moduleHasUpToDateDependencies(moduleName string, forceRegen bool) bool {
	moduleDepNames := getImportsForModule(moduleName, forceRegen)
	if moduleDepNames == nil {
		return false
	}
	// log.Printf("module %s depends on %s\n", moduleName, moduleNames.ToSlice())
	for moduleDepName := range moduleDepNames.Iter() {
		moduleDepNameStr := moduleDepName.(string)
		moduleStateMutex.RLock()
		isReady := moduleState[moduleDepNameStr] == moduleReady
		moduleStateMutex.RUnlock()
		if !isReady {
			log.Printf("@(dim:%s is missing dep %s)\n", moduleName, moduleDepNameStr)
			return false
		}
		if !moduleHasUpToDateDependencies(moduleDepNameStr, false) {
			log.Printf("  (required by %s)\n", moduleName)
			return false
		}
	}
	return true
}

func triggerDependenciesOfModule(moduleName string) {
	triggers := []string{}
	importsMapMutex.RLock()
	for otherModuleName, moduleDepNames := range importsMap {
		// log.Println(moduleName, moduleDepNames)
		if moduleDepNames.Contains(moduleName) {
			triggers = append(triggers, otherModuleName)
		}
	}
	importsMapMutex.RUnlock()
	for _, trigger := range triggers {
		// log.Println("trigger", trigger)
		moduleTriggerChan <- trigger
	}
}

func printStateSummary() {
	moduleStateMutex.RLock()
	counts := make(map[int]int)
	for name, state := range moduleState {
		counts[state] = counts[state] + 1
		_ = name
		// if state == moduleDirtyIdle {
		// 	log.Printf("@(dim:  idle:) %s\n", name)
		// }
	}
	moduleStateMutex.RUnlock()
	log.Printf("@(dim:idle:) %d@(dim:, queued:) %d@(dim:, building:) %d@(dim:, building+dirty:) %d@(dim:, ready:) %d\n", counts[0], counts[1], counts[2], counts[3], counts[4])
}

func (b *builder) buildModule(moduleName string) {
	defer func() {
		builders <- b
	}()
	abort := func() {
		moduleStateMutex.Lock()
		currState := moduleState[moduleName]
		moduleState[moduleName] = moduleDirtyIdle
		moduleStateMutex.Unlock()
		if currState == moduleBuildingButDirty {
			moduleTriggerChan <- moduleName
		}
	}
	moduleStateMutex.Lock()
	moduleState[moduleName] = moduleBuilding
	moduleStateMutex.Unlock()
	if !moduleHasUpToDateDependencies(moduleName, true) {
		// If this module is missing any up-to-date dependecies, send
		// it to the end of the queue after a brief pause
		log.Printf("@(dim:Not building) %s @(dim:yet, dependencies not ready.)\n", moduleName)
		// go func() {
		// 	time.Sleep(1000 * time.Millisecond)
		// 	dirtyModuleQueue <- moduleName
		// }()
		abort()
		return
	}
	ctx := b.ctx
	log.Printf("@(dim:Building) %s@(dim:...)\n", moduleName)
	absPath := filepath.Join(srcRoot, moduleName)
	retCode, err := ctx.QuoteCwd("go-install", absPath, "go", "install")
	if retCode != 0 {
		log.Printf("@(error:Failed to build) %s@(error:.)\n", moduleName)
		abort()
		return
	}
	if err != nil {
		log.Printf("@(error:Failed to install %s@(error:: %s)\n", moduleName, err)
		abort()
		return
	}
	log.Printf("@(green:Successfully built) %s@(green:.)\n", moduleName)
	moduleStateMutex.Lock()
	currState := moduleState[moduleName]
	if currState == moduleBuildingButDirty {
		moduleState[moduleName] = moduleDirtyIdle
		moduleStateMutex.Unlock()
		moduleTriggerChan <- moduleName
	} else {
		moduleState[moduleName] = moduleReady
		moduleStateMutex.Unlock()
	}
	// printStateSummary()
	go triggerDependenciesOfModule(moduleName)
}

func processBuildQueue() {
	for {
		moduleName := <-dirtyModuleQueue
		b := <-builders
		go b.buildModule(moduleName)
	}
}

func processModuleTriggers() {
	neverChan := make(<-chan time.Time)
	for {
		timeoutChan := neverChan
		moduleNames := set.NewSet()
		for {
			select {
			case moduleName := <-moduleTriggerChan:
				moduleNames.Add(moduleName)
				// Debounce additional triggers for this module. This is particularly
				// important for filesystem events, as they often come in pairs.
				timeoutChan = time.After(10 * time.Millisecond)
				continue
			case <-timeoutChan:
				goto breakFor
			}
		}
	breakFor:
		for ifaceModuleName := range moduleNames.Iter() {
			moduleName := ifaceModuleName.(string)
			if strings.Contains(moduleName, "/test") {
				// Skip modules that look like test data
				continue
			}
			moduleStateMutex.Lock()
			currState := moduleState[moduleName]
			if currState == moduleBuilding {
				log.Printf("@(dim:Marking in-progress build of) %s @(dim:dirty.)\n", moduleName)
				moduleState[moduleName] = moduleBuildingButDirty
				moduleStateMutex.Unlock()
			} else if currState == moduleDirtyIdle || currState == moduleReady {
				log.Printf("@(dim:Queueing rebuild of) %s@(dim:.)\n", moduleName)
				moduleState[moduleName] = moduleDirtyQueued
				moduleStateMutex.Unlock()
				dirtyModuleQueue <- moduleName
			} else {
				moduleStateMutex.Unlock()
			}
		}
	}
}

func processPathTriggers() {
	for {
		// var verb string
		var path string
		var moduleName string
		select {
		case err := <-watcher.Error:
			log.Printf("Watcher error: %s\n", err)
			continue
		case ev := <-watcher.Event:
			path = ev.Name
			// verb = "change in"
			var err error
			moduleName, err = filepath.Rel(srcRoot, filepath.Dir(path))
			if err != nil {
				log.Bail(err)
			}
			if ev.IsCreate() {
				stat, err := os.Stat(path)
				if err == nil && stat.IsDir() {
					relPath, err := filepath.Rel(srcRoot, path)
					if err != nil {
						log.Printf("@(error:Error resolving relative path from %s to %s: %s)\n", srcRoot, path, err)
					} else {
						log.Printf("@(dim:Adding watcher for new directory) %s@(dim:.)\n", relPath)
						watchRecursive(relPath)
					}
				}
			}
		case path = <-pathDiscoveryChan:
			// verb = "discovery of"
			moduleName = filepath.Dir(path)
		}
		if !rebuildExts.Contains(filepath.Ext(path)) {
			continue
		}
		// log.Printf("@(dim:Triggering module) %s @(dim:due to %s) %s\n", moduleName, verb, path)
		moduleTriggerChan <- moduleName
	}
}

func watchRecursive(relPath string) {
	absPath := filepath.Join(srcRoot, relPath)
	base := filepath.Base(absPath)
	if pathSkipSet.Contains(base) {
		return
	}
	stat, err := os.Stat(absPath)
	if err != nil {
		log.Printf("Error calling stat on %s: %s\n", absPath, err)
		return
	}
	if stat.IsDir() {
		watcher.Watch(absPath)
		entries, err := ioutil.ReadDir(absPath)
		if err != nil {
			log.Printf("Error reading directory %s: %s\n", absPath, err)
			return
		}
		for _, entry := range entries {
			watchRecursive(filepath.Join(relPath, entry.Name()))
		}
	} else {
		pathDiscoveryChan <- relPath
	}
}

func main() {
	log.EnableMultilineMode()
	log.EnableColorTemplate()
	log.SetFlags(0)
	log.SetPrefix("@(dim){isodate micros} ")
	log.AddAnsiColorCode("error", 31)
	log.AddAnsiColorCode("warn", 35)
	autorestart.CleanUpChildZombies()

	var err error
	watcher, err = fsnotify.NewWatcher()
	if err != nil {
		log.Bail(err)
	}
	go watchRecursive("")
	go processPathTriggers()
	go processModuleTriggers()
	go autorestart.RestartOnChange()
	numBuilders := runtime.NumCPU()
	builders = make(chan *builder, numBuilders)
	for i := 0; i < numBuilders; i++ {
		builders <- newBuilder()
	}
	processBuildQueue()
}
<|endoftext|>


<commit_before>package engines

import (
	"encoding/json"
	"fmt"
	"net/url"
	"os"
	"strconv"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/camptocamp/bivac/metrics"
	"github.com/camptocamp/bivac/orchestrators"
	"github.com/camptocamp/bivac/util"
	"github.com/camptocamp/bivac/volume"
)

// ResticEngine implements a backup engine with Restic
type ResticEngine struct {
	Orchestrator orchestrators.Orchestrator
	Volume       *volume.Volume
}

// Snapshot is a struct returned by the function snapshots()
type Snapshot struct {
	Time     time.Time `json:"time"`
	Parent   string    `json:"parent"`
	Tree     string    `json:"tree"`
	Path     []string  `json:"path"`
	Hostname string    `json:"hostname"`
	ID       string    `json:"id"`
	ShortID  string    `json:"short_id"`
}

// GetName returns the engine name
func (*ResticEngine) GetName() string {
	return "Restic"
}

// Backup performs the backup of the passed volume
func (r *ResticEngine) Backup() (err error) {

	v := r.Volume

	targetURL, err := url.Parse(v.Config.TargetURL)
	if err != nil {
		err = fmt.Errorf("failed to parse target URL: %v", err)
		return
	}

	v.Target = targetURL.String() + "/" + v.Name
	v.BackupDir = v.Mountpoint + "/" + v.BackupDir
	v.Mount = v.Name + ":" + v.Mountpoint + ":ro"

	err = util.Retry(3, r.init)
	if err != nil {
		err = fmt.Errorf("failed to create a secure repository: %v", err)
		return
	}

	err = util.Retry(3, r.resticBackup)
	if err != nil {
		err = fmt.Errorf("failed to backup the volume: %v", err)
		return
	}

	err = util.Retry(3, r.forget)
	if err != nil {
		err = fmt.Errorf("failed to forget the oldest snapshots: %v", err)
		return
	}

	c := r.Orchestrator.GetHandler()
	if _, err := c.IsCheckScheduled(v); err == nil {
		err = util.Retry(3, r.verify)
		if err != nil {
			err = fmt.Errorf("failed to verify backup: %v", err)
			return err
		}
	}

	return
}

// init initialize a secure repository
func (r *ResticEngine) init() (err error) {
	v := r.Volume

	// Check if the repository already exists
	state, _, err := r.launchRestic(
		[]string{
			"-r",
			v.Target,
			"snapshots",
		},
		[]*volume.Volume{},
	)
	if err != nil {
		err = fmt.Errorf("failed to launch Restic to verify the existence of the repository: %v", err)
		return
	}
	if state == 0 {
		log.WithFields(log.Fields{
			"volume": v.Name,
		}).Info("The repository already exists, skipping initialization.")
		return nil
	}

	// Initialize the repository
	state, _, err = r.launchRestic(
		[]string{
			"-r",
			v.Target,
			"init",
		},
		[]*volume.Volume{
			v,
		},
	)
	if err != nil {
		err = fmt.Errorf("failed to launch Restic to initialize the repository: %v", err)
		return
	}
	if state != 0 {
		err = fmt.Errorf("Restic exited with state %v while initializing the repository", state)
		return
	}
	return
}

// resticBackup performs the backup of a volume with Restic
func (r *ResticEngine) resticBackup() (err error) {
	c := r.Orchestrator.GetHandler()
	v := r.Volume
	state, _, err := r.launchRestic(
		[]string{
			"--hostname",
			c.Hostname,
			"-r",
			v.Target,
			"backup",
			v.BackupDir,
		},
		[]*volume.Volume{
			v,
		},
	)
	if err != nil {
		err = fmt.Errorf("failed to launch Restic to backup the volume: %v", err)
	}
	if state != 0 {
		err = fmt.Errorf("Restic exited with state %v while backuping the volume", state)
	}
	return
}

// verify checks that the backup is usable
func (r *ResticEngine) verify() (err error) {
	v := r.Volume
	state, _, err := r.launchRestic(
		[]string{
			"-r",
			v.Target,
			"check",
		},
		[]*volume.Volume{},
	)
	if err != nil {
		err = fmt.Errorf("failed to launch Restic to check the backup: %v", err)
		return
	}
	if state == 0 {
		now := time.Now().Local()
		os.Chtimes(v.Mountpoint+"/.bivac_last_check", now, now)
	} else {
		err = fmt.Errorf("Restic exited with state %v while checking the backup", state)
	}

	metric := r.Volume.MetricsHandler.NewMetric("bivac_verifyExitCode", "gauge")
	err = metric.UpdateEvent(
		&metrics.Event{
			Labels: map[string]string{
				"volume": v.Name,
			},
			Value: strconv.Itoa(state),
		},
	)
	return
}

// forget removes a snapshot
func (r *ResticEngine) forget() (err error) {

	v := r.Volume

	snapshots, err := r.snapshots()

	duration, err := util.GetDurationFromInterval(v.Config.RemoveOlderThan)
	if err != nil {
		return err
	}

	validSnapshots := 0
	now := time.Now()
	for _, snapshot := range snapshots {
		expiration := snapshot.Time.Add(duration)
		if now.Before(expiration) {
			validSnapshots++
		}
	}

	state, output, err := r.launchRestic(
		[]string{
			"-r",
			v.Target,
			"forget",
			"--prune",
			"--keep-last",
			fmt.Sprintf("%d", validSnapshots),
		},
		[]*volume.Volume{},
	)
	if err != nil {
		err = fmt.Errorf("failed to launch Restic to forget the snapshot: %v", err)
		return err
	}

	if state != 0 {
		err = fmt.Errorf("restic failed to forget old snapshots: %v", output)
		return err
	}
	return
}

// snapshots lists snapshots
func (r *ResticEngine) snapshots() (snapshots []Snapshot, err error) {
	v := r.Volume

	_, output, err := r.launchRestic(
		[]string{
			"-r",
			v.Target,
			"snapshots",
			"--json",
		},
		[]*volume.Volume{},
	)
	if err != nil {
		err = fmt.Errorf("failed to launch Restic to check the backup: %v", err)
		return
	}

	if err := json.Unmarshal([]byte(output), &snapshots); err != nil {
		err = fmt.Errorf("failed to parse JSON output: %v", err)
		return snapshots, err
	}
	return
}

// launchRestic starts a restic container with the given command
func (r *ResticEngine) launchRestic(cmd []string, volumes []*volume.Volume) (state int, stdout string, err error) {
	config := r.Orchestrator.GetHandler().Config
	image := config.Restic.Image

	// Disable cache to avoid volume issues with Kubernetes
	cmd = append(cmd, "--no-cache")

	env := map[string]string{
		"AWS_ACCESS_KEY_ID":     config.AWS.AccessKeyID,
		"AWS_SECRET_ACCESS_KEY": config.AWS.SecretAccessKey,
		"OS_USERNAME":           config.Swift.Username,
		"OS_PASSWORD":           config.Swift.Password,
		"OS_AUTH_URL":           config.Swift.AuthURL,
		"OS_TENANT_NAME":        config.Swift.TenantName,
		"OS_REGION_NAME":        config.Swift.RegionName,
		"RESTIC_PASSWORD":       config.Restic.Password,
	}

	return r.Orchestrator.LaunchContainer(image, env, cmd, volumes)
}
<commit_msg>Use hostname in Restic path<commit_after>package engines

import (
	"encoding/json"
	"fmt"
	"net/url"
	"os"
	"strconv"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/camptocamp/bivac/metrics"
	"github.com/camptocamp/bivac/orchestrators"
	"github.com/camptocamp/bivac/util"
	"github.com/camptocamp/bivac/volume"
)

// ResticEngine implements a backup engine with Restic
type ResticEngine struct {
	Orchestrator orchestrators.Orchestrator
	Volume       *volume.Volume
}

// Snapshot is a struct returned by the function snapshots()
type Snapshot struct {
	Time     time.Time `json:"time"`
	Parent   string    `json:"parent"`
	Tree     string    `json:"tree"`
	Path     []string  `json:"path"`
	Hostname string    `json:"hostname"`
	ID       string    `json:"id"`
	ShortID  string    `json:"short_id"`
}

// GetName returns the engine name
func (*ResticEngine) GetName() string {
	return "Restic"
}

// Backup performs the backup of the passed volume
func (r *ResticEngine) Backup() (err error) {

	v := r.Volume

	targetURL, err := url.Parse(v.Config.TargetURL)
	if err != nil {
		err = fmt.Errorf("failed to parse target URL: %v", err)
		return
	}

	c := r.Orchestrator.GetHandler()
	v.Target = targetURL.String() + "/" + c.Hostname + "/" + v.Name
	v.BackupDir = v.Mountpoint + "/" + v.BackupDir
	v.Mount = v.Name + ":" + v.Mountpoint + ":ro"

	err = util.Retry(3, r.init)
	if err != nil {
		err = fmt.Errorf("failed to create a secure repository: %v", err)
		return
	}

	err = util.Retry(3, r.resticBackup)
	if err != nil {
		err = fmt.Errorf("failed to backup the volume: %v", err)
		return
	}

	err = util.Retry(3, r.forget)
	if err != nil {
		err = fmt.Errorf("failed to forget the oldest snapshots: %v", err)
		return
	}

	if _, err := c.IsCheckScheduled(v); err == nil {
		err = util.Retry(3, r.verify)
		if err != nil {
			err = fmt.Errorf("failed to verify backup: %v", err)
			return err
		}
	}

	return
}

// init initialize a secure repository
func (r *ResticEngine) init() (err error) {
	v := r.Volume

	// Check if the repository already exists
	state, _, err := r.launchRestic(
		[]string{
			"-r",
			v.Target,
			"snapshots",
		},
		[]*volume.Volume{},
	)
	if err != nil {
		err = fmt.Errorf("failed to launch Restic to verify the existence of the repository: %v", err)
		return
	}
	if state == 0 {
		log.WithFields(log.Fields{
			"volume": v.Name,
		}).Info("The repository already exists, skipping initialization.")
		return nil
	}

	// Initialize the repository
	state, _, err = r.launchRestic(
		[]string{
			"-r",
			v.Target,
			"init",
		},
		[]*volume.Volume{
			v,
		},
	)
	if err != nil {
		err = fmt.Errorf("failed to launch Restic to initialize the repository: %v", err)
		return
	}
	if state != 0 {
		err = fmt.Errorf("Restic exited with state %v while initializing the repository", state)
		return
	}
	return
}

// resticBackup performs the backup of a volume with Restic
func (r *ResticEngine) resticBackup() (err error) {
	c := r.Orchestrator.GetHandler()
	v := r.Volume
	state, _, err := r.launchRestic(
		[]string{
			"--hostname",
			c.Hostname,
			"-r",
			v.Target,
			"backup",
			v.BackupDir,
		},
		[]*volume.Volume{
			v,
		},
	)
	if err != nil {
		err = fmt.Errorf("failed to launch Restic to backup the volume: %v", err)
	}
	if state != 0 {
		err = fmt.Errorf("Restic exited with state %v while backuping the volume", state)
	}
	return
}

// verify checks that the backup is usable
func (r *ResticEngine) verify() (err error) {
	v := r.Volume
	state, _, err := r.launchRestic(
		[]string{
			"-r",
			v.Target,
			"check",
		},
		[]*volume.Volume{},
	)
	if err != nil {
		err = fmt.Errorf("failed to launch Restic to check the backup: %v", err)
		return
	}
	if state == 0 {
		now := time.Now().Local()
		os.Chtimes(v.Mountpoint+"/.bivac_last_check", now, now)
	} else {
		err = fmt.Errorf("Restic exited with state %v while checking the backup", state)
	}

	metric := r.Volume.MetricsHandler.NewMetric("bivac_verifyExitCode", "gauge")
	err = metric.UpdateEvent(
		&metrics.Event{
			Labels: map[string]string{
				"volume": v.Name,
			},
			Value: strconv.Itoa(state),
		},
	)
	return
}

// forget removes a snapshot
func (r *ResticEngine) forget() (err error) {

	v := r.Volume

	snapshots, err := r.snapshots()

	duration, err := util.GetDurationFromInterval(v.Config.RemoveOlderThan)
	if err != nil {
		return err
	}

	validSnapshots := 0
	now := time.Now()
	for _, snapshot := range snapshots {
		expiration := snapshot.Time.Add(duration)
		if now.Before(expiration) {
			validSnapshots++
		}
	}

	state, output, err := r.launchRestic(
		[]string{
			"-r",
			v.Target,
			"forget",
			"--prune",
			"--keep-last",
			fmt.Sprintf("%d", validSnapshots),
		},
		[]*volume.Volume{},
	)
	if err != nil {
		err = fmt.Errorf("failed to launch Restic to forget the snapshot: %v", err)
		return err
	}

	if state != 0 {
		err = fmt.Errorf("restic failed to forget old snapshots: %v", output)
		return err
	}
	return
}

// snapshots lists snapshots
func (r *ResticEngine) snapshots() (snapshots []Snapshot, err error) {
	v := r.Volume

	_, output, err := r.launchRestic(
		[]string{
			"-r",
			v.Target,
			"snapshots",
			"--json",
		},
		[]*volume.Volume{},
	)
	if err != nil {
		err = fmt.Errorf("failed to launch Restic to check the backup: %v", err)
		return
	}

	if err := json.Unmarshal([]byte(output), &snapshots); err != nil {
		err = fmt.Errorf("failed to parse JSON output: %v", err)
		return snapshots, err
	}
	return
}

// launchRestic starts a restic container with the given command
func (r *ResticEngine) launchRestic(cmd []string, volumes []*volume.Volume) (state int, stdout string, err error) {
	config := r.Orchestrator.GetHandler().Config
	image := config.Restic.Image

	// Disable cache to avoid volume issues with Kubernetes
	cmd = append(cmd, "--no-cache")

	env := map[string]string{
		"AWS_ACCESS_KEY_ID":     config.AWS.AccessKeyID,
		"AWS_SECRET_ACCESS_KEY": config.AWS.SecretAccessKey,
		"OS_USERNAME":           config.Swift.Username,
		"OS_PASSWORD":           config.Swift.Password,
		"OS_AUTH_URL":           config.Swift.AuthURL,
		"OS_TENANT_NAME":        config.Swift.TenantName,
		"OS_REGION_NAME":        config.Swift.RegionName,
		"RESTIC_PASSWORD":       config.Restic.Password,
	}

	return r.Orchestrator.LaunchContainer(image, env, cmd, volumes)
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package mapper

import (
	"net"
	"strconv"
	"strings"

	"github.com/ernestio/definition-mapper/libmapper/providers/azure/components"
	"github.com/ernestio/definition-mapper/libmapper/providers/azure/definition"
	"github.com/ernestio/ernestprovider/providers/azure/networkinterface"
	graph "gopkg.in/r3labs/graph.v2"
)

// MapNetworkInterfaces ...
func MapNetworkInterfaces(d *definition.Definition) (interfaces []*components.NetworkInterface) {
	for _, rg := range d.ResourceGroups {
		for _, vm := range rg.VirtualMachines {
			for _, ni := range vm.NetworkInterfaces {
				var addresses []net.IP

				for _, config := range ni.IPConfigurations {
					addresses = append(addresses, net.ParseIP(config.PrivateIPAddress).To4())
				}

				for i := 1; i < vm.Count+1; i++ {
					cv := &components.NetworkInterface{}
					cv.Name = ni.Name + "-" + strconv.Itoa(i)
					cv.NetworkSecurityGroup = ni.SecurityGroup
					cv.DNSServers = ni.DNSServers
					cv.InternalDNSNameLabel = ni.InternalDNSNameLabel
					cv.ResourceGroupName = rg.Name
					cv.VirtualMachineID = components.TYPEVIRTUALMACHINE + components.TYPEDELIMITER + vm.Name
					cv.Location = rg.Location
					cv.Tags = mapTags(ni.Name, d.Name)

					for x, ip := range ni.IPConfigurations {
						subnet := strings.Split(ip.Subnet, ":")[1]

						nIP := networkinterface.IPConfiguration{
							Name:                       ip.Name,
							Subnet:                     subnet,
							PrivateIPAddress:           addresses[x].String(),
							PrivateIPAddressAllocation: ip.PrivateIPAddressAllocation,
							PublicIPAddress:            ip.PublicIPAddressID,
						}
						if nIP.PrivateIPAddressAllocation == "" {
							nIP.PrivateIPAddressAllocation = "static"
						}
						cv.IPConfigurations = append(cv.IPConfigurations, nIP)

						addresses[x][3]++
					}

					if ni.ID != "" {
						cv.SetAction("none")
					}

					cv.SetDefaultVariables()

					interfaces = append(interfaces, cv)
				}
			}
		}
	}

	return
}

// MapDefinitionNetworkInterfaces : ...
func MapDefinitionNetworkInterfaces(g *graph.Graph, vm *definition.VirtualMachine) (nis []definition.NetworkInterface) {
	for _, c := range g.GetComponents().ByType("network_interface") {
		ni := c.(*components.NetworkInterface)

		if ni.VirtualMachineID != components.TYPEVIRTUALMACHINE+components.TYPEDELIMITER+vm.Name {
			continue
		}

		nNi := definition.NetworkInterface{
			ID:                   ni.GetProviderID(),
			Name:                 ni.Name,
			SecurityGroup:        ni.NetworkSecurityGroup,
			DNSServers:           ni.DNSServers,
			InternalDNSNameLabel: ni.InternalDNSNameLabel,
		}

		for _, ip := range ni.IPConfigurations {
			nIP := definition.IPConfiguration{
				Name:                       ip.Name,
				Subnet:                     ip.Subnet,
				PrivateIPAddress:           ip.PrivateIPAddress,
				PrivateIPAddressAllocation: ip.PrivateIPAddressAllocation,
				PublicIPAddressID:          ip.PublicIPAddress,
			}
			nNi.IPConfigurations = append(nNi.IPConfigurations, nIP)
		}

		nis = append(nis, nNi)
	}

	return
}
<commit_msg>fixing panic<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package mapper

import (
	"fmt"
	"net"
	"strconv"
	"strings"

	"github.com/ernestio/definition-mapper/libmapper/providers/azure/components"
	"github.com/ernestio/definition-mapper/libmapper/providers/azure/definition"
	"github.com/ernestio/ernestprovider/providers/azure/networkinterface"
	graph "gopkg.in/r3labs/graph.v2"
)

// MapNetworkInterfaces ...
func MapNetworkInterfaces(d *definition.Definition) (interfaces []*components.NetworkInterface) {
	for _, rg := range d.ResourceGroups {
		for _, vm := range rg.VirtualMachines {
			for _, ni := range vm.NetworkInterfaces {
				var addresses []net.IP

				for _, config := range ni.IPConfigurations {
					fmt.Println(config)
					addresses = append(addresses, net.ParseIP(config.PrivateIPAddress).To4())
				}

				fmt.Println(addresses)

				for i := 1; i < vm.Count+1; i++ {
					cv := &components.NetworkInterface{}
					cv.Name = ni.Name + "-" + strconv.Itoa(i)
					cv.NetworkSecurityGroup = ni.SecurityGroup
					cv.DNSServers = ni.DNSServers
					cv.InternalDNSNameLabel = ni.InternalDNSNameLabel
					cv.ResourceGroupName = rg.Name
					cv.VirtualMachineID = components.TYPEVIRTUALMACHINE + components.TYPEDELIMITER + vm.Name
					cv.Location = rg.Location
					cv.Tags = mapTags(ni.Name, d.Name)

					for x, ip := range ni.IPConfigurations {
						subnet := strings.Split(ip.Subnet, ":")[1]

						nIP := networkinterface.IPConfiguration{
							Name:   ip.Name,
							Subnet: subnet,

							PrivateIPAddressAllocation: ip.PrivateIPAddressAllocation,
							PublicIPAddress:            ip.PublicIPAddressID,
						}
						if nIP.PrivateIPAddressAllocation == "" {
							nIP.PrivateIPAddressAllocation = "static"
						}

						fmt.Println(addresses)
						fmt.Println(len(addresses))

						if nIP.PrivateIPAddressAllocation == "static" {
							nIP.PrivateIPAddress = addresses[x].String()
							addresses[x][3]++
						}
						cv.IPConfigurations = append(cv.IPConfigurations, nIP)
					}

					if ni.ID != "" {
						cv.SetAction("none")
					}

					cv.SetDefaultVariables()

					interfaces = append(interfaces, cv)
				}
			}
		}
	}

	return
}

// MapDefinitionNetworkInterfaces : ...
func MapDefinitionNetworkInterfaces(g *graph.Graph, vm *definition.VirtualMachine) (nis []definition.NetworkInterface) {
	for _, c := range g.GetComponents().ByType("network_interface") {
		ni := c.(*components.NetworkInterface)

		if ni.VirtualMachineID != components.TYPEVIRTUALMACHINE+components.TYPEDELIMITER+vm.Name {
			continue
		}

		nNi := definition.NetworkInterface{
			ID:                   ni.GetProviderID(),
			Name:                 ni.Name,
			SecurityGroup:        ni.NetworkSecurityGroup,
			DNSServers:           ni.DNSServers,
			InternalDNSNameLabel: ni.InternalDNSNameLabel,
		}

		for _, ip := range ni.IPConfigurations {
			nIP := definition.IPConfiguration{
				Name:                       ip.Name,
				Subnet:                     ip.Subnet,
				PrivateIPAddress:           ip.PrivateIPAddress,
				PrivateIPAddressAllocation: ip.PrivateIPAddressAllocation,
				PublicIPAddressID:          ip.PublicIPAddress,
			}
			nNi.IPConfigurations = append(nNi.IPConfigurations, nIP)
		}

		nis = append(nis, nNi)
	}

	return
}
<|endoftext|>