<commit_before>// Package colly implements a HTTP scraping framework
package colly

import (
	"bytes"
	"errors"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/html"

	"github.com/PuerkitoBio/goquery"
)

// Collector provides the scraper instance for a scraping job
type Collector struct {
	// UserAgent is the User-Agent string used by HTTP requests
	UserAgent string
	// MaxDepth limits the recursion depth of visited URLs.
	// Set it to 0 for infinite recursion (default).
	MaxDepth int
	// AllowedDomains is a domain whitelist.
	// Leave it blank to allow any domains to be visited
	AllowedDomains []string
	// AllowURLRevisit allows multiple downloads of the same URL
	AllowURLRevisit bool
	// MaxBodySize is the limit of the retrieved response body in bytes.
	// `0` means unlimited.
	// The default value for MaxBodySize is 10MB (10 * 1024 * 1024 bytes).
	MaxBodySize int
	// CacheDir specifies a location where GET requests are cached as files.
	// When it's not defined, caching is disabled.
	CacheDir          string
	visitedURLs       []string
	htmlCallbacks     map[string]HTMLCallback
	requestCallbacks  []RequestCallback
	responseCallbacks []ResponseCallback
	backend           *httpBackend
	wg                *sync.WaitGroup
	lock              *sync.Mutex
}

// Request is the representation of a HTTP request made by a Collector
type Request struct {
	// URL is the parsed URL of the HTTP request
	URL *url.URL
	// Headers contains the Request's HTTP headers
	Headers *http.Header
	// Ctx is a context between a Request and a Response
	Ctx *Context
	// Depth is the number of the parents of this request
	Depth     int
	collector *Collector
}

// Response is the representation of a HTTP response made by a Collector
type Response struct {
	// StatusCode is the status code of the Response
	StatusCode int
	// Body is the content of the Response
	Body []byte
	// Ctx is a context between a Request and a Response
	Ctx *Context
	// Request is the Request object of the response
	Request *Request
	// Headers contains the Response's HTTP headers
	Headers *http.Header
}

// HTMLElement is the representation of a HTML tag.
type HTMLElement struct {
	// Name is the name of the tag
	Name       string
	Text       string
	attributes []html.Attribute
	// Request is the request object of the element's HTML document
	Request *Request
	// Response is the Response object of the element's HTML document
	Response *Response
	// DOM is the goquery parsed DOM object of the page. DOM is relative
	// to the current HTMLElement
	DOM *goquery.Selection
}

// Context provides a tiny layer for passing data between callbacks
type Context struct {
	ContextMap map[string]string
	lock       *sync.Mutex
}

// RequestCallback is a type alias for OnRequest callback functions
type RequestCallback func(*Request)

// ResponseCallback is a type alias for OnResponse callback functions
type ResponseCallback func(*Response)

// HTMLCallback is a type alias for OnHTML callback functions
type HTMLCallback func(*HTMLElement)

// NewCollector creates a new Collector instance with default configuration
func NewCollector() *Collector {
	c := &Collector{}
	c.Init()
	return c
}

// NewContext initializes a new Context instance
func NewContext() *Context {
	return &Context{
		ContextMap: make(map[string]string),
		lock:       &sync.Mutex{},
	}
}

// Init initializes the Collector's private variables and sets default
// configuration for the Collector
func (c *Collector) Init() {
	c.UserAgent = "colly - https://github.com/asciimoo/colly"
	c.MaxDepth = 0
	c.visitedURLs = make([]string, 0, 8)
	c.htmlCallbacks = make(map[string]HTMLCallback, 0)
	c.requestCallbacks = make([]RequestCallback, 0, 8)
	c.responseCallbacks = make([]ResponseCallback, 0, 8)
	c.MaxBodySize = 10 * 1024 * 1024
	c.backend = &httpBackend{}
	c.backend.Init()
	c.wg = &sync.WaitGroup{}
	c.lock = &sync.Mutex{}
}

// Visit starts Collector's collecting job by creating a
// request to the URL specified in parameter.
// Visit also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (c *Collector) Visit(URL string) error {
	return c.scrape(URL, "GET", 1, nil)
}

// Post starts collecting job by creating a POST
// request.
// Post also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (c *Collector) Post(URL string, requestData map[string]string) error {
	return c.scrape(URL, "POST", 1, requestData)
}

func (c *Collector) scrape(u, method string, depth int, requestData map[string]string) error {
	c.wg.Add(1)
	defer c.wg.Done()
	if u == "" {
		return errors.New("Missing URL")
	}
	if c.MaxDepth > 0 && c.MaxDepth < depth {
		return errors.New("Max depth limit reached")
	}
	if !c.AllowURLRevisit {
		visited := false
		for _, u2 := range c.visitedURLs {
			if u2 == u {
				visited = true
				break
			}
		}
		if visited {
			return errors.New("URL already visited")
		}
	}
	parsedURL, err := url.Parse(u)
	if err != nil {
		return err
	}
	allowed := false
	if c.AllowedDomains == nil || len(c.AllowedDomains) == 0 {
		allowed = true
	} else {
		for _, d := range c.AllowedDomains {
			if d == parsedURL.Host {
				allowed = true
				break
			}
		}
	}
	if !allowed {
		return errors.New("Forbidden domain")
	}
	if !c.AllowURLRevisit {
		c.lock.Lock()
		c.visitedURLs = append(c.visitedURLs, u)
		c.lock.Unlock()
	}
	var form url.Values
	if method == "POST" {
		form = url.Values{}
		for k, v := range requestData {
			form.Add(k, v)
		}
	}
	req, err := http.NewRequest(method, u, strings.NewReader(form.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", c.UserAgent)
	if method == "POST" {
		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	}
	ctx := NewContext()
	request := &Request{
		URL:       parsedURL,
		Headers:   &req.Header,
		Ctx:       ctx,
		Depth:     depth,
		collector: c,
	}
	if len(c.requestCallbacks) > 0 {
		c.handleOnRequest(request)
	}
	response, err := c.backend.Cache(req, c.MaxBodySize, c.CacheDir)
	// TODO add OnError callback to handle these cases
	if err != nil {
		return err
	}
	response.Ctx = ctx
	response.Request = request
	if strings.Index(strings.ToLower(response.Headers.Get("Content-Type")), "html") > -1 {
		c.handleOnHTML(response.Body, request, response)
	}
	if len(c.responseCallbacks) > 0 {
		c.handleOnResponse(response)
	}
	return nil
}

// Wait returns when the collector jobs are finished
func (c *Collector) Wait() {
	c.wg.Wait()
}

// OnRequest registers a function. Function will be executed on every
// request made by the Collector
func (c *Collector) OnRequest(f RequestCallback) {
	c.lock.Lock()
	c.requestCallbacks = append(c.requestCallbacks, f)
	c.lock.Unlock()
}

// OnResponse registers a function. Function will be executed on every response
func (c *Collector) OnResponse(f ResponseCallback) {
	c.lock.Lock()
	c.responseCallbacks = append(c.responseCallbacks, f)
	c.lock.Unlock()
}

// OnHTML registers a function. Function will be executed on every HTML
// element matched by the `goquerySelector` parameter.
// `goquerySelector` is a selector used by https://github.com/PuerkitoBio/goquery
func (c *Collector) OnHTML(goquerySelector string, f HTMLCallback) {
	c.lock.Lock()
	c.htmlCallbacks[goquerySelector] = f
	c.lock.Unlock()
}

// WithTransport allows you to set a custom http.Transport for this collector.
func (c *Collector) WithTransport(transport *http.Transport) {
	c.backend.Client.Transport = transport
}

// DisableCookies turns off cookie handling for this collector
func (c *Collector) DisableCookies() {
	c.backend.Client.Jar = nil
}

// SetRequestTimeout overrides the default timeout (10 seconds) for this collector
func (c *Collector) SetRequestTimeout(timeout time.Duration) {
	c.backend.Client.Timeout = timeout
}

func (c *Collector) handleOnRequest(r *Request) {
	for _, f := range c.requestCallbacks {
		f(r)
	}
}

func (c *Collector) handleOnResponse(r *Response) {
	for _, f := range c.responseCallbacks {
		f(r)
	}
}

func (c *Collector) handleOnHTML(body []byte, req *Request, resp *Response) {
	doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(body))
	if err != nil {
		return
	}
	for expr, f := range c.htmlCallbacks {
		doc.Find(expr).Each(func(i int, s *goquery.Selection) {
			for _, n := range s.Nodes {
				e := &HTMLElement{
					Name:       n.Data,
					Request:    req,
					Response:   resp,
					Text:       goquery.NewDocumentFromNode(n).Text(),
					DOM:        s,
					attributes: n.Attr,
				}
				f(e)
			}
		})
	}
}

// Limit adds a new `LimitRule` to the collector
func (c *Collector) Limit(rule *LimitRule) error {
	return c.backend.Limit(rule)
}

// Limits adds new `LimitRule`s to the collector
func (c *Collector) Limits(rules []*LimitRule) error {
	return c.backend.Limits(rules)
}

// Attr returns the selected attribute of a HTMLElement or empty string
// if no attribute found
func (h *HTMLElement) Attr(k string) string {
	for _, a := range h.attributes {
		if a.Key == k {
			return a.Val
		}
	}
	return ""
}

// AbsoluteURL returns with the resolved absolute URL of an URL chunk.
// AbsoluteURL returns empty string if the URL chunk is a fragment or
// could not be parsed
func (r *Request) AbsoluteURL(u string) string {
	if strings.HasPrefix(u, "#") {
		return ""
	}
	absURL, err := r.URL.Parse(u)
	if err != nil {
		return ""
	}
	absURL.Fragment = ""
	if absURL.Scheme == "//" {
		absURL.Scheme = r.URL.Scheme
	}
	return absURL.String()
}

// Visit continues Collector's collecting job by creating a
// request to the URL specified in parameter.
// Visit also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (r *Request) Visit(URL string) error {
	return r.collector.scrape(r.AbsoluteURL(URL), "GET", r.Depth+1, nil)
}

// Post continues a collector job by creating a POST request.
// Post also calls the previously provided OnRequest, OnResponse, OnHTML callbacks
func (r *Request) Post(URL string, requestData map[string]string) error {
	return r.collector.scrape(r.AbsoluteURL(URL), "POST", r.Depth+1, requestData)
}

// Put stores a value in Context
func (c *Context) Put(key, value string) {
	c.lock.Lock()
	c.ContextMap[key] = value
	c.lock.Unlock()
}

// Get retrieves a value from Context. If no value found for `k`
// Get returns an empty string if key not found
func (c *Context) Get(key string) string {
	if v, ok := c.ContextMap[key]; ok {
		return v
	}
	return ""
}
<commit_msg>avoid serializing context<commit_after>// Package colly implements a HTTP scraping framework
package colly

import (
	"bytes"
	"errors"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/html"

	"github.com/PuerkitoBio/goquery"
)

// Collector provides the scraper instance for a scraping job
type Collector struct {
	// UserAgent is the User-Agent string used by HTTP requests
	UserAgent string
	// MaxDepth limits the recursion depth of visited URLs.
	// Set it to 0 for infinite recursion (default).
	MaxDepth int
	// AllowedDomains is a domain whitelist.
	// Leave it blank to allow any domains to be visited
	AllowedDomains []string
	// AllowURLRevisit allows multiple downloads of the same URL
	AllowURLRevisit bool
	// MaxBodySize is the limit of the retrieved response body in bytes.
	// `0` means unlimited.
	// The default value for MaxBodySize is 10MB (10 * 1024 * 1024 bytes).
	MaxBodySize int
	// CacheDir specifies a location where GET requests are cached as files.
	// When it's not defined, caching is disabled.
	CacheDir          string
	visitedURLs       []string
	htmlCallbacks     map[string]HTMLCallback
	requestCallbacks  []RequestCallback
	responseCallbacks []ResponseCallback
	backend           *httpBackend
	wg                *sync.WaitGroup
	lock              *sync.Mutex
}

// Request is the representation of a HTTP request made by a Collector
type Request struct {
	// URL is the parsed URL of the HTTP request
	URL *url.URL
	// Headers contains the Request's HTTP headers
	Headers *http.Header
	// Ctx is a context between a Request and a Response
	Ctx *Context
	// Depth is the number of the parents of this request
	Depth     int
	collector *Collector
}

// Response is the representation of a HTTP response made by a Collector
type Response struct {
	// StatusCode is the status code of the Response
	StatusCode int
	// Body is the content of the Response
	Body []byte
	// Ctx is a context between a Request and a Response
	Ctx *Context
	// Request is the Request object of the response
	Request *Request
	// Headers contains the Response's HTTP headers
	Headers *http.Header
}

// HTMLElement is the representation of a HTML tag.
type HTMLElement struct {
	// Name is the name of the tag
	Name       string
	Text       string
	attributes []html.Attribute
	// Request is the request object of the element's HTML document
	Request *Request
	// Response is the Response object of the element's HTML document
	Response *Response
	// DOM is the goquery parsed DOM object of the page. DOM is relative
	// to the current HTMLElement
	DOM *goquery.Selection
}

// Context provides a tiny layer for passing data between callbacks
type Context struct {
	contextMap map[string]string
	lock       *sync.Mutex
}

// RequestCallback is a type alias for OnRequest callback functions
type RequestCallback func(*Request)

// ResponseCallback is a type alias for OnResponse callback functions
type ResponseCallback func(*Response)

// HTMLCallback is a type alias for OnHTML callback functions
type HTMLCallback func(*HTMLElement)

// NewCollector creates a new Collector instance with default configuration
func NewCollector() *Collector {
	c := &Collector{}
	c.Init()
	return c
}

// NewContext initializes a new Context instance
func NewContext() *Context {
	return &Context{
		contextMap: make(map[string]string),
		lock:       &sync.Mutex{},
	}
}

// Init initializes the Collector's private variables and sets default
// configuration for the Collector
func (c *Collector) Init() {
	c.UserAgent = "colly - https://github.com/asciimoo/colly"
	c.MaxDepth = 0
	c.visitedURLs = make([]string, 0, 8)
	c.htmlCallbacks = make(map[string]HTMLCallback, 0)
	c.requestCallbacks = make([]RequestCallback, 0, 8)
	c.responseCallbacks = make([]ResponseCallback, 0, 8)
	c.MaxBodySize = 10 * 1024 * 1024
	c.backend = &httpBackend{}
	c.backend.Init()
	c.wg = &sync.WaitGroup{}
	c.lock = &sync.Mutex{}
}

// Visit starts Collector's collecting job by creating a
// request to the URL specified in parameter.
// Visit also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (c *Collector) Visit(URL string) error {
	return c.scrape(URL, "GET", 1, nil)
}

// Post starts collecting job by creating a POST
// request.
// Post also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (c *Collector) Post(URL string, requestData map[string]string) error {
	return c.scrape(URL, "POST", 1, requestData)
}

func (c *Collector) scrape(u, method string, depth int, requestData map[string]string) error {
	c.wg.Add(1)
	defer c.wg.Done()
	if u == "" {
		return errors.New("Missing URL")
	}
	if c.MaxDepth > 0 && c.MaxDepth < depth {
		return errors.New("Max depth limit reached")
	}
	if !c.AllowURLRevisit {
		visited := false
		for _, u2 := range c.visitedURLs {
			if u2 == u {
				visited = true
				break
			}
		}
		if visited {
			return errors.New("URL already visited")
		}
	}
	parsedURL, err := url.Parse(u)
	if err != nil {
		return err
	}
	allowed := false
	if c.AllowedDomains == nil || len(c.AllowedDomains) == 0 {
		allowed = true
	} else {
		for _, d := range c.AllowedDomains {
			if d == parsedURL.Host {
				allowed = true
				break
			}
		}
	}
	if !allowed {
		return errors.New("Forbidden domain")
	}
	if !c.AllowURLRevisit {
		c.lock.Lock()
		c.visitedURLs = append(c.visitedURLs, u)
		c.lock.Unlock()
	}
	var form url.Values
	if method == "POST" {
		form = url.Values{}
		for k, v := range requestData {
			form.Add(k, v)
		}
	}
	req, err := http.NewRequest(method, u, strings.NewReader(form.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", c.UserAgent)
	if method == "POST" {
		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	}
	ctx := NewContext()
	request := &Request{
		URL:       parsedURL,
		Headers:   &req.Header,
		Ctx:       ctx,
		Depth:     depth,
		collector: c,
	}
	if len(c.requestCallbacks) > 0 {
		c.handleOnRequest(request)
	}
	response, err := c.backend.Cache(req, c.MaxBodySize, c.CacheDir)
	// TODO add OnError callback to handle these cases
	if err != nil {
		return err
	}
	response.Ctx = ctx
	response.Request = request
	if strings.Index(strings.ToLower(response.Headers.Get("Content-Type")), "html") > -1 {
		c.handleOnHTML(response.Body, request, response)
	}
	if len(c.responseCallbacks) > 0 {
		c.handleOnResponse(response)
	}
	return nil
}

// Wait returns when the collector jobs are finished
func (c *Collector) Wait() {
	c.wg.Wait()
}

// OnRequest registers a function. Function will be executed on every
// request made by the Collector
func (c *Collector) OnRequest(f RequestCallback) {
	c.lock.Lock()
	c.requestCallbacks = append(c.requestCallbacks, f)
	c.lock.Unlock()
}

// OnResponse registers a function. Function will be executed on every response
func (c *Collector) OnResponse(f ResponseCallback) {
	c.lock.Lock()
	c.responseCallbacks = append(c.responseCallbacks, f)
	c.lock.Unlock()
}

// OnHTML registers a function. Function will be executed on every HTML
// element matched by the `goquerySelector` parameter.
// `goquerySelector` is a selector used by https://github.com/PuerkitoBio/goquery
func (c *Collector) OnHTML(goquerySelector string, f HTMLCallback) {
	c.lock.Lock()
	c.htmlCallbacks[goquerySelector] = f
	c.lock.Unlock()
}

// WithTransport allows you to set a custom http.Transport for this collector.
func (c *Collector) WithTransport(transport *http.Transport) {
	c.backend.Client.Transport = transport
}

// DisableCookies turns off cookie handling for this collector
func (c *Collector) DisableCookies() {
	c.backend.Client.Jar = nil
}

// SetRequestTimeout overrides the default timeout (10 seconds) for this collector
func (c *Collector) SetRequestTimeout(timeout time.Duration) {
	c.backend.Client.Timeout = timeout
}

func (c *Collector) handleOnRequest(r *Request) {
	for _, f := range c.requestCallbacks {
		f(r)
	}
}

func (c *Collector) handleOnResponse(r *Response) {
	for _, f := range c.responseCallbacks {
		f(r)
	}
}

func (c *Collector) handleOnHTML(body []byte, req *Request, resp *Response) {
	doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(body))
	if err != nil {
		return
	}
	for expr, f := range c.htmlCallbacks {
		doc.Find(expr).Each(func(i int, s *goquery.Selection) {
			for _, n := range s.Nodes {
				e := &HTMLElement{
					Name:       n.Data,
					Request:    req,
					Response:   resp,
					Text:       goquery.NewDocumentFromNode(n).Text(),
					DOM:        s,
					attributes: n.Attr,
				}
				f(e)
			}
		})
	}
}

// Limit adds a new `LimitRule` to the collector
func (c *Collector) Limit(rule *LimitRule) error {
	return c.backend.Limit(rule)
}

// Limits adds new `LimitRule`s to the collector
func (c *Collector) Limits(rules []*LimitRule) error {
	return c.backend.Limits(rules)
}

// Attr returns the selected attribute of a HTMLElement or empty string
// if no attribute found
func (h *HTMLElement) Attr(k string) string {
	for _, a := range h.attributes {
		if a.Key == k {
			return a.Val
		}
	}
	return ""
}

// AbsoluteURL returns with the resolved absolute URL of an URL chunk.
// AbsoluteURL returns empty string if the URL chunk is a fragment or
// could not be parsed
func (r *Request) AbsoluteURL(u string) string {
	if strings.HasPrefix(u, "#") {
		return ""
	}
	absURL, err := r.URL.Parse(u)
	if err != nil {
		return ""
	}
	absURL.Fragment = ""
	if absURL.Scheme == "//" {
		absURL.Scheme = r.URL.Scheme
	}
	return absURL.String()
}

// Visit continues Collector's collecting job by creating a
// request to the URL specified in parameter.
// Visit also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (r *Request) Visit(URL string) error {
	return r.collector.scrape(r.AbsoluteURL(URL), "GET", r.Depth+1, nil)
}

// Post continues a collector job by creating a POST request.
// Post also calls the previously provided OnRequest, OnResponse, OnHTML callbacks
func (r *Request) Post(URL string, requestData map[string]string) error {
	return r.collector.scrape(r.AbsoluteURL(URL), "POST", r.Depth+1, requestData)
}

func (c *Context) UnmarshalBinary(data []byte) error {
	return nil
}

func (c *Context) MarshalBinary() (data []byte, err error) {
	return nil, nil
}

// Put stores a value in Context
func (c *Context) Put(key, value string) {
	c.lock.Lock()
	c.contextMap[key] = value
	c.lock.Unlock()
}

// Get retrieves a value from Context. If no value found for `k`
// Get returns an empty string if key not found
func (c *Context) Get(key string) string {
	if v, ok := c.contextMap[key]; ok {
		return v
	}
	return ""
}
<|endoftext|>


<commit_before>package pixel

import "image/color"

// RGBA represents a alpha-premultiplied RGBA color with components within range [0, 1].
//
// The difference between color.RGBA is that the value range is [0, 1] and the values are floats.
type RGBA struct {
	R, G, B, A float64
}

// Add adds color d to color c component-wise and returns the result (the components are not
// clamped).
func (c RGBA) Add(d RGBA) RGBA {
	return RGBA{
		R: c.R + d.R,
		G: c.G + d.G,
		B: c.B + d.B,
		A: c.A + d.A,
	}
}

// Sub subtracts color d from color c component-wise and returns the result (the components
// are not clamped).
func (c RGBA) Sub(d RGBA) RGBA {
	return RGBA{
		R: c.R - d.R,
		G: c.G - d.G,
		B: c.B - d.B,
		A: c.A - d.A,
	}
}

// Mul multiplies color c by color d component-wise (the components are not clamped).
func (c RGBA) Mul(d RGBA) RGBA {
	return RGBA{
		R: c.R * d.R,
		G: c.G * d.G,
		B: c.B * d.B,
		A: c.A * d.A,
	}
}

// Scaled multiplies each component of color c by scale and returns the result (the components
// are not clamped).
func (c RGBA) Scaled(scale float64) RGBA {
	return RGBA{
		R: c.R * scale,
		G: c.G * scale,
		B: c.B * scale,
		A: c.A * scale,
	}
}

// RGBA returns alpha-premultiplied red, green, blue and alpha components of the RGBA color.
func (c RGBA) RGBA() (r, g, b, a uint32) {
	r = uint32(0xffff * c.R)
	g = uint32(0xffff * c.G)
	b = uint32(0xffff * c.B)
	a = uint32(0xffff * c.A)
	return
}

// ToRGBA converts a color to RGBA format. Using this function is preferred to using RGBAModel, for
// performance (using RGBAModel introduced additional unnecessary allocations).
func ToRGBA(c color.Color) RGBA {
	if c, ok := c.(RGBA); ok {
		return c
	}
	if c, ok := c.(color.RGBA); ok {
		return RGBA{
			R: float64(c.R) / 255,
			G: float64(c.G) / 255,
			B: float64(c.B) / 255,
			A: float64(c.A) / 255,
		}
	}
	r, g, b, a := c.RGBA()
	return RGBA{
		float64(r) / 0xffff,
		float64(g) / 0xffff,
		float64(b) / 0xffff,
		float64(a) / 0xffff,
	}
}

// RGBAModel converts colors to RGBA format.
var RGBAModel = color.ModelFunc(rgbaModel)

func rgbaModel(c color.Color) color.Color {
	return ToRGBA(c)
}
<commit_msg>add RGB and Alpha functions<commit_after>package pixel

import "image/color"

// RGBA represents a alpha-premultiplied RGBA color with components within range [0, 1].
//
// The difference between color.RGBA is that the value range is [0, 1] and the values are floats.
type RGBA struct {
	R, G, B, A float64
}

// RGB returns a fully opaque RGBA color with the given RGB values.
func RGB(r, g, b float64) RGBA {
	return RGBA{r, g, b, 1}
}

// Alpha returns a while RGBA color with the  given alpha component.
func Alpha(a float64) RGBA {
	return RGBA{a, a, a, a}
}

// Add adds color d to color c component-wise and returns the result (the components are not
// clamped).
func (c RGBA) Add(d RGBA) RGBA {
	return RGBA{
		R: c.R + d.R,
		G: c.G + d.G,
		B: c.B + d.B,
		A: c.A + d.A,
	}
}

// Sub subtracts color d from color c component-wise and returns the result (the components
// are not clamped).
func (c RGBA) Sub(d RGBA) RGBA {
	return RGBA{
		R: c.R - d.R,
		G: c.G - d.G,
		B: c.B - d.B,
		A: c.A - d.A,
	}
}

// Mul multiplies color c by color d component-wise (the components are not clamped).
func (c RGBA) Mul(d RGBA) RGBA {
	return RGBA{
		R: c.R * d.R,
		G: c.G * d.G,
		B: c.B * d.B,
		A: c.A * d.A,
	}
}

// Scaled multiplies each component of color c by scale and returns the result (the components
// are not clamped).
func (c RGBA) Scaled(scale float64) RGBA {
	return RGBA{
		R: c.R * scale,
		G: c.G * scale,
		B: c.B * scale,
		A: c.A * scale,
	}
}

// RGBA returns alpha-premultiplied red, green, blue and alpha components of the RGBA color.
func (c RGBA) RGBA() (r, g, b, a uint32) {
	r = uint32(0xffff * c.R)
	g = uint32(0xffff * c.G)
	b = uint32(0xffff * c.B)
	a = uint32(0xffff * c.A)
	return
}

// ToRGBA converts a color to RGBA format. Using this function is preferred to using RGBAModel, for
// performance (using RGBAModel introduced additional unnecessary allocations).
func ToRGBA(c color.Color) RGBA {
	if c, ok := c.(RGBA); ok {
		return c
	}
	if c, ok := c.(color.RGBA); ok {
		return RGBA{
			R: float64(c.R) / 255,
			G: float64(c.G) / 255,
			B: float64(c.B) / 255,
			A: float64(c.A) / 255,
		}
	}
	r, g, b, a := c.RGBA()
	return RGBA{
		float64(r) / 0xffff,
		float64(g) / 0xffff,
		float64(b) / 0xffff,
		float64(a) / 0xffff,
	}
}

// RGBAModel converts colors to RGBA format.
var RGBAModel = color.ModelFunc(rgbaModel)

func rgbaModel(c color.Color) color.Color {
	return ToRGBA(c)
}
<|endoftext|>


<commit_before>// +build linux

package ghw

import (
	"bufio"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

const (
	PathMtab        = "/etc/mtab"
	PathSysBlock    = "/sys/block"
	PathDevDiskById = "/dev/disk/by-id"
)

var RegexNVMeDev = regexp.MustCompile(`^nvme\d+n\d+$`)
var RegexNVMePart = regexp.MustCompile(`^(nvme\d+n\d+)p\d+$`)

func blockFillInfo(info *BlockInfo) error {
	info.Disks = Disks()
	var tpb uint64
	for _, d := range info.Disks {
		tpb += d.SizeBytes
	}
	info.TotalPhysicalBytes = tpb
	return nil
}

func DiskSectorSizeBytes(disk string) uint64 {
	// We can find the sector size in Linux by looking at the
	// /sys/block/$DEVICE/queue/physical_block_size file in sysfs
	path := filepath.Join(PathSysBlock, disk, "queue", "physical_block_size")
	contents, err := ioutil.ReadFile(path)
	if err != nil {
		return 0
	}
	i, err := strconv.Atoi(strings.TrimSpace(string(contents)))
	if err != nil {
		return 0
	}
	return uint64(i)
}

func DiskSizeBytes(disk string) uint64 {
	// We can find the number of 512-byte sectors by examining the contents of
	// /sys/block/$DEVICE/size and calculate the physical bytes accordingly.
	path := filepath.Join(PathSysBlock, disk, "size")
	contents, err := ioutil.ReadFile(path)
	if err != nil {
		return 0
	}
	ss := DiskSectorSizeBytes(disk)
	i, err := strconv.Atoi(strings.TrimSpace(string(contents)))
	if err != nil {
		return 0
	}
	return uint64(i) * ss
}

func DiskVendor(disk string) string {
	// In Linux, the vendor for a disk device is found in the
	// /sys/block/$DEVICE/device/vendor file in sysfs
	path := filepath.Join(PathSysBlock, disk, "device", "vendor")
	contents, err := ioutil.ReadFile(path)
	if err != nil {
		return "unknown"
	}
	return strings.TrimSpace(string(contents))
}

func DiskSerialNumber(disk string) string {
	// Finding the serial number of a disk without root privileges in Linux is
	// a little tricky. The /dev/disk/by-id directory contains a bunch of
	// symbolic links to disk devices and partitions. The serial number is
	// embedded as part of the symbolic link. For example, on my system, the
	// primary SCSI disk (/dev/sda) is represented as a symbolic link named
	// /dev/disk/by-id/scsi-3600508e000000000f8253aac9a1abd0c. The serial
	// number is 3600508e000000000f8253aac9a1abd0c.
	path := filepath.Join(PathDevDiskById)
	links, err := ioutil.ReadDir(path)
	if err != nil {
		return "unknown"
	}
	for _, link := range links {
		lname := link.Name()
		lpath := filepath.Join(PathDevDiskById, lname)
		dest, err := os.Readlink(lpath)
		if err != nil {
			continue
		}
		dest = filepath.Base(dest)
		if dest != disk {
			continue
		}
		pos := strings.LastIndexAny(lname, "-_")
		if pos >= 0 {
			return lname[pos+1:]
		}
	}
	return "unknown"
}

func DiskPartitions(disk string) []*Partition {
	out := make([]*Partition, 0)
	path := filepath.Join(PathSysBlock, disk)
	files, err := ioutil.ReadDir(path)
	if err != nil {
		return nil
	}
	for _, file := range files {
		fname := file.Name()
		if !strings.HasPrefix(fname, disk) {
			continue
		}
		size := PartitionSizeBytes(fname)
		mp, pt, ro := PartitionInfo(fname)
		p := &Partition{
			Name:       fname,
			SizeBytes:  size,
			MountPoint: mp,
			Type:       pt,
			IsReadOnly: ro,
		}
		out = append(out, p)
	}
	return out
}

func Disks() []*Disk {
	// In Linux, we could use the fdisk, lshw or blockdev commands to list disk
	// information, however all of these utilities require root privileges to
	// run. We can get all of this information by examining the /sys/block
	// and /sys/class/block files
	disks := make([]*Disk, 0)
	files, err := ioutil.ReadDir(PathSysBlock)
	if err != nil {
		return nil
	}
	for _, file := range files {
		dname := file.Name()

		var busType string
		if strings.HasPrefix(dname, "sd") {
			busType = "SCSI"
		} else if strings.HasPrefix(dname, "hd") {
			busType = "IDE"
		} else if RegexNVMeDev.MatchString(dname) {
			busType = "NVMe"
		}
		if busType == "" {
			continue
		}

		size := DiskSizeBytes(dname)
		ss := DiskSectorSizeBytes(dname)
		vendor := DiskVendor(dname)
		serialNo := DiskSerialNumber(dname)

		d := &Disk{
			Name:            dname,
			SizeBytes:       size,
			SectorSizeBytes: ss,
			BusType:         busType,
			Vendor:          vendor,
			SerialNumber:    serialNo,
		}

		parts := DiskPartitions(dname)
		// Map this Disk object into the Partition...
		for _, part := range parts {
			part.Disk = d
		}
		d.Partitions = parts

		disks = append(disks, d)
	}

	return disks
}

func PartitionSizeBytes(part string) uint64 {
	// Allow calling PartitionSize with either the full partition name
	// "/dev/sda1" or just "sda1"
	if strings.HasPrefix(part, "/dev") {
		part = part[4:len(part)]
	}
	disk := part[0:3]
	if m := RegexNVMePart.FindStringSubmatch(part); len(m) > 0 {
		disk = m[1]
	}
	path := filepath.Join(PathSysBlock, disk, part, "size")
	contents, err := ioutil.ReadFile(path)
	if err != nil {
		return 0
	}
	ss := DiskSectorSizeBytes(disk)
	i, err := strconv.Atoi(strings.TrimSpace(string(contents)))
	if err != nil {
		return 0
	}
	return uint64(i) * ss
}

// Given a full or short partition name, returns the mount point, the type of
// the partition and whether it's readonly
func PartitionInfo(part string) (string, string, bool) {
	// Allow calling PartitionInfo with either the full partition name
	// "/dev/sda1" or just "sda1"
	if !strings.HasPrefix(part, "/dev") {
		part = "/dev/" + part
	}

	// /etc/mtab entries for mounted partitions look like this:
	// /dev/sda6 / ext4 rw,relatime,errors=remount-ro,data=ordered 0 0
	var r io.ReadCloser
	r, err := os.Open(PathMtab)
	if err != nil {
		return "", "", true
	}
	defer r.Close()

	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		if line[0] != '/' {
			continue
		}
		fields := strings.Fields(line)
		if fields[0] != part {
			continue
		}
		opts := strings.Split(fields[3], ",")
		ro := true
		for _, opt := range opts {
			if opt == "rw" {
				ro = false
				break
			}
		}

		return fields[1], fields[2], ro
	}
	return "", "", true
}

func PartitionMountPoint(part string) string {
	mp, _, _ := PartitionInfo(part)
	return mp
}

func PartitionType(part string) string {
	_, pt, _ := PartitionInfo(part)
	return pt
}

func PartitionIsReadOnly(part string) bool {
	_, _, ro := PartitionInfo(part)
	return ro
}
<commit_msg>Add some commentary around odd SATA disk serial number identification<commit_after>// +build linux

package ghw

import (
	"bufio"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

const (
	PathMtab        = "/etc/mtab"
	PathSysBlock    = "/sys/block"
	PathDevDiskById = "/dev/disk/by-id"
)

var RegexNVMeDev = regexp.MustCompile(`^nvme\d+n\d+$`)
var RegexNVMePart = regexp.MustCompile(`^(nvme\d+n\d+)p\d+$`)

func blockFillInfo(info *BlockInfo) error {
	info.Disks = Disks()
	var tpb uint64
	for _, d := range info.Disks {
		tpb += d.SizeBytes
	}
	info.TotalPhysicalBytes = tpb
	return nil
}

func DiskSectorSizeBytes(disk string) uint64 {
	// We can find the sector size in Linux by looking at the
	// /sys/block/$DEVICE/queue/physical_block_size file in sysfs
	path := filepath.Join(PathSysBlock, disk, "queue", "physical_block_size")
	contents, err := ioutil.ReadFile(path)
	if err != nil {
		return 0
	}
	i, err := strconv.Atoi(strings.TrimSpace(string(contents)))
	if err != nil {
		return 0
	}
	return uint64(i)
}

func DiskSizeBytes(disk string) uint64 {
	// We can find the number of 512-byte sectors by examining the contents of
	// /sys/block/$DEVICE/size and calculate the physical bytes accordingly.
	path := filepath.Join(PathSysBlock, disk, "size")
	contents, err := ioutil.ReadFile(path)
	if err != nil {
		return 0
	}
	ss := DiskSectorSizeBytes(disk)
	i, err := strconv.Atoi(strings.TrimSpace(string(contents)))
	if err != nil {
		return 0
	}
	return uint64(i) * ss
}

func DiskVendor(disk string) string {
	// In Linux, the vendor for a disk device is found in the
	// /sys/block/$DEVICE/device/vendor file in sysfs
	path := filepath.Join(PathSysBlock, disk, "device", "vendor")
	contents, err := ioutil.ReadFile(path)
	if err != nil {
		return "unknown"
	}
	return strings.TrimSpace(string(contents))
}

func DiskSerialNumber(disk string) string {
	// Finding the serial number of a disk without root privileges in Linux is
	// a little tricky. The /dev/disk/by-id directory contains a bunch of
	// symbolic links to disk devices and partitions. The serial number is
	// embedded as part of the symbolic link. For example, on my system, the
	// primary SCSI disk (/dev/sda) is represented as a symbolic link named
	// /dev/disk/by-id/scsi-3600508e000000000f8253aac9a1abd0c. The serial
	// number is 3600508e000000000f8253aac9a1abd0c.
	//
	// Some SATA drives (or rather, disk drive vendors) use inconsistent ways
	// of putting the serial numbers of the disks in this symbolic link name.
	// For example, here are two SATA drive identifiers (examples come from
	// @antylama on GH Issue #19):
	//
	// /dev/disk/by-id/ata-AXIOMTEK_Corp.-FSA032G300MW5T-H_BCA11704240020001
	//
	// in the above identifier, "BCA11704240020001" is the drive serial number.
	// The vendor name along with what appears to be a vendor model name
	// (FSA032G300MW5T-H) are also included in the symbolic link name.
	//
	// /dev/disk/by-id/ata-WDC_WD10JFCX-68N6GN0_WD-WX31A76R3KFS
	//
	// in the above identifier, the serial number of the disk is actually
	// WD-WX31A76R3KFS, not WX31A76R3KFS. Go figure...
	path := filepath.Join(PathDevDiskById)
	links, err := ioutil.ReadDir(path)
	if err != nil {
		return "unknown"
	}
	for _, link := range links {
		lname := link.Name()
		lpath := filepath.Join(PathDevDiskById, lname)
		dest, err := os.Readlink(lpath)
		if err != nil {
			continue
		}
		dest = filepath.Base(dest)
		if dest != disk {
			continue
		}
		pos := strings.LastIndexAny(lname, "-_")
		if pos >= 0 {
			return lname[pos+1:]
		}
	}
	return "unknown"
}

func DiskPartitions(disk string) []*Partition {
	out := make([]*Partition, 0)
	path := filepath.Join(PathSysBlock, disk)
	files, err := ioutil.ReadDir(path)
	if err != nil {
		return nil
	}
	for _, file := range files {
		fname := file.Name()
		if !strings.HasPrefix(fname, disk) {
			continue
		}
		size := PartitionSizeBytes(fname)
		mp, pt, ro := PartitionInfo(fname)
		p := &Partition{
			Name:       fname,
			SizeBytes:  size,
			MountPoint: mp,
			Type:       pt,
			IsReadOnly: ro,
		}
		out = append(out, p)
	}
	return out
}

func Disks() []*Disk {
	// In Linux, we could use the fdisk, lshw or blockdev commands to list disk
	// information, however all of these utilities require root privileges to
	// run. We can get all of this information by examining the /sys/block
	// and /sys/class/block files
	disks := make([]*Disk, 0)
	files, err := ioutil.ReadDir(PathSysBlock)
	if err != nil {
		return nil
	}
	for _, file := range files {
		dname := file.Name()

		var busType string
		if strings.HasPrefix(dname, "sd") {
			busType = "SCSI"
		} else if strings.HasPrefix(dname, "hd") {
			busType = "IDE"
		} else if RegexNVMeDev.MatchString(dname) {
			busType = "NVMe"
		}
		if busType == "" {
			continue
		}

		size := DiskSizeBytes(dname)
		ss := DiskSectorSizeBytes(dname)
		vendor := DiskVendor(dname)
		serialNo := DiskSerialNumber(dname)

		d := &Disk{
			Name:            dname,
			SizeBytes:       size,
			SectorSizeBytes: ss,
			BusType:         busType,
			Vendor:          vendor,
			SerialNumber:    serialNo,
		}

		parts := DiskPartitions(dname)
		// Map this Disk object into the Partition...
		for _, part := range parts {
			part.Disk = d
		}
		d.Partitions = parts

		disks = append(disks, d)
	}

	return disks
}

func PartitionSizeBytes(part string) uint64 {
	// Allow calling PartitionSize with either the full partition name
	// "/dev/sda1" or just "sda1"
	if strings.HasPrefix(part, "/dev") {
		part = part[4:len(part)]
	}
	disk := part[0:3]
	if m := RegexNVMePart.FindStringSubmatch(part); len(m) > 0 {
		disk = m[1]
	}
	path := filepath.Join(PathSysBlock, disk, part, "size")
	contents, err := ioutil.ReadFile(path)
	if err != nil {
		return 0
	}
	ss := DiskSectorSizeBytes(disk)
	i, err := strconv.Atoi(strings.TrimSpace(string(contents)))
	if err != nil {
		return 0
	}
	return uint64(i) * ss
}

// Given a full or short partition name, returns the mount point, the type of
// the partition and whether it's readonly
func PartitionInfo(part string) (string, string, bool) {
	// Allow calling PartitionInfo with either the full partition name
	// "/dev/sda1" or just "sda1"
	if !strings.HasPrefix(part, "/dev") {
		part = "/dev/" + part
	}

	// /etc/mtab entries for mounted partitions look like this:
	// /dev/sda6 / ext4 rw,relatime,errors=remount-ro,data=ordered 0 0
	var r io.ReadCloser
	r, err := os.Open(PathMtab)
	if err != nil {
		return "", "", true
	}
	defer r.Close()

	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		if line[0] != '/' {
			continue
		}
		fields := strings.Fields(line)
		if fields[0] != part {
			continue
		}
		opts := strings.Split(fields[3], ",")
		ro := true
		for _, opt := range opts {
			if opt == "rw" {
				ro = false
				break
			}
		}

		return fields[1], fields[2], ro
	}
	return "", "", true
}

func PartitionMountPoint(part string) string {
	mp, _, _ := PartitionInfo(part)
	return mp
}

func PartitionType(part string) string {
	_, pt, _ := PartitionInfo(part)
	return pt
}

func PartitionIsReadOnly(part string) bool {
	_, _, ro := PartitionInfo(part)
	return ro
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/couchbaselabs/dparval"
	"github.com/couchbaselabs/go-couchbase"
	"github.com/couchbaselabs/indexing/api"
	"github.com/couchbaselabs/indexing/server"
	ast "github.com/couchbaselabs/tuqtng/ast"
	"github.com/dustin/gomemcached/client"
	"github.com/prataprc/golib"
	"github.com/prataprc/monster"
	"log"
	"math/rand"
	"strconv"
)

var options struct {
	host     string
	port     int
	userProd string
	projProd string
	seed     int
	count    int
}

func argParse() {
	flag.StringVar(&options.host, "host", "localhost", "Port to connect")
	flag.IntVar(&options.port, "port", 11212, "Host to connect")
	flag.StringVar(&options.userProd, "userProd", "", "monster production for users")
	flag.StringVar(&options.projProd, "projProd", "", "monster production for project")
	flag.IntVar(&options.seed, "seed", 100, "seed for production")
	flag.IntVar(&options.count, "count", 10, "number of documents to generate")
	flag.Parse()
}

func main() {
	argParse()

	//mcdport := int(8091)
	//if options.port == 11212 {
	//    mcdport = 12000
	//}

	// Couchbase client, pool and default bucket
	url := "http://" + options.host + ":" + strconv.Itoa(options.port)
	couch, err := couchbase.Connect(url)
	if err != nil {
		log.Fatalf("Error connecting:  %v", err)
	}
	pool, err := couch.GetPool("default")
	if err != nil {
		log.Fatalf("Error getting pool:  %v", err)
	}

	imURL := "http://" + options.host + ":" + strconv.Itoa(8094)
	client := server.NewRestClient(imURL)
	// nodes := client.Nodes()

	notifych := make(chan string, 1)
	tapch := make(chan []interface{}, 100)

	if users, err := pool.GetBucket("users"); err == nil {
		go populateUsers(users)
	} else {
		panic("Unable to get-bucket `users`")
	}
	if projects, err := pool.GetBucket("projects"); err == nil {
		go populateProjects(projects)
	} else {
		panic("Unable to get-bucket `users`")
	}

	tryCreateIndex(client)
	for {
		serverUuid, indexinfos, err := client.List("")
		if err != nil {
			panic(err)
		}
		bucketexprs := parseExpression(indexinfos)

		feeds := tapStream(pool, indexinfos, tapch)
		go waitNotify(client, serverUuid, notifych)

		for {
			select {
			case <-notifych:
				closeFeeds(feeds)
				break
			case msg := <-tapch:
				bucket, tevent := msg[0].(string), msg[1].(memcached.TapEvent)
				for idx, astexprs := range bucketexprs {
					if idx[0] != bucket {
						continue
					}
					secKey := evaluate(tevent.Value, astexprs)
					log.Println(idx[0], idx[1], secKey, string(tevent.Key))
				}
			}
		}
	}
}

func evaluate(value []byte, astexprs []ast.Expression) []string {
	secKey := make([]string, 0)
	for _, expr := range astexprs {
		key, err := expr.Evaluate(dparval.NewValueFromBytes([]byte(value)))
		if err != nil {
			log.Println(err)
			secKey = append(secKey, "")
		} else {
			secKey = append(secKey, string(key.Bytes()))
		}
	}
	return secKey
}

func parseExpression(indexinfos []api.IndexInfo) map[[3]string][]ast.Expression {
	bucketexprs := make(map[[3]string][]ast.Expression)
	for _, indexinfo := range indexinfos {
		idx := [3]string{indexinfo.Bucket, indexinfo.Name, indexinfo.Uuid}
		astexprs := make([]ast.Expression, 0)
		for _, expr := range indexinfo.OnExprList {
			if ex, err := ast.UnmarshalExpression([]byte(expr)); err == nil {
				astexprs = append(astexprs, ex)
			} else {
				panic(err)
			}
		}
		bucketexprs[idx] = astexprs
	}
	return bucketexprs
}

func tryCreateIndex(client *server.RestClient) (err error) {
	if _, indexinfos, err := client.List(""); err != nil {
		panic(err)
	} else if len(indexinfos) == 4 {
		return nil
	}
	indexinfos := []api.IndexInfo{
		api.IndexInfo{
			Name:       "emailid",
			Using:      api.Llrb,
			OnExprList: []string{`{"type":"property","path":"emailid"}`},
			Bucket:     "users",
			IsPrimary:  false,
			Exprtype:   "simple",
		},
		api.IndexInfo{
			Name:       "age",
			Using:      api.Llrb,
			OnExprList: []string{`{"type":"property","path":"age"}`},
			Bucket:     "users",
			IsPrimary:  false,
			Exprtype:   "simple",
		},
		api.IndexInfo{
			Name:       "projects-members",
			Using:      api.Llrb,
			OnExprList: []string{`{"type":"property","path":"members"}`},
			Bucket:     "projects",
			IsPrimary:  false,
			Exprtype:   "simple",
		},
		api.IndexInfo{
			Name:       "project-language",
			Using:      api.Llrb,
			OnExprList: []string{`{"type":"property","path":"language"}`},
			Bucket:     "projects",
			IsPrimary:  false,
			Exprtype:   "simple",
		},
	}
	for _, indexinfo := range indexinfos {
		if _, _, err = client.Create(indexinfo); err != nil {
			panic(err)
		}
	}
	return
}

func waitNotify(client *server.RestClient, serverUuid string, ch chan string) {
	if _, serverUuid, err := client.Notify(serverUuid); err != nil {
		panic(err)
	} else {
		ch <- serverUuid
	}
}

func tapStream(pool couchbase.Pool, indexinfos []api.IndexInfo,
	tapch chan []interface{}) (feeds []*couchbase.TapFeed) {

	var bucket *couchbase.Bucket
	var err error

	for _, indexinfo := range indexinfos {
		if bucket, err = pool.GetBucket(indexinfo.Bucket); err != nil {
			panic(err)
		}
		args := memcached.TapArguments{
			Dump:       false,
			SupportAck: false,
			KeysOnly:   false,
			Checkpoint: true,
			ClientName: "indexer",
		}
		feed, err := bucket.StartTapFeed(&args)
		if err != nil {
			panic(err)
		}
		feeds = append(feeds, feed)
		go runFeed(feed, indexinfo.Bucket, tapch)
	}
	return
}

func runFeed(feed *couchbase.TapFeed, bucket string, tapch chan []interface{}) {
	log.Println("feed for bucket", bucket, "...")
	for {
		if event, ok := <-feed.C; ok {
			tapch <- []interface{}{bucket, event}
		} else {
			log.Println("closing tap feed for", bucket)
			break
		}
	}
}

func closeFeeds(feeds []*couchbase.TapFeed) {
	for _, feed := range feeds {
		feed.Close()
	}
}

func populateUsers(bucket *couchbase.Bucket) {
	// Map of interfaces can receive any value types
	value := map[string]interface{}{}

	if options.userProd == "" {
		return
	}

	conf := make(golib.Config)
	start := monster.Parse(options.userProd, conf)

	c := make(monster.Context)
	nonterminals, root := monster.Build(start)
	c["_random"] = rand.New(rand.NewSource(int64(options.seed)))
	c["_nonterminals"] = nonterminals
	for i := 0; i < options.count; i++ {
		jsbytes := []byte(root.Generate(c))
		if err := json.Unmarshal(jsbytes, &value); err != nil {
			panic(err)
		}
		docid := fmt.Sprintf("user%v", i)
		bucket.Set(docid, 0, value)
	}
}

func populateProjects(bucket *couchbase.Bucket) {
	// Map of interfaces can receive any value types
	value := map[string]interface{}{}

	if options.projProd == "" {
		return
	}

	conf := make(golib.Config)
	start := monster.Parse(options.projProd, conf)

	c := make(monster.Context)
	nonterminals, root := monster.Build(start)
	c["_random"] = rand.New(rand.NewSource(int64(options.seed)))
	c["_nonterminals"] = nonterminals
	for i := 0; i < options.count; i++ {
		jsbytes := []byte(root.Generate(c))
		if err := json.Unmarshal(jsbytes, &value); err != nil {
			panic(err)
		}
		docid := fmt.Sprintf("project%v", i)
		bucket.Set(docid, 0, value)
	}
}
<commit_msg>Integrated with mutation manager.<commit_after>package main

import (
    "encoding/json"
    "flag"
    "fmt"
    "github.com/couchbaselabs/dparval"
    "github.com/couchbaselabs/go-couchbase"
    "github.com/couchbaselabs/indexing/api"
    "github.com/couchbaselabs/indexing/server"
    ast "github.com/couchbaselabs/tuqtng/ast"
    memcached "github.com/dustin/gomemcached"
    mc "github.com/dustin/gomemcached/client"
    "github.com/prataprc/golib"
    "github.com/prataprc/monster"
    "log"
    "math/rand"
    "net"
    "net/rpc"
    "net/rpc/jsonrpc"
    "strconv"
)

var options struct {
    host     string
    port     int
    userProd string
    projProd string
    seed     int
    count    int
}

func argParse() {
    flag.StringVar(&options.host, "host", "localhost", "Port to connect")
    flag.IntVar(&options.port, "port", 11211, "Host to connect")
    flag.StringVar(&options.userProd, "userProd", "", "monster production for users")
    flag.StringVar(&options.projProd, "projProd", "", "monster production for project")
    flag.IntVar(&options.seed, "seed", 100, "seed for production")
    flag.IntVar(&options.count, "count", 10, "number of documents to generate")
    flag.Parse()
}

func main() {
    argParse()

    //mcdport := int(8091)
    //if options.port == 11212 {
    //    mcdport = 12000
    //}

    // Couchbase client, pool and default bucket
    url := "http://" + options.host + ":" + strconv.Itoa(options.port)
    couch, err := couchbase.Connect(url)
    if err != nil {
        log.Fatalf("Error connecting:  %v", err)
    }
    pool, err := couch.GetPool("default")
    if err != nil {
        log.Fatalf("Error getting pool:  %v", err)
    }

    imURL := "http://" + options.host + ":" + strconv.Itoa(8094)
    imanager := imclient.NewRestClient(imURL)
    // nodes := imanager.Nodes()

    // Start pumping user mutations if user-production file is available
    if options.userProd != "" {
        if users, err := pool.GetBucket("users"); err == nil {
            go populateUsers(users)
        } else {
            panic("Unable to get-bucket `users`")
        }
    }
    // Start pumping projects mutations if user-production file is available
    if options.projProd != "" {
        if projects, err := pool.GetBucket("projects"); err == nil {
            go populateProjects(projects)
        } else {
            panic("Unable to get-bucket `users`")
        }
    }

    rpcconn, err := net.Dial("tcp", "localhost:8096")
    if err != nil {
        panic(err)
    }
    defer rpcconn.Close()
    c := jsonrpc.NewClient(rpcconn)

    for {
        serverUuid, indexinfos, err := imanager.List("")
        fmt.Println(indexinfos)
        if err != nil {
            panic(err)
        }
        notifych := make(chan string, 1)
        tapch := make(chan []interface{}, 100)
        bucketexprs := parseExpression(indexinfos)
        feeds := tapStream(pool, indexinfos, tapch)
        go waitNotify(imanager, serverUuid, notifych)
        loop(c, notifych, tapch, bucketexprs)
        closeFeeds(feeds)
    }
}

var tapop2type = map[memcached.CommandCode]string{
    memcached.TAP_MUTATION: "INSERT",
    memcached.TAP_DELETE:   "DELETE",
}

func loop(c *rpc.Client, notifych chan string, tapch chan []interface{},
    bucketexprs map[[3]string][]ast.Expression) {

    for {
        select {
        case <-notifych:
            break
        case msg := <-tapch:
            uuid, bucket := msg[0].(string), msg[1].(string)
            tevent := msg[2].(mc.TapEvent)
            for idx, astexprs := range bucketexprs {
                if idx[0] != bucket {
                    continue
                }
                m := api.Mutation{
                    Type:         tapop2type[memcached.CommandCode(tevent.Opcode)],
                    Indexid:      uuid,
                    SecondaryKey: evaluate(tevent.Value, astexprs),
                    Docid:        string(tevent.Key),
                }
                log.Println(m)
                //var r bool
                //err := c.Call("MutationManager.ProcessSingleMutation", m, &r)
                //if err != nil {
                //    log.Fatal("Mutation error:", err)
                //}
            }
        }
    }
}

func evaluate(value []byte, astexprs []ast.Expression) [][]byte {
    secKey := make([][]byte, 0)
    for _, expr := range astexprs {
        key, err := expr.Evaluate(dparval.NewValueFromBytes([]byte(value)))
        if err != nil {
            log.Println(err)
            secKey = append(secKey, []byte{})
        } else {
            secKey = append(secKey, key.Bytes())
        }
    }
    return secKey
}

func parseExpression(indexinfos []api.IndexInfo) map[[3]string][]ast.Expression {
    bucketexprs := make(map[[3]string][]ast.Expression)
    for _, indexinfo := range indexinfos {
        idx := [3]string{indexinfo.Bucket, indexinfo.Name, indexinfo.Uuid}
        astexprs := make([]ast.Expression, 0)
        for _, expr := range indexinfo.OnExprList {
            if ex, err := ast.UnmarshalExpression([]byte(expr)); err == nil {
                astexprs = append(astexprs, ex)
            } else {
                panic(err)
            }
        }
        bucketexprs[idx] = astexprs
    }
    return bucketexprs
}

func waitNotify(imanager *imclient.RestClient, serverUuid string, ch chan string) {
    if _, serverUuid, err := imanager.Notify(serverUuid); err != nil {
        panic(err)
    } else {
        ch <- serverUuid
    }
}

func tapStream(pool couchbase.Pool, indexinfos []api.IndexInfo,
    tapch chan []interface{}) (feeds []*couchbase.TapFeed) {

    var bucket *couchbase.Bucket
    var err error

    for _, indexinfo := range indexinfos {
        if bucket, err = pool.GetBucket(indexinfo.Bucket); err != nil {
            panic(err)
        }
        args := mc.TapArguments{
            Dump:       false,
            SupportAck: false,
            KeysOnly:   false,
            Checkpoint: true,
            ClientName: "indexer",
        }
        feed, err := bucket.StartTapFeed(&args)
        if err != nil {
            panic(err)
        }
        feeds = append(feeds, feed)
        go runFeed(feed, indexinfo, tapch)
    }
    return
}

func runFeed(feed *couchbase.TapFeed, indexinfo api.IndexInfo, tapch chan []interface{}) {
    bucket := indexinfo.Bucket
    log.Println("feed for bucket", bucket, "...")
    for {
        if event, ok := <-feed.C; ok {
            tapch <- []interface{}{indexinfo.Uuid, bucket, event}
        } else {
            log.Println("closing tap feed for", bucket)
            break
        }
    }
}

func closeFeeds(feeds []*couchbase.TapFeed) {
    for _, feed := range feeds {
        feed.Close()
    }
}

func populateUsers(bucket *couchbase.Bucket) {
    // Map of interfaces can receive any value types
    value := map[string]interface{}{}

    if options.userProd == "" {
        return
    }

    conf := make(golib.Config)
    start := monster.Parse(options.userProd, conf)

    c := make(monster.Context)
    nonterminals, root := monster.Build(start)
    c["_random"] = rand.New(rand.NewSource(int64(options.seed)))
    c["_nonterminals"] = nonterminals
    for i := 0; i < options.count; i++ {
        jsbytes := []byte(root.Generate(c))
        if err := json.Unmarshal(jsbytes, &value); err != nil {
            panic(err)
        }
        docid := fmt.Sprintf("user%v", i)
        bucket.Set(docid, 0, value)
    }
}

func populateProjects(bucket *couchbase.Bucket) {
    // Map of interfaces can receive any value types
    value := map[string]interface{}{}

    if options.projProd == "" {
        return
    }

    conf := make(golib.Config)
    start := monster.Parse(options.projProd, conf)

    c := make(monster.Context)
    nonterminals, root := monster.Build(start)
    c["_random"] = rand.New(rand.NewSource(int64(options.seed)))
    c["_nonterminals"] = nonterminals
    for i := 0; i < options.count; i++ {
        jsbytes := []byte(root.Generate(c))
        if err := json.Unmarshal(jsbytes, &value); err != nil {
            panic(err)
        }
        docid := fmt.Sprintf("project%v", i)
        bucket.Set(docid, 0, value)
    }
}
<|endoftext|>


<commit_before>package main

const NGINX_BUILD_VERSION = "0.6.6"

// nginx
const (
	NGINX_VERSION             = "1.9.10"
	NGINX_DOWNLOAD_URL_PREFIX = "http://nginx.org/download"
)

// pcre
const (
	PCRE_VERSION             = "8.38"
	PCRE_DOWNLOAD_URL_PREFIX = "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre"
)

// openssl
const (
	OPENSSL_VERSION             = "1.0.2e"
	OPENSSL_DOWNLOAD_URL_PREFIX = "http://www.openssl.org/source"
)

// zlib
const (
	ZLIB_VERSION             = "1.2.8"
	ZLIB_DOWNLOAD_URL_PREFIX = "http://zlib.net"
)

// openResty
const (
	OPENRESTY_VERSION             = "1.9.7.1"
	OPENRESTY_DOWNLOAD_URL_PREFIX = "http://openresty.org/download"
)

// tengine
const (
	TENGINE_VERSION             = "2.1.2"
	TENGINE_DOWNLOAD_URL_PREFIX = "http://tengine.taobao.org/download"
)

// component enumerations
const (
	COMPONENT_NGINX = iota
	COMPONENT_OPENRESTY
	COMPONENT_TENGINE
	COMPONENT_PCRE
	COMPONENT_OPENSSL
	COMPONENT_ZLIB
	COMPONENT_MAX
)
<commit_msg>bumped openresty version to 1.9.7.2.<commit_after>package main

const NGINX_BUILD_VERSION = "0.6.6"

// nginx
const (
	NGINX_VERSION             = "1.9.10"
	NGINX_DOWNLOAD_URL_PREFIX = "http://nginx.org/download"
)

// pcre
const (
	PCRE_VERSION             = "8.38"
	PCRE_DOWNLOAD_URL_PREFIX = "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre"
)

// openssl
const (
	OPENSSL_VERSION             = "1.0.2e"
	OPENSSL_DOWNLOAD_URL_PREFIX = "http://www.openssl.org/source"
)

// zlib
const (
	ZLIB_VERSION             = "1.2.8"
	ZLIB_DOWNLOAD_URL_PREFIX = "http://zlib.net"
)

// openResty
const (
	OPENRESTY_VERSION             = "1.9.7.2"
	OPENRESTY_DOWNLOAD_URL_PREFIX = "http://openresty.org/download"
)

// tengine
const (
	TENGINE_VERSION             = "2.1.2"
	TENGINE_DOWNLOAD_URL_PREFIX = "http://tengine.taobao.org/download"
)

// component enumerations
const (
	COMPONENT_NGINX = iota
	COMPONENT_OPENRESTY
	COMPONENT_TENGINE
	COMPONENT_PCRE
	COMPONENT_OPENSSL
	COMPONENT_ZLIB
	COMPONENT_MAX
)
<|endoftext|>


<commit_before>package scipipe

const (
	// Version is the SciPipe version in string format
	Version = "0.9.14"
)
<commit_msg>Bump version to 0.10.0<commit_after>package scipipe

const (
	// Version is the SciPipe version in string format
	Version = "0.10.0"
)
<|endoftext|>


<commit_before>package shared

import "errors"

/*
Errors of Tinzenite.
*/
var (
	ErrUnsupported      = errors.New("feature currently unsupported")
	ErrIsTinzenite      = errors.New("already a Tinzenite directory")
	ErrNotTinzenite     = errors.New("path is not valid Tinzenite directory")
	ErrNoTinIgnore      = errors.New("no .tinignore file found")
	ErrUntracked        = errors.New("object is not tracked in the model")
	ErrNilInternalState = errors.New("internal state has illegal NIL values")
	ErrConflict         = errors.New("conflict, can not apply")
	ErrIllegalFileState = errors.New("illegal file state detected")
)

/*
Internal errors of Tinzenite.
*/
var (
	errWrongObject = errors.New("wrong ObjectInfo")
)

// constant value here
const (
	/*RANDOMSEEDLENGTH is the amount of bytes used as cryptographic hash seed.*/
	RANDOMSEEDLENGTH = 32
	/*IDMAXLENGTH is the length in chars of new random identification hashes.*/
	IDMAXLENGTH = 16
	/*KEYLENGTH is the length of the encryption key used for challenges and file encryption.*/
	KEYLENGTH = 256
	/*FILEPERMISSIONMODE used for all file operations.*/
	FILEPERMISSIONMODE = 0777
	/*CHUNKSIZE for hashing and encryption.*/
	CHUNKSIZE = 8 * 1024
)

// Path constants here
const (
	TINZENITEDIR  = ".tinzenite"
	TINIGNORE     = ".tinignore"
	DIRECTORYLIST = "directory.list"
	LOCALDIR      = "local"
	TEMPDIR       = "temp"
	RECEIVINGDIR  = "receiving"
	REMOVEDIR     = "removed"
	ORGDIR        = "org"
	PEERSDIR      = "peers"
	ENDING        = ".json"
	AUTHJSON      = "auth" + ENDING
	MODELJSON     = "model" + ENDING
	SELFPEERJSON  = "self" + ENDING
)
<commit_msg>bootstrapping work<commit_after>package shared

import "errors"

/*
Errors of Tinzenite.
*/
var (
	ErrUnsupported      = errors.New("feature currently unsupported")
	ErrIsTinzenite      = errors.New("already a Tinzenite directory")
	ErrNotTinzenite     = errors.New("path is not valid Tinzenite directory")
	ErrNoTinIgnore      = errors.New("no .tinignore file found")
	ErrUntracked        = errors.New("object is not tracked in the model")
	ErrNilInternalState = errors.New("internal state has illegal NIL values")
	ErrConflict         = errors.New("conflict, can not apply")
	ErrIllegalFileState = errors.New("illegal file state detected")
)

/*
Internal errors of Tinzenite.
*/
var (
	errWrongObject = errors.New("wrong ObjectInfo")
)

// constant value here
const (
	/*RANDOMSEEDLENGTH is the amount of bytes used as cryptographic hash seed.*/
	RANDOMSEEDLENGTH = 32
	/*IDMAXLENGTH is the length in chars of new random identification hashes.*/
	IDMAXLENGTH = 16
	/*KEYLENGTH is the length of the encryption key used for challenges and file encryption.*/
	KEYLENGTH = 256
	/*FILEPERMISSIONMODE used for all file operations.*/
	FILEPERMISSIONMODE = 0777
	/*CHUNKSIZE for hashing and encryption.*/
	CHUNKSIZE = 8 * 1024
)

// Path constants here
const (
	TINZENITEDIR  = ".tinzenite"
	TINIGNORE     = ".tinignore"
	DIRECTORYLIST = "directory.list"
	LOCALDIR      = "local"
	TEMPDIR       = "temp"
	RECEIVINGDIR  = "receiving"
	REMOVEDIR     = "removed"
	ORGDIR        = "org"
	PEERSDIR      = "peers"
	ENDING        = ".json"
	AUTHJSON      = "auth" + ENDING
	MODELJSON     = "model" + ENDING
	SELFPEERJSON  = "self" + ENDING
	BOOTJSON      = "boot" + ENDING
)
<|endoftext|>


<commit_before>package main

const NGINX_BUILD_VERSION = "0.3.1"

// nginx
const (
	NGINX_VERSION             = "1.7.10"
	NGINX_DOWNLOAD_URL_PREFIX = "http://nginx.org/download"
)

// pcre
const (
	PCRE_VERSION             = "8.36"
	PCRE_DOWNLOAD_URL_PREFIX = "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre"
)

// openssl
const (
	OPENSSL_VERSION             = "1.0.2a"
	OPENSSL_DOWNLOAD_URL_PREFIX = "http://www.openssl.org/source"
)

// zlib
const (
	ZLIB_VERSION             = "1.2.8"
	ZLIB_DOWNLOAD_URL_PREFIX = "http://zlib.net"
)

// openResty
const (
	OPENRESTY_VERSION             = "1.7.10.1"
	OPENRESTY_DOWNLOAD_URL_PREFIX = "http://openresty.org/download"
)

// tengine
const (
	TENGINE_VERSION             = "2.1.0"
	TENGINE_DOWNLOAD_URL_PREFIX = "http://tengine.taobao.org/download"
)

// component enumerations
const (
	COMPONENT_NGINX = iota
	COMPONENT_OPENRESTY
	COMPONENT_TENGINE
	COMPONENT_PCRE
	COMPONENT_OPENSSL
	COMPONENT_ZLIB
	COMPONENT_MAX
)
<commit_msg>bumped version to 0.3.2.<commit_after>package main

const NGINX_BUILD_VERSION = "0.3.2"

// nginx
const (
	NGINX_VERSION             = "1.7.10"
	NGINX_DOWNLOAD_URL_PREFIX = "http://nginx.org/download"
)

// pcre
const (
	PCRE_VERSION             = "8.36"
	PCRE_DOWNLOAD_URL_PREFIX = "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre"
)

// openssl
const (
	OPENSSL_VERSION             = "1.0.2a"
	OPENSSL_DOWNLOAD_URL_PREFIX = "http://www.openssl.org/source"
)

// zlib
const (
	ZLIB_VERSION             = "1.2.8"
	ZLIB_DOWNLOAD_URL_PREFIX = "http://zlib.net"
)

// openResty
const (
	OPENRESTY_VERSION             = "1.7.10.1"
	OPENRESTY_DOWNLOAD_URL_PREFIX = "http://openresty.org/download"
)

// tengine
const (
	TENGINE_VERSION             = "2.1.0"
	TENGINE_DOWNLOAD_URL_PREFIX = "http://tengine.taobao.org/download"
)

// component enumerations
const (
	COMPONENT_NGINX = iota
	COMPONENT_OPENRESTY
	COMPONENT_TENGINE
	COMPONENT_PCRE
	COMPONENT_OPENSSL
	COMPONENT_ZLIB
	COMPONENT_MAX
)
<|endoftext|>


<commit_before>package project

import (
	"crypto/md5"
	"fmt"

	"github.com/hashicorp/hcl"
	"github.com/juhovuori/builder/fetcher"
)

// Project represents a single managed project
type Project interface {
	Manager
	Attributes
}

// Manager represents the manager of a single project
type Manager interface {
	URL() string
	ID() string
	Err() error
}

// Attributes represents the configuration of a single project
type Attributes interface {
	Name() string
	Description() string
	Script() string
}

// defaultProject is the main implementation of Project
type defaultProject struct {
	Purl         string `json:"url"`
	Pmd5         string `json:"id"`
	Perr         error  `json:"error"`
	Pname        string `hcl:"name"`
	Pdescription string `hcl:"description"`
	Pscript      string `hcl:"script"`
}

func (p *defaultProject) URL() string {
	return p.Purl
}

func (p *defaultProject) ID() string {
	return p.Pmd5
}

func (p *defaultProject) Err() error {
	return p.Perr
}

func (p *defaultProject) Name() string {
	return p.Pname
}

func (p *defaultProject) Description() string {
	return p.Pdescription
}

func (p *defaultProject) Script() string {
	return p.Pscript
}

func fetchConfig(filename string) (string, error) {
	bytes, err := fetcher.Fetch(filename)
	return string(bytes), err
}

func newProject(config string, URL string, err error) (*defaultProject, error) {
	var p defaultProject
	if err == nil {
		err = hcl.Decode(&p, config)
	}
	MD5 := md5.Sum([]byte(URL))
	p.Purl = URL
	p.Pmd5 = fmt.Sprintf("%x", MD5)
	p.Perr = err
	return &p, err
}

// NewProject creates a new project
func NewProject(URL string) (Project, error) {
	config, err := fetchConfig(URL)
	return newProject(config, URL, err)
}

// NewFromString creates a new project from configuration string
func NewFromString(config string) (Project, error) {
	return newProject(config, "", nil)
}
<commit_msg>Fix project JSON serialisation<commit_after>package project

import (
	"crypto/md5"
	"fmt"

	"github.com/hashicorp/hcl"
	"github.com/juhovuori/builder/fetcher"
)

// Project represents a single managed project
type Project interface {
	Manager
	Attributes
}

// Manager represents the manager of a single project
type Manager interface {
	URL() string
	ID() string
	Err() error
}

// Attributes represents the configuration of a single project
type Attributes interface {
	Name() string
	Description() string
	Script() string
}

// defaultProject is the main implementation of Project
type defaultProject struct {
	Purl         string `json:"url"`
	Pmd5         string `json:"id"`
	Perr         error  `json:"error"`
	Pname        string `json:"name" hcl:"name"`
	Pdescription string `json:"description" hcl:"description"`
	Pscript      string `json:"script" hcl:"script"`
}

func (p *defaultProject) URL() string {
	return p.Purl
}

func (p *defaultProject) ID() string {
	return p.Pmd5
}

func (p *defaultProject) Err() error {
	return p.Perr
}

func (p *defaultProject) Name() string {
	return p.Pname
}

func (p *defaultProject) Description() string {
	return p.Pdescription
}

func (p *defaultProject) Script() string {
	return p.Pscript
}

func fetchConfig(filename string) (string, error) {
	bytes, err := fetcher.Fetch(filename)
	return string(bytes), err
}

func newProject(config string, URL string, err error) (*defaultProject, error) {
	var p defaultProject
	if err == nil {
		err = hcl.Decode(&p, config)
	}
	MD5 := md5.Sum([]byte(URL))
	p.Purl = URL
	p.Pmd5 = fmt.Sprintf("%x", MD5)
	p.Perr = err
	return &p, err
}

// NewProject creates a new project
func NewProject(URL string) (Project, error) {
	config, err := fetchConfig(URL)
	return newProject(config, URL, err)
}

// NewFromString creates a new project from configuration string
func NewFromString(config string) (Project, error) {
	return newProject(config, "", nil)
}
<|endoftext|>


<commit_before>// -*- tab-width: 4 -*-
package couch

import (
    "bytes"
    "fmt"
    "os"
    "json"
    "http"
    "net"
    "io/ioutil"
)

const (
    Id  = "_id"
    Rev = "_rev"
)

var (
    defaultHeaders = map[string]string{}
)

//
// Helper and utility functions (private)
//

// Converts given URL to string containing the body of the response.
func url_to_string(url string) string {
    if r, _, err := http.Get(url); err == nil {
        b, err := ioutil.ReadAll(r.Body)
        r.Body.Close()
        if err == nil {
            return string(b)
        }
    }
    return ""
}

// Marshal given interface to JSON string
func to_JSON(p interface{}) (result string, err os.Error) {
    err = nil
    result = ""
    if buf, err := json.Marshal(p); err == nil {
        result = string(buf)
    }
    return
}

// Unmarshal JSON string to given interface
func from_JSON(s string, p interface{}) (err os.Error) {
    err = json.Unmarshal([]byte(s), p)
    return
}

type IdAndRev struct {
    Id  string "_id"
    Rev string "_rev"
}

// Simply extract id and rev from a given JSON string (typically a document)
func extract_id_and_rev(json_str string) (string, string, os.Error) {
    id_rev := new(IdAndRev)
    if err := from_JSON(json_str, id_rev); err != nil {
        return "", "", err
    }
    return id_rev.Id, id_rev.Rev, nil
}

/*
 * Sends a query to CouchDB and parses the response back.
 *
 * method: the name of the HTTP method (POST, PUT,...)
 * url: the URL to interact with
 * headers: additional headers to pass to the request
 * in: body of the request
 * out: a structure to fill in with the returned JSON document
 */
func (p Database) interact(method, url string, headers map[string]string, in *[]byte, out interface{}) (int, os.Error) {
    fullHeaders := map[string]string{}
    for k, v := range headers {
        fullHeaders[k] = v
    }

    bodyLength := 0
    if in != nil {
        bodyLength = len(*in)
        fullHeaders["Content-Type"] = "application/json"
    }
    req := http.Request{
        Method:        method,
        ProtoMajor:    1,
        ProtoMinor:    1,
        Close:         true,
        ContentLength: int64(bodyLength),
        Header:        fullHeaders,
    }

    req.TransferEncoding = []string{"chunked"}
    req.URL, _ = http.ParseURL(url)
    if in != nil {
        req.Body = &buffer{bytes.NewBuffer(*in)}
    }

    // Make connection
    conn, err := net.Dial("tcp", "", fmt.Sprintf("%s:%s", p.Host, p.Port))
    if err != nil {
        return 0, err
    }
    http_conn := http.NewClientConn(conn, nil)
    defer http_conn.Close()
    if err := http_conn.Write(&req); err != nil {
        return 0, err
    }

    // Read response
    r, err := http_conn.Read()
    if err != nil {
        return 0, err
    }
    if r.StatusCode < 200 || r.StatusCode >= 300 {
        b := []byte{}
        r.Body.Read(b)
        fmt.Printf("%v\n", bytes.NewBuffer(b).String())
        return r.StatusCode, os.NewError("server said: " + r.Status)
    }

    decoder := json.NewDecoder(r.Body)
    if err = decoder.Decode(out); err != nil {
        return 0, err
    }
    r.Body.Close()

    return r.StatusCode, nil
}

func (p Database) create_database() os.Error {
    ir := response{}
    if _, err := p.interact("PUT", p.DBURL(), defaultHeaders, nil, &ir); err != nil {
        return err
    }
    if !ir.Ok {
        return os.NewError(fmt.Sprintf("%s: %s", ir.Error, ir.Reason))
    }
    return nil
}

type buffer struct {
    b *bytes.Buffer
}

func (b *buffer) Read(out []byte) (int, os.Error) {
    return b.b.Read(out)
}

func (b *buffer) Close() os.Error { return nil }

//
// Database object + public methods
//

type Database struct {
    Host string
    Port string
    Name string
}

func (p Database) BaseURL() string {
    return fmt.Sprintf("http://%s:%s", p.Host, p.Port)
}

func (p Database) DBURL() string {
    return fmt.Sprintf("%s/%s", p.BaseURL(), p.Name)
}

// Test whether CouchDB is running (ignores Database.Name)
func (p Database) Running() bool {
    url := fmt.Sprintf("%s/%s", p.BaseURL(), "_all_dbs")
    s := url_to_string(url)
    if len(s) > 0 {
        return true
    }
    return false
}

type DatabaseInfo struct {
    Db_name string
    // other stuff too, ignore for now
}

// Test whether specified database exists in specified CouchDB instance
func (p Database) Exists() bool {
    di := new(DatabaseInfo)
    if err := from_JSON(url_to_string(p.DBURL()), di); err != nil {
        return false
    }
    if di.Db_name != p.Name {
        return false
    }
    return true
}

func NewDatabase(host, port, name string) (Database, os.Error) {
    db := Database{host, port, name}
    if !db.Running() {
        return db, os.NewError("CouchDB not running")
    }
    if !db.Exists() {
        if err := db.create_database(); err != nil {
            return db, err
        }
    }
    return db, nil
}

func clean_JSON(d interface{}, id, rev string) (json_buf []byte, err os.Error) {
    json_buf, err = json.Marshal(d)
    if err != nil {
        return
    }
    tmp := map[string]interface{}{}
    err = json.Unmarshal(json_buf, &tmp)
    if err != nil {
        return
    }
    if id != "" {
        tmp[Id] = id
    }
    if rev != "" {
        tmp[Rev] = rev
    }
    json_buf, err = json.Marshal(tmp)
    return
}

type response struct {
    Ok     bool
    Id     string
    Rev    string
    Error  string
    Reason string
}

// Inserts document to CouchDB, returning id and rev on success.
func (p Database) Insert(d interface{}) (string, string, os.Error) {
    json_buf, err := json.Marshal(d) //clean_JSON(d, "", "")
    if err != nil {
        return "", "", err
    }
    ir := response{}
    if _, err = p.interact("POST", p.DBURL(), defaultHeaders, &json_buf, &ir); err != nil {
        return "", "", err
    }
    if !ir.Ok {
        return "", "", os.NewError(fmt.Sprintf("%s: %s", ir.Error, ir.Reason))
    }
    return ir.Id, ir.Rev, nil
}

// Edits the given document, which must specify both Id and Rev fields, and
// returns the new revision.
func (p Database) Edit(d interface{}) (string, os.Error) {
    json_buf, err := json.Marshal(d) //clean_JSON(d, "", "")
    if err != nil {
        return "", err
    }
    id_rev := new(IdAndRev)
    err = json.Unmarshal(json_buf, id_rev)
    if err != nil {
        return "", err
    }
    if id_rev.Id == "" {
        return "", os.NewError("Id not specified in interface")
    }
    url := fmt.Sprintf("%s/%s", p.DBURL(), http.URLEscape(id_rev.Id))
    ir := response{}
    if _, err = p.interact("PUT", url, defaultHeaders, &json_buf, &ir); err != nil {
        return "", err
    }
    return ir.Rev, nil
}

type RetrieveError struct {
    Error  string
    Reason string
}

// Unmarshals the document matching id to the given interface, returning rev.
func (p Database) Retrieve(id string, d interface{}) (string, os.Error) {
    if id == "" {
        return "", os.NewError("no id specified")
    }
    json_str := url_to_string(fmt.Sprintf("%s/%s", p.DBURL(), id))
    retrieved_id, rev, err := extract_id_and_rev(json_str)
    if err != nil {
        return "", err
    }
    if retrieved_id != id {
        return "", os.NewError("invalid id specified")
    }
    return rev, from_JSON(json_str, d)
}

// Deletes document given by id and rev.
func (p Database) Delete(id, rev string) os.Error {
    headers := map[string]string{
        "If-Match": rev,
    }
    url := fmt.Sprintf("%s/%s", p.DBURL(), id)

    ir := response{}
    if _, err := p.interact("DELETE", url, headers, nil, &ir); err != nil {
        return err
    }
    if !ir.Ok {
        return os.NewError(fmt.Sprintf("%s: %s", ir.Error, ir.Reason))
    }
    return nil
}

type Row struct {
    Id  string
    Key string
}

type KeyedViewResponse struct {
    Total_rows uint64
    Offset     uint64
    Rows       []Row
}

// Return array of document ids as returned by the given view/options combo.
// view should be eg. "_design/my_foo/_view/my_bar"
// options should be eg. { "limit": 10, "key": "baz" }
func (p Database) Query(view string, options map[string]interface{}) ([]string, os.Error) {
    if view == "" {
        return make([]string, 0), os.NewError("empty view")
    }

    parameters := ""
    for k, v := range options {
        switch t := v.(type) {
        case string:
            parameters += fmt.Sprintf(`%s="%s"&`, k, http.URLEscape(t))
        case int:
            parameters += fmt.Sprintf(`%s=%d&`, k, t)
        case bool:
            parameters += fmt.Sprintf(`%s=%v&`, k, t)
        default:
            // TODO more types are supported
            panic(fmt.Sprintf("unsupported value-type %T in Query", t))
        }
    }
    full_url := fmt.Sprintf("%s/%s?%s", p.DBURL(), view, parameters)
    json_str := url_to_string(full_url)
    kvr := new(KeyedViewResponse)
    if err := from_JSON(json_str, kvr); err != nil {
        return make([]string, 0), err
    }

    ids := make([]string, len(kvr.Rows))
    for i, row := range kvr.Rows {
        ids[i] = row.Id
    }
    return ids, nil
}
<commit_msg>Remove dead code for now<commit_after>// -*- tab-width: 4 -*-
package couch

import (
    "bytes"
    "fmt"
    "os"
    "json"
    "http"
    "net"
    "io/ioutil"
)

const (
    Id  = "_id"
    Rev = "_rev"
)

var (
    defaultHeaders = map[string]string{}
)

//
// Helper and utility functions (private)
//

// Converts given URL to string containing the body of the response.
func url_to_string(url string) string {
    if r, _, err := http.Get(url); err == nil {
        b, err := ioutil.ReadAll(r.Body)
        r.Body.Close()
        if err == nil {
            return string(b)
        }
    }
    return ""
}

// Marshal given interface to JSON string
func to_JSON(p interface{}) (result string, err os.Error) {
    err = nil
    result = ""
    if buf, err := json.Marshal(p); err == nil {
        result = string(buf)
    }
    return
}

// Unmarshal JSON string to given interface
func from_JSON(s string, p interface{}) (err os.Error) {
    err = json.Unmarshal([]byte(s), p)
    return
}

type IdAndRev struct {
    Id  string "_id"
    Rev string "_rev"
}

// Simply extract id and rev from a given JSON string (typically a document)
func extract_id_and_rev(json_str string) (string, string, os.Error) {
    id_rev := new(IdAndRev)
    if err := from_JSON(json_str, id_rev); err != nil {
        return "", "", err
    }
    return id_rev.Id, id_rev.Rev, nil
}

/*
 * Sends a query to CouchDB and parses the response back.
 *
 * method: the name of the HTTP method (POST, PUT,...)
 * url: the URL to interact with
 * headers: additional headers to pass to the request
 * in: body of the request
 * out: a structure to fill in with the returned JSON document
 */
func (p Database) interact(method, url string, headers map[string]string, in *[]byte, out interface{}) (int, os.Error) {
    fullHeaders := map[string]string{}
    for k, v := range headers {
        fullHeaders[k] = v
    }

    bodyLength := 0
    if in != nil {
        bodyLength = len(*in)
        fullHeaders["Content-Type"] = "application/json"
    }
    req := http.Request{
        Method:        method,
        ProtoMajor:    1,
        ProtoMinor:    1,
        Close:         true,
        ContentLength: int64(bodyLength),
        Header:        fullHeaders,
    }

    req.TransferEncoding = []string{"chunked"}
    req.URL, _ = http.ParseURL(url)
    if in != nil {
        req.Body = &buffer{bytes.NewBuffer(*in)}
    }

    // Make connection
    conn, err := net.Dial("tcp", "", fmt.Sprintf("%s:%s", p.Host, p.Port))
    if err != nil {
        return 0, err
    }
    http_conn := http.NewClientConn(conn, nil)
    defer http_conn.Close()
    if err := http_conn.Write(&req); err != nil {
        return 0, err
    }

    // Read response
    r, err := http_conn.Read()
    if err != nil {
        return 0, err
    }
    if r.StatusCode < 200 || r.StatusCode >= 300 {
        b := []byte{}
        r.Body.Read(b)
        fmt.Printf("%v\n", bytes.NewBuffer(b).String())
        return r.StatusCode, os.NewError("server said: " + r.Status)
    }

    decoder := json.NewDecoder(r.Body)
    if err = decoder.Decode(out); err != nil {
        return 0, err
    }
    r.Body.Close()

    return r.StatusCode, nil
}

func (p Database) create_database() os.Error {
    ir := response{}
    if _, err := p.interact("PUT", p.DBURL(), defaultHeaders, nil, &ir); err != nil {
        return err
    }
    if !ir.Ok {
        return os.NewError(fmt.Sprintf("%s: %s", ir.Error, ir.Reason))
    }
    return nil
}

type buffer struct {
    b *bytes.Buffer
}

func (b *buffer) Read(out []byte) (int, os.Error) {
    return b.b.Read(out)
}

func (b *buffer) Close() os.Error { return nil }

//
// Database object + public methods
//

type Database struct {
    Host string
    Port string
    Name string
}

func (p Database) BaseURL() string {
    return fmt.Sprintf("http://%s:%s", p.Host, p.Port)
}

func (p Database) DBURL() string {
    return fmt.Sprintf("%s/%s", p.BaseURL(), p.Name)
}

// Test whether CouchDB is running (ignores Database.Name)
func (p Database) Running() bool {
    url := fmt.Sprintf("%s/%s", p.BaseURL(), "_all_dbs")
    s := url_to_string(url)
    if len(s) > 0 {
        return true
    }
    return false
}

type DatabaseInfo struct {
    Db_name string
    // other stuff too, ignore for now
}

// Test whether specified database exists in specified CouchDB instance
func (p Database) Exists() bool {
    di := new(DatabaseInfo)
    if err := from_JSON(url_to_string(p.DBURL()), di); err != nil {
        return false
    }
    if di.Db_name != p.Name {
        return false
    }
    return true
}

func NewDatabase(host, port, name string) (Database, os.Error) {
    db := Database{host, port, name}
    if !db.Running() {
        return db, os.NewError("CouchDB not running")
    }
    if !db.Exists() {
        if err := db.create_database(); err != nil {
            return db, err
        }
    }
    return db, nil
}

type response struct {
    Ok     bool
    Id     string
    Rev    string
    Error  string
    Reason string
}

// Inserts document to CouchDB, returning id and rev on success.
func (p Database) Insert(d interface{}) (string, string, os.Error) {
    json_buf, err := json.Marshal(d)
    if err != nil {
        return "", "", err
    }
    ir := response{}
    if _, err = p.interact("POST", p.DBURL(), defaultHeaders, &json_buf, &ir); err != nil {
        return "", "", err
    }
    if !ir.Ok {
        return "", "", os.NewError(fmt.Sprintf("%s: %s", ir.Error, ir.Reason))
    }
    return ir.Id, ir.Rev, nil
}

// Edits the given document, which must specify both Id and Rev fields, and
// returns the new revision.
func (p Database) Edit(d interface{}) (string, os.Error) {
    json_buf, err := json.Marshal(d)
    if err != nil {
        return "", err
    }
    id_rev := new(IdAndRev)
    err = json.Unmarshal(json_buf, id_rev)
    if err != nil {
        return "", err
    }
    if id_rev.Id == "" {
        return "", os.NewError("Id not specified in interface")
    }
    url := fmt.Sprintf("%s/%s", p.DBURL(), http.URLEscape(id_rev.Id))
    ir := response{}
    if _, err = p.interact("PUT", url, defaultHeaders, &json_buf, &ir); err != nil {
        return "", err
    }
    return ir.Rev, nil
}

type RetrieveError struct {
    Error  string
    Reason string
}

// Unmarshals the document matching id to the given interface, returning rev.
func (p Database) Retrieve(id string, d interface{}) (string, os.Error) {
    if id == "" {
        return "", os.NewError("no id specified")
    }
    json_str := url_to_string(fmt.Sprintf("%s/%s", p.DBURL(), id))
    retrieved_id, rev, err := extract_id_and_rev(json_str)
    if err != nil {
        return "", err
    }
    if retrieved_id != id {
        return "", os.NewError("invalid id specified")
    }
    return rev, from_JSON(json_str, d)
}

// Deletes document given by id and rev.
func (p Database) Delete(id, rev string) os.Error {
    headers := map[string]string{
        "If-Match": rev,
    }
    url := fmt.Sprintf("%s/%s", p.DBURL(), id)

    ir := response{}
    if _, err := p.interact("DELETE", url, headers, nil, &ir); err != nil {
        return err
    }
    if !ir.Ok {
        return os.NewError(fmt.Sprintf("%s: %s", ir.Error, ir.Reason))
    }
    return nil
}

type Row struct {
    Id  string
    Key string
}

type KeyedViewResponse struct {
    Total_rows uint64
    Offset     uint64
    Rows       []Row
}

// Return array of document ids as returned by the given view/options combo.
// view should be eg. "_design/my_foo/_view/my_bar"
// options should be eg. { "limit": 10, "key": "baz" }
func (p Database) Query(view string, options map[string]interface{}) ([]string, os.Error) {
    if view == "" {
        return make([]string, 0), os.NewError("empty view")
    }

    parameters := ""
    for k, v := range options {
        switch t := v.(type) {
        case string:
            parameters += fmt.Sprintf(`%s="%s"&`, k, http.URLEscape(t))
        case int:
            parameters += fmt.Sprintf(`%s=%d&`, k, t)
        case bool:
            parameters += fmt.Sprintf(`%s=%v&`, k, t)
        default:
            // TODO more types are supported
            panic(fmt.Sprintf("unsupported value-type %T in Query", t))
        }
    }
    full_url := fmt.Sprintf("%s/%s?%s", p.DBURL(), view, parameters)
    json_str := url_to_string(full_url)
    kvr := new(KeyedViewResponse)
    if err := from_JSON(json_str, kvr); err != nil {
        return make([]string, 0), err
    }

    ids := make([]string, len(kvr.Rows))
    for i, row := range kvr.Rows {
        ids[i] = row.Id
    }
    return ids, nil
}
<|endoftext|>


<commit_before>package main

import (
	tiff "github.com/garyhouston/tiff66"
	"io/ioutil"
	"log"
	"os"
)

// Decode a TIFF file, then re-encode it and write to a new file.
func main() {
	logger := log.New(os.Stderr, "", 0)
	if len(os.Args) != 3 {
		logger.Fatalf("Usage: %s file outfile\n", os.Args[0])
	}
	buf, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		logger.Fatal(err)
	}
	valid, order, ifdPos := tiff.GetHeader(buf)
	if !valid {
		logger.Fatal("Not a valid TIFF file")
	}
	root, err := tiff.GetIFDTree(buf, order, ifdPos, tiff.TIFFSpace)
	if err != nil {
		logger.Print(err)
		logger.Print("Error(s) occurred during decoding, but will repack anyway.")
	}
	root.Fix()
	fileSize := tiff.HeaderSize + root.TreeSize()
	out := make([]byte, fileSize)
	tiff.PutHeader(out, order, tiff.HeaderSize)
	next, err := root.PutIFDTree(out, tiff.HeaderSize)
	if err != nil {
		logger.Fatal(err)
	}
	out = out[:next]
	if err = ioutil.WriteFile(os.Args[2], out, 0644); err != nil {
		logger.Fatal(err)
	}
}
<commit_msg>Remove empty IFDs before writing.<commit_after>package main

import (
	tiff "github.com/garyhouston/tiff66"
	"io/ioutil"
	"log"
	"os"
)

// Decode a TIFF file, then re-encode it and write to a new file.
func main() {
	logger := log.New(os.Stderr, "", 0)
	if len(os.Args) != 3 {
		logger.Fatalf("Usage: %s file outfile\n", os.Args[0])
	}
	buf, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		logger.Fatal(err)
	}
	valid, order, ifdPos := tiff.GetHeader(buf)
	if !valid {
		logger.Fatal("Not a valid TIFF file")
	}
	root, err := tiff.GetIFDTree(buf, order, ifdPos, tiff.TIFFSpace)
	if err != nil {
		logger.Print(err)
		logger.Print("Error(s) occurred during decoding, but will repack anyway.")
	}
	root.Fix()
	root = root.DeleteEmptyIFDs()
	if root == nil {
		logger.Fatal("Output TIFF file would have no fields; invalid according to TIFF spec.")
	}
	fileSize := tiff.HeaderSize + root.TreeSize()
	out := make([]byte, fileSize)
	tiff.PutHeader(out, order, tiff.HeaderSize)
	next, err := root.PutIFDTree(out, tiff.HeaderSize)
	if err != nil {
		logger.Fatal(err)
	}
	out = out[:next]
	if err = ioutil.WriteFile(os.Args[2], out, 0644); err != nil {
		logger.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

var profile string // The profile to read; the value of -html or -func
<commit_msg>remove unsed file.<commit_after><|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/go-uuid/uuid"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
)

// @author Robin Verlangen

type Cmd struct {
	Command string
	Pending bool
	Id string
	Timeout int // in seconds
}

// Execute
func (c *Cmd) Execute() {
	log.Printf("Executing %s: %s", c.Id, c.Command)

	// File contents
	var fileBytes bytes.Buffer
	fileBytes.WriteString("#!/bin/bash\n")
	fileBytes.WriteString(c.Command)

	// Write tmp file
	tmpFileName := fmt.Sprintf("/tmp/indispenso_%s", c.Id)
	ioutil.WriteFile(tmpFileName, fileBytes.Bytes(), 0644)

	// Run file
	cmd := exec.Command("bash", "-c", tmpFileName)
	err := cmd.Start()
	if err != nil {
		log.Printf("Failed to run command: %s", err)
	}

	// Wait for completion
	cmd.Wait()
	log.Printf("Finished %s", c.Id)

	// Remove file
	os.Remove(tmpFileName)
}

func newCmd(command string, timeout int) *Cmd {
	return &Cmd{
		Id : uuid.New(),
		Command: command,
		Pending: true,
		Timeout: timeout,
	}
}<commit_msg>Fix exec<commit_after>package main

import (
	"code.google.com/p/go-uuid/uuid"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
)

// @author Robin Verlangen

type Cmd struct {
	Command string
	Pending bool
	Id string
	Timeout int // in seconds
}

// Execute
func (c *Cmd) Execute() {
	log.Printf("Executing %s: %s", c.Id, c.Command)

	// File contents
	var fileBytes bytes.Buffer
	fileBytes.WriteString("#!/bin/bash\n")
	fileBytes.WriteString(c.Command)

	// Write tmp file
	tmpFileName := fmt.Sprintf("/tmp/indispenso_%s", c.Id)
	ioutil.WriteFile(tmpFileName, fileBytes.Bytes(), 0644)

	// Run file
	cmd := exec.Command("bash", tmpFileName)
	err := cmd.Start()
	if err != nil {
		log.Printf("Failed to run command: %s", err)
	}

	// Wait for completion
	cmd.Wait()
	log.Printf("Finished %s", c.Id)

	// Remove file
	os.Remove(tmpFileName)
}

func newCmd(command string, timeout int) *Cmd {
	return &Cmd{
		Id : uuid.New(),
		Command: command,
		Pending: true,
		Timeout: timeout,
	}
}<|endoftext|>


<commit_before>package gps

import (
	"bytes"
	"context"
	"fmt"
	"os/exec"
	"sync"
	"time"

	"github.com/Masterminds/vcs"
)

// monitoredCmd wraps a cmd and will keep monitoring the process until it
// finishes, the provided context is canceled, or a certain amount of time has
// passed and the command showed no signs of activity.
type monitoredCmd struct {
	cmd     *exec.Cmd
	timeout time.Duration
	ctx     context.Context
	stdout  *activityBuffer
	stderr  *activityBuffer
}

func newMonitoredCmd(cmd *exec.Cmd, timeout time.Duration) *monitoredCmd {
	stdout, stderr := newActivityBuffer(), newActivityBuffer()
	cmd.Stdout, cmd.Stderr = stdout, stderr
	return &monitoredCmd{
		cmd:     cmd,
		timeout: timeout,
		stdout:  stdout,
		stderr:  stderr,
	}
}

// run will wait for the command to finish and return the error, if any. If the
// command does not show any activity for more than the specified timeout the
// process will be killed.
func (c *monitoredCmd) run(ctx context.Context) error {
	// Check for cancellation before even starting
	if ctx.Err() != nil {
		return ctx.Err()
	}

	ticker := time.NewTicker(c.timeout)
	done := make(chan error, 1)
	defer ticker.Stop()
	go func() { done <- c.cmd.Run() }()

	for {
		select {
		case <-ticker.C:
			if c.hasTimedOut() {
				// On windows it is apparently (?) possible for the process
				// pointer to become nil without Run() having returned (and
				// thus, passing through the done channel). Guard against this.
				if c.cmd.Process != nil {
					if err := c.cmd.Process.Kill(); err != nil {
						return &killCmdError{err}
					}
				}

				return &timeoutError{c.timeout}
			}
		case <-ctx.Done():
			if err := c.cmd.Process.Kill(); err != nil {
				return &killCmdError{err}
			}
			return c.ctx.Err()
		case err := <-done:
			return err
		}
	}
}

func (c *monitoredCmd) hasTimedOut() bool {
	t := time.Now().Add(-c.timeout)
	return c.stderr.lastActivity().Before(t) &&
		c.stdout.lastActivity().Before(t)
}

func (c *monitoredCmd) combinedOutput(ctx context.Context) ([]byte, error) {
	if err := c.run(ctx); err != nil {
		return c.stderr.buf.Bytes(), err
	}

	return c.stdout.buf.Bytes(), nil
}

// activityBuffer is a buffer that keeps track of the last time a Write
// operation was performed on it.
type activityBuffer struct {
	sync.Mutex
	buf               *bytes.Buffer
	lastActivityStamp time.Time
}

func newActivityBuffer() *activityBuffer {
	return &activityBuffer{
		buf: bytes.NewBuffer(nil),
	}
}

func (b *activityBuffer) Write(p []byte) (int, error) {
	b.Lock()
	b.lastActivityStamp = time.Now()
	defer b.Unlock()
	return b.buf.Write(p)
}

func (b *activityBuffer) lastActivity() time.Time {
	b.Lock()
	defer b.Unlock()
	return b.lastActivityStamp
}

type timeoutError struct {
	timeout time.Duration
}

func (e timeoutError) Error() string {
	return fmt.Sprintf("command killed after %s of no activity", e.timeout)
}

type killCmdError struct {
	err error
}

func (e killCmdError) Error() string {
	return fmt.Sprintf("error killing command: %s", e.err)
}

func runFromCwd(ctx context.Context, cmd string, args ...string) ([]byte, error) {
	c := newMonitoredCmd(exec.Command(cmd, args...), 2*time.Minute)
	return c.combinedOutput(ctx)
}

func runFromRepoDir(ctx context.Context, repo vcs.Repo, cmd string, args ...string) ([]byte, error) {
	c := newMonitoredCmd(repo.CmdFromDir(cmd, args...), 2*time.Minute)
	return c.combinedOutput(ctx)
}
<commit_msg>Check if Process non-nil before calling Kill<commit_after>package gps

import (
	"bytes"
	"context"
	"fmt"
	"os/exec"
	"sync"
	"time"

	"github.com/Masterminds/vcs"
)

// monitoredCmd wraps a cmd and will keep monitoring the process until it
// finishes, the provided context is canceled, or a certain amount of time has
// passed and the command showed no signs of activity.
type monitoredCmd struct {
	cmd     *exec.Cmd
	timeout time.Duration
	ctx     context.Context
	stdout  *activityBuffer
	stderr  *activityBuffer
}

func newMonitoredCmd(cmd *exec.Cmd, timeout time.Duration) *monitoredCmd {
	stdout, stderr := newActivityBuffer(), newActivityBuffer()
	cmd.Stdout, cmd.Stderr = stdout, stderr
	return &monitoredCmd{
		cmd:     cmd,
		timeout: timeout,
		stdout:  stdout,
		stderr:  stderr,
	}
}

// run will wait for the command to finish and return the error, if any. If the
// command does not show any activity for more than the specified timeout the
// process will be killed.
func (c *monitoredCmd) run(ctx context.Context) error {
	// Check for cancellation before even starting
	if ctx.Err() != nil {
		return ctx.Err()
	}

	ticker := time.NewTicker(c.timeout)
	done := make(chan error, 1)
	defer ticker.Stop()
	go func() { done <- c.cmd.Run() }()

	for {
		select {
		case <-ticker.C:
			if c.hasTimedOut() {
				// On windows it is apparently (?) possible for the process
				// pointer to become nil without Run() having returned (and
				// thus, passing through the done channel). Guard against this.
				if c.cmd.Process != nil {
					if err := c.cmd.Process.Kill(); err != nil {
						return &killCmdError{err}
					}
				}

				return &timeoutError{c.timeout}
			}
		case <-ctx.Done():
			if c.cmd.Process != nil {
				if err := c.cmd.Process.Kill(); err != nil {
					return &killCmdError{err}
				}
			}
			return c.ctx.Err()
		case err := <-done:
			return err
		}
	}
}

func (c *monitoredCmd) hasTimedOut() bool {
	t := time.Now().Add(-c.timeout)
	return c.stderr.lastActivity().Before(t) &&
		c.stdout.lastActivity().Before(t)
}

func (c *monitoredCmd) combinedOutput(ctx context.Context) ([]byte, error) {
	if err := c.run(ctx); err != nil {
		return c.stderr.buf.Bytes(), err
	}

	return c.stdout.buf.Bytes(), nil
}

// activityBuffer is a buffer that keeps track of the last time a Write
// operation was performed on it.
type activityBuffer struct {
	sync.Mutex
	buf               *bytes.Buffer
	lastActivityStamp time.Time
}

func newActivityBuffer() *activityBuffer {
	return &activityBuffer{
		buf: bytes.NewBuffer(nil),
	}
}

func (b *activityBuffer) Write(p []byte) (int, error) {
	b.Lock()
	b.lastActivityStamp = time.Now()
	defer b.Unlock()
	return b.buf.Write(p)
}

func (b *activityBuffer) lastActivity() time.Time {
	b.Lock()
	defer b.Unlock()
	return b.lastActivityStamp
}

type timeoutError struct {
	timeout time.Duration
}

func (e timeoutError) Error() string {
	return fmt.Sprintf("command killed after %s of no activity", e.timeout)
}

type killCmdError struct {
	err error
}

func (e killCmdError) Error() string {
	return fmt.Sprintf("error killing command: %s", e.err)
}

func runFromCwd(ctx context.Context, cmd string, args ...string) ([]byte, error) {
	c := newMonitoredCmd(exec.Command(cmd, args...), 2*time.Minute)
	return c.combinedOutput(ctx)
}

func runFromRepoDir(ctx context.Context, repo vcs.Repo, cmd string, args ...string) ([]byte, error) {
	c := newMonitoredCmd(repo.CmdFromDir(cmd, args...), 2*time.Minute)
	return c.combinedOutput(ctx)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"

	"github.com/remind101/empire/cli/pkg/plugin"
)

var pluginDeploy = plugin.Plugin{
	Name:   "deploy",
	Action: runDeploy,
}

type ImageParts struct {
	Repo string `json:"repo"`
	ID   string `json:"id"`
}

type Image struct {
	Image *ImageParts `json:"image"`
}

func runDeploy(c *plugin.Context) {
	repo, id := strings.Split(c.Args[0], ":")[0], strings.Split(c.Args[0], ":")[1]
	image := &Image{&ImageParts{repo, id}}

	err := c.Client.Post(nil, "/deploys", image)
	if err != nil {
		fmt.Printf("Failed to deploy %s:%s\n", repo, id)
		plugin.Must(err)
	}

	fmt.Printf("Deployed %s:%s\n", repo, id)
}
<commit_msg>Better naming.<commit_after>package main

import (
	"fmt"
	"strings"

	"github.com/remind101/empire/cli/pkg/plugin"
)

var pluginDeploy = plugin.Plugin{
	Name:   "deploy",
	Action: runDeploy,
}

type Image struct {
	Repo string `json:"repo"`
	ID   string `json:"id"`
}

type PostDeployForm struct {
	Image *Image `json:"image"`
}

func runDeploy(c *plugin.Context) {
	repo, id := strings.Split(c.Args[0], ":")[0], strings.Split(c.Args[0], ":")[1]
	form := &PostDeployForm{&Image{repo, id}}

	err := c.Client.Post(nil, "/deploys", form)
	if err != nil {
		fmt.Printf("Failed to deploy %s:%s\n", repo, id)
		plugin.Must(err)
	}

	fmt.Printf("Deployed %s:%s\n", repo, id)
}
<|endoftext|>