<commit_before>package qiniu

import (
	"encoding/base64"
	"github.com/oxfeeefeee/appgo"
	qndigest "github.com/qiniu/api.v6/auth/digest"
	"github.com/qiniu/api.v6/conf"
	qnio "github.com/qiniu/api.v6/io"
	"github.com/qiniu/api.v6/rs"
	qnurl "github.com/qiniu/api.v6/url"
	"io"
	"strconv"
	"strings"
)

var (
	bucketName string
	domain     string
	useHttps   bool
	putPolicy  rs.PutPolicy
	getPolicy  rs.GetPolicy
)

func init() {
	params := appgo.Conf.Qiniu
	conf.ACCESS_KEY = params.AccessKey
	conf.SECRET_KEY = params.Secret
	bucketName = params.Bucket
	domain = params.Domain
	useHttps = params.UseHttps
	putPolicy = rs.PutPolicy{
		Expires:    uint32(params.DefaultExpires),
		InsertOnly: 1,
	}
	getPolicy = rs.GetPolicy{
		Expires: uint32(params.DefaultExpires),
	}
}

func Domain() string {
	return domain
}

func GetUrl(key string, expiryUnix int64, params string) string {
	baseUrl := makeBaseUrl(key)
	url := baseUrl + "?" + params
	if expiryUnix == 0 {
		return getPolicy.MakeRequest(url, nil)
	}
	return makeRequest(url, expiryUnix, nil)
}

func PublicGetUrl(key string) string {
	return makeBaseUrl(key)
}

func NoKeyToken() string {
	putPolicy.Scope = bucketName
	return putPolicy.Token(nil)
}

func PutToken(key string) string {
	putPolicy.Scope = bucketName + ":" + key
	return putPolicy.Token(nil)
}

func PutFile(key string, r io.Reader, size int64) (string, error) {
	putPolicy.Scope = bucketName + ":" + key
	token := putPolicy.Token(nil)
	//var ret qnio.PutRet
	if err := qnio.Put2(nil, nil /*&ret*/, token, key, r, size, nil); err != nil {
		return "", err
	} else {
		return makeBaseUrl(key), nil
	}
}

func FetchToken(url, key string) (string, string) {
	encodedUrl := base64.StdEncoding.EncodeToString([]byte(url))
	encodedTo := base64.StdEncoding.EncodeToString([]byte(bucketName + ":" + key))
	fullPath := "/fetch/" + encodedUrl + "/to/" + encodedTo
	token := qndigest.Sign(nil, []byte(fullPath))
	return fullPath, token
}

func makeBaseUrl(key string) string {
	//return rs.MakeBaseUrl(domain, key)
	p := "http"
	if useHttps {
		p = "https"
	}
	return p + "://" + domain + "/" + qnurl.Escape(key)
}

func makeRequest(baseUrl string, expiryUnix int64, mac *qndigest.Mac) (privateUrl string) {
	if strings.Contains(baseUrl, "?") {
		baseUrl += "&e="
	} else {
		baseUrl += "?e="
	}
	baseUrl += strconv.FormatInt(expiryUnix, 10)

	token := qndigest.Sign(mac, []byte(baseUrl))
	return baseUrl + "&token=" + token
}
<commit_msg>qiniu fetch url support<commit_after>package qiniu

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"github.com/oxfeeefeee/appgo"
	qndigest "github.com/qiniu/api.v6/auth/digest"
	"github.com/qiniu/api.v6/conf"
	qnio "github.com/qiniu/api.v6/io"
	"github.com/qiniu/api.v6/rs"
	qnurl "github.com/qiniu/api.v6/url"
	"io"
	"strconv"
	"strings"
)

var (
	bucketName string
	domain     string
	useHttps   bool
	putPolicy  rs.PutPolicy
	getPolicy  rs.GetPolicy
)

func init() {
	params := appgo.Conf.Qiniu
	conf.ACCESS_KEY = params.AccessKey
	conf.SECRET_KEY = params.Secret
	bucketName = params.Bucket
	domain = params.Domain
	useHttps = params.UseHttps
	putPolicy = rs.PutPolicy{
		Expires:    uint32(params.DefaultExpires),
		InsertOnly: 1,
	}
	getPolicy = rs.GetPolicy{
		Expires: uint32(params.DefaultExpires),
	}
}

func Domain() string {
	return domain
}

func GetUrl(key string, expiryUnix int64, params string) string {
	baseUrl := makeBaseUrl(key)
	url := baseUrl + "?" + params
	if expiryUnix == 0 {
		return getPolicy.MakeRequest(url, nil)
	}
	return makeRequest(url, expiryUnix, nil)
}

func PublicGetUrl(key string) string {
	return makeBaseUrl(key)
}

func NoKeyToken() string {
	putPolicy.Scope = bucketName
	return putPolicy.Token(nil)
}

func PutToken(key string) string {
	putPolicy.Scope = bucketName + ":" + key
	return putPolicy.Token(nil)
}

func PutFile(key string, r io.Reader, size int64) (string, error) {
	putPolicy.Scope = bucketName + ":" + key
	token := putPolicy.Token(nil)
	//var ret qnio.PutRet
	if err := qnio.Put2(nil, nil /*&ret*/, token, key, r, size, nil); err != nil {
		return "", err
	} else {
		return makeBaseUrl(key), nil
	}
}

func FetchToken(url, key string) (string, string, string) {
	encodedUrl := base64.URLEncoding.EncodeToString([]byte(url))
	encodedTo := base64.URLEncoding.EncodeToString([]byte(bucketName + ":" + key))
	fullPath := "/fetch/" + encodedUrl + "/to/" + encodedTo

	h := hmac.New(sha1.New, []byte(conf.SECRET_KEY))
	io.WriteString(h, fullPath+"\n")
	sign := base64.URLEncoding.EncodeToString(h.Sum(nil))
	token := conf.ACCESS_KEY + ":" + sign
	return fullPath, makeBaseUrl(key), token
}

func makeBaseUrl(key string) string {
	//return rs.MakeBaseUrl(domain, key)
	p := "http"
	if useHttps {
		p = "https"
	}
	return p + "://" + domain + "/" + qnurl.Escape(key)
}

func makeRequest(baseUrl string, expiryUnix int64, mac *qndigest.Mac) (privateUrl string) {
	if strings.Contains(baseUrl, "?") {
		baseUrl += "&e="
	} else {
		baseUrl += "?e="
	}
	baseUrl += strconv.FormatInt(expiryUnix, 10)

	token := qndigest.Sign(mac, []byte(baseUrl))
	return baseUrl + "&token=" + token
}
<|endoftext|>


<commit_before>package api

import (
	"context"
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/prometheus/client_golang/prometheus/promhttp"

	gocache "github.com/patrickmn/go-cache"
	macaron "gopkg.in/macaron.v1"

	"github.com/grafana/grafana/pkg/api/live"
	httpstatic "github.com/grafana/grafana/pkg/api/static"
	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/middleware"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/plugins"
	"github.com/grafana/grafana/pkg/setting"
)

type HttpServer struct {
	log           log.Logger
	macaron       *macaron.Macaron
	context       context.Context
	streamManager *live.StreamManager
	cache         *gocache.Cache

	httpSrv *http.Server
}

func NewHttpServer() *HttpServer {
	return &HttpServer{
		log:   log.New("http.server"),
		cache: gocache.New(5*time.Minute, 10*time.Minute),
	}
}

func (hs *HttpServer) Start(ctx context.Context) error {
	var err error

	hs.context = ctx
	hs.streamManager = live.NewStreamManager()
	hs.macaron = hs.newMacaron()
	hs.registerRoutes()

	hs.streamManager.Run(ctx)

	listenAddr := fmt.Sprintf("%s:%s", setting.HttpAddr, setting.HttpPort)
	hs.log.Info("Initializing HTTP Server", "address", listenAddr, "protocol", setting.Protocol, "subUrl", setting.AppSubUrl, "socket", setting.SocketPath)

	hs.httpSrv = &http.Server{Addr: listenAddr, Handler: hs.macaron}
	switch setting.Protocol {
	case setting.HTTP:
		err = hs.httpSrv.ListenAndServe()
		if err == http.ErrServerClosed {
			hs.log.Debug("server was shutdown gracefully")
			return nil
		}
	case setting.HTTPS:
		err = hs.listenAndServeTLS(setting.CertFile, setting.KeyFile)
		if err == http.ErrServerClosed {
			hs.log.Debug("server was shutdown gracefully")
			return nil
		}
	case setting.SOCKET:
		ln, err := net.Listen("unix", setting.SocketPath)
		if err != nil {
			hs.log.Debug("server was shutdown gracefully")
			return nil
		}

		err = hs.httpSrv.Serve(ln)
		if err != nil {
			hs.log.Debug("server was shutdown gracefully")
			return nil
		}
	default:
		hs.log.Error("Invalid protocol", "protocol", setting.Protocol)
		err = errors.New("Invalid Protocol")
	}

	return err
}

func (hs *HttpServer) Shutdown(ctx context.Context) error {
	err := hs.httpSrv.Shutdown(ctx)
	hs.log.Info("stopped http server")
	return err
}

func (hs *HttpServer) listenAndServeTLS(certfile, keyfile string) error {
	if certfile == "" {
		return fmt.Errorf("cert_file cannot be empty when using HTTPS")
	}

	if keyfile == "" {
		return fmt.Errorf("cert_key cannot be empty when using HTTPS")
	}

	if _, err := os.Stat(setting.CertFile); os.IsNotExist(err) {
		return fmt.Errorf(`Cannot find SSL cert_file at %v`, setting.CertFile)
	}

	if _, err := os.Stat(setting.KeyFile); os.IsNotExist(err) {
		return fmt.Errorf(`Cannot find SSL key_file at %v`, setting.KeyFile)
	}

	tlsCfg := &tls.Config{
		MinVersion:               tls.VersionTLS12,
		PreferServerCipherSuites: true,
		CipherSuites: []uint16{
			tls.TLS_RSA_WITH_AES_128_CBC_SHA,
			tls.TLS_RSA_WITH_AES_256_CBC_SHA,
			tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
		},
	}

	hs.httpSrv.TLSConfig = tlsCfg
	hs.httpSrv.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0)

	return hs.httpSrv.ListenAndServeTLS(setting.CertFile, setting.KeyFile)
}

func (hs *HttpServer) newMacaron() *macaron.Macaron {
	macaron.Env = setting.Env
	m := macaron.New()

	m.Use(middleware.Logger())
	m.Use(middleware.Recovery())

	if setting.EnableGzip {
		m.Use(middleware.Gziper())
	}

	for _, route := range plugins.StaticRoutes {
		pluginRoute := path.Join("/public/plugins/", route.PluginId)
		logger.Debug("Plugins: Adding route", "route", pluginRoute, "dir", route.Directory)
		hs.mapStatic(m, route.Directory, "", pluginRoute)
	}

	hs.mapStatic(m, setting.StaticRootPath, "", "public")
	hs.mapStatic(m, setting.StaticRootPath, "robots.txt", "robots.txt")

	m.Use(macaron.Renderer(macaron.RenderOptions{
		Directory:  path.Join(setting.StaticRootPath, "views"),
		IndentJSON: macaron.Env != macaron.PROD,
		Delims:     macaron.Delims{Left: "[[", Right: "]]"},
	}))

	m.Use(hs.healthHandler)
	m.Use(hs.metricsEndpoint)
	m.Use(middleware.GetContextHandler())
	m.Use(middleware.Sessioner(&setting.SessionOptions))
	m.Use(middleware.OrgRedirect())

	// needs to be after context handler
	if setting.EnforceDomain {
		m.Use(middleware.ValidateHostHeader(setting.Domain))
	}

	m.Use(middleware.AddDefaultResponseHeaders())

	return m
}

func (hs *HttpServer) metricsEndpoint(ctx *macaron.Context) {
	if ctx.Req.Method != "GET" || ctx.Req.URL.Path != "/metrics" {
		return
	}

	promhttp.Handler().ServeHTTP(ctx.Resp, ctx.Req.Request)
}

func (hs *HttpServer) healthHandler(ctx *macaron.Context) {
	if ctx.Req.Method != "GET" || ctx.Req.URL.Path != "/api/health" {
		return
	}

	data := simplejson.New()
	data.Set("database", "ok")
	data.Set("version", setting.BuildVersion)
	data.Set("commit", setting.BuildCommit)

	if err := bus.Dispatch(&models.GetDBHealthQuery{}); err != nil {
		data.Set("database", "failing")
		ctx.Resp.Header().Set("Content-Type", "application/json; charset=UTF-8")
		ctx.Resp.WriteHeader(503)
	} else {
		ctx.Resp.Header().Set("Content-Type", "application/json; charset=UTF-8")
		ctx.Resp.WriteHeader(200)
	}

	dataBytes, _ := data.EncodePretty()
	ctx.Resp.Write(dataBytes)
}

func (hs *HttpServer) mapStatic(m *macaron.Macaron, rootDir string, dir string, prefix string) {
	headers := func(c *macaron.Context) {
		c.Resp.Header().Set("Cache-Control", "public, max-age=3600")
	}

	if setting.Env == setting.DEV {
		headers = func(c *macaron.Context) {
			c.Resp.Header().Set("Cache-Control", "max-age=0, must-revalidate, no-cache")
		}
	}

	m.Use(httpstatic.Static(
		path.Join(rootDir, dir),
		httpstatic.StaticOptions{
			SkipLogging: true,
			Prefix:      prefix,
			AddHeaders:  headers,
		},
	))
}
<commit_msg>metrics: disable gzip for /metrics endpoint (#9468)<commit_after>package api

import (
	"context"
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/prometheus/client_golang/prometheus/promhttp"

	gocache "github.com/patrickmn/go-cache"
	macaron "gopkg.in/macaron.v1"

	"github.com/grafana/grafana/pkg/api/live"
	httpstatic "github.com/grafana/grafana/pkg/api/static"
	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/middleware"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/plugins"
	"github.com/grafana/grafana/pkg/setting"
)

type HttpServer struct {
	log           log.Logger
	macaron       *macaron.Macaron
	context       context.Context
	streamManager *live.StreamManager
	cache         *gocache.Cache

	httpSrv *http.Server
}

func NewHttpServer() *HttpServer {
	return &HttpServer{
		log:   log.New("http.server"),
		cache: gocache.New(5*time.Minute, 10*time.Minute),
	}
}

func (hs *HttpServer) Start(ctx context.Context) error {
	var err error

	hs.context = ctx
	hs.streamManager = live.NewStreamManager()
	hs.macaron = hs.newMacaron()
	hs.registerRoutes()

	hs.streamManager.Run(ctx)

	listenAddr := fmt.Sprintf("%s:%s", setting.HttpAddr, setting.HttpPort)
	hs.log.Info("Initializing HTTP Server", "address", listenAddr, "protocol", setting.Protocol, "subUrl", setting.AppSubUrl, "socket", setting.SocketPath)

	hs.httpSrv = &http.Server{Addr: listenAddr, Handler: hs.macaron}
	switch setting.Protocol {
	case setting.HTTP:
		err = hs.httpSrv.ListenAndServe()
		if err == http.ErrServerClosed {
			hs.log.Debug("server was shutdown gracefully")
			return nil
		}
	case setting.HTTPS:
		err = hs.listenAndServeTLS(setting.CertFile, setting.KeyFile)
		if err == http.ErrServerClosed {
			hs.log.Debug("server was shutdown gracefully")
			return nil
		}
	case setting.SOCKET:
		ln, err := net.Listen("unix", setting.SocketPath)
		if err != nil {
			hs.log.Debug("server was shutdown gracefully")
			return nil
		}

		err = hs.httpSrv.Serve(ln)
		if err != nil {
			hs.log.Debug("server was shutdown gracefully")
			return nil
		}
	default:
		hs.log.Error("Invalid protocol", "protocol", setting.Protocol)
		err = errors.New("Invalid Protocol")
	}

	return err
}

func (hs *HttpServer) Shutdown(ctx context.Context) error {
	err := hs.httpSrv.Shutdown(ctx)
	hs.log.Info("stopped http server")
	return err
}

func (hs *HttpServer) listenAndServeTLS(certfile, keyfile string) error {
	if certfile == "" {
		return fmt.Errorf("cert_file cannot be empty when using HTTPS")
	}

	if keyfile == "" {
		return fmt.Errorf("cert_key cannot be empty when using HTTPS")
	}

	if _, err := os.Stat(setting.CertFile); os.IsNotExist(err) {
		return fmt.Errorf(`Cannot find SSL cert_file at %v`, setting.CertFile)
	}

	if _, err := os.Stat(setting.KeyFile); os.IsNotExist(err) {
		return fmt.Errorf(`Cannot find SSL key_file at %v`, setting.KeyFile)
	}

	tlsCfg := &tls.Config{
		MinVersion:               tls.VersionTLS12,
		PreferServerCipherSuites: true,
		CipherSuites: []uint16{
			tls.TLS_RSA_WITH_AES_128_CBC_SHA,
			tls.TLS_RSA_WITH_AES_256_CBC_SHA,
			tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
		},
	}

	hs.httpSrv.TLSConfig = tlsCfg
	hs.httpSrv.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0)

	return hs.httpSrv.ListenAndServeTLS(setting.CertFile, setting.KeyFile)
}

func (hs *HttpServer) newMacaron() *macaron.Macaron {
	macaron.Env = setting.Env
	m := macaron.New()

	m.Use(middleware.Logger())
	m.Use(middleware.Recovery())

	if setting.EnableGzip {
		m.Use(middleware.Gziper())
	}

	for _, route := range plugins.StaticRoutes {
		pluginRoute := path.Join("/public/plugins/", route.PluginId)
		logger.Debug("Plugins: Adding route", "route", pluginRoute, "dir", route.Directory)
		hs.mapStatic(m, route.Directory, "", pluginRoute)
	}

	hs.mapStatic(m, setting.StaticRootPath, "", "public")
	hs.mapStatic(m, setting.StaticRootPath, "robots.txt", "robots.txt")

	m.Use(macaron.Renderer(macaron.RenderOptions{
		Directory:  path.Join(setting.StaticRootPath, "views"),
		IndentJSON: macaron.Env != macaron.PROD,
		Delims:     macaron.Delims{Left: "[[", Right: "]]"},
	}))

	m.Use(hs.healthHandler)
	m.Use(hs.metricsEndpoint)
	m.Use(middleware.GetContextHandler())
	m.Use(middleware.Sessioner(&setting.SessionOptions))
	m.Use(middleware.OrgRedirect())

	// needs to be after context handler
	if setting.EnforceDomain {
		m.Use(middleware.ValidateHostHeader(setting.Domain))
	}

	m.Use(middleware.AddDefaultResponseHeaders())

	return m
}

func (hs *HttpServer) metricsEndpoint(ctx *macaron.Context) {
	if ctx.Req.Method != "GET" || ctx.Req.URL.Path != "/metrics" {
		return
	}

	promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{
		DisableCompression: true,
	}).ServeHTTP(ctx.Resp, ctx.Req.Request)
}

func (hs *HttpServer) healthHandler(ctx *macaron.Context) {
	if ctx.Req.Method != "GET" || ctx.Req.URL.Path != "/api/health" {
		return
	}

	data := simplejson.New()
	data.Set("database", "ok")
	data.Set("version", setting.BuildVersion)
	data.Set("commit", setting.BuildCommit)

	if err := bus.Dispatch(&models.GetDBHealthQuery{}); err != nil {
		data.Set("database", "failing")
		ctx.Resp.Header().Set("Content-Type", "application/json; charset=UTF-8")
		ctx.Resp.WriteHeader(503)
	} else {
		ctx.Resp.Header().Set("Content-Type", "application/json; charset=UTF-8")
		ctx.Resp.WriteHeader(200)
	}

	dataBytes, _ := data.EncodePretty()
	ctx.Resp.Write(dataBytes)
}

func (hs *HttpServer) mapStatic(m *macaron.Macaron, rootDir string, dir string, prefix string) {
	headers := func(c *macaron.Context) {
		c.Resp.Header().Set("Cache-Control", "public, max-age=3600")
	}

	if setting.Env == setting.DEV {
		headers = func(c *macaron.Context) {
			c.Resp.Header().Set("Cache-Control", "max-age=0, must-revalidate, no-cache")
		}
	}

	m.Use(httpstatic.Static(
		path.Join(rootDir, dir),
		httpstatic.StaticOptions{
			SkipLogging: true,
			Prefix:      prefix,
			AddHeaders:  headers,
		},
	))
}
<|endoftext|>


<commit_before>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package compose

import (
	"context"
	"fmt"

	"github.com/docker/compose/v2/pkg/api"
)

// logPrinter watch application containers an collect their logs
type logPrinter interface {
	HandleEvent(event api.ContainerEvent)
	Run(ctx context.Context, cascadeStop bool, exitCodeFrom string, stopFn func() error) (int, error)
	Cancel()
}

type printer struct {
	queue    chan api.ContainerEvent
	consumer api.LogConsumer
}

// newLogPrinter builds a LogPrinter passing containers logs to LogConsumer
func newLogPrinter(consumer api.LogConsumer) logPrinter {
	queue := make(chan api.ContainerEvent)
	printer := printer{
		consumer: consumer,
		queue:    queue,
	}
	return &printer
}

func (p *printer) Cancel() {
	p.queue <- api.ContainerEvent{
		Type: api.UserCancel,
	}
}

func (p *printer) HandleEvent(event api.ContainerEvent) {
	p.queue <- event
}

//nolint:gocyclo
func (p *printer) Run(ctx context.Context, cascadeStop bool, exitCodeFrom string, stopFn func() error) (int, error) {
	var (
		aborting bool
		exitCode int
	)
	containers := map[string]struct{}{}
	for {
		select {
		case <-ctx.Done():
			return exitCode, ctx.Err()
		case event := <-p.queue:
			container := event.Container
			switch event.Type {
			case api.UserCancel:
				aborting = true
			case api.ContainerEventAttach:
				if _, ok := containers[container]; ok {
					continue
				}
				containers[container] = struct{}{}
				p.consumer.Register(container)
			case api.ContainerEventExit, api.ContainerEventStopped:
				if !event.Restarting {
					delete(containers, container)
				}
				if !aborting {
					p.consumer.Status(container, fmt.Sprintf("exited with code %d", event.ExitCode))
				}
				if cascadeStop {
					if !aborting {
						aborting = true
						fmt.Println("Aborting on container exit...")
						err := stopFn()
						if err != nil {
							return 0, err
						}
					}
					if exitCodeFrom == "" {
						exitCodeFrom = event.Service
					}
					if exitCodeFrom == event.Service {
						exitCode = event.ExitCode
					}
				}
				if len(containers) == 0 {
					// Last container terminated, done
					return exitCode, nil
				}
			case api.ContainerEventLog:
				if !aborting {
					p.consumer.Log(container, event.Service, event.Line)
				}
			}
		}
	}
}
<commit_msg>Only capture exit codes from `exit` events<commit_after>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package compose

import (
	"context"
	"fmt"

	"github.com/docker/compose/v2/pkg/api"
)

// logPrinter watch application containers an collect their logs
type logPrinter interface {
	HandleEvent(event api.ContainerEvent)
	Run(ctx context.Context, cascadeStop bool, exitCodeFrom string, stopFn func() error) (int, error)
	Cancel()
}

type printer struct {
	queue    chan api.ContainerEvent
	consumer api.LogConsumer
}

// newLogPrinter builds a LogPrinter passing containers logs to LogConsumer
func newLogPrinter(consumer api.LogConsumer) logPrinter {
	queue := make(chan api.ContainerEvent)
	printer := printer{
		consumer: consumer,
		queue:    queue,
	}
	return &printer
}

func (p *printer) Cancel() {
	p.queue <- api.ContainerEvent{
		Type: api.UserCancel,
	}
}

func (p *printer) HandleEvent(event api.ContainerEvent) {
	p.queue <- event
}

//nolint:gocyclo
func (p *printer) Run(ctx context.Context, cascadeStop bool, exitCodeFrom string, stopFn func() error) (int, error) {
	var (
		aborting bool
		exitCode int
	)
	containers := map[string]struct{}{}
	for {
		select {
		case <-ctx.Done():
			return exitCode, ctx.Err()
		case event := <-p.queue:
			container := event.Container
			switch event.Type {
			case api.UserCancel:
				aborting = true
			case api.ContainerEventAttach:
				if _, ok := containers[container]; ok {
					continue
				}
				containers[container] = struct{}{}
				p.consumer.Register(container)
			case api.ContainerEventExit, api.ContainerEventStopped:
				if !event.Restarting {
					delete(containers, container)
				}
				if !aborting {
					p.consumer.Status(container, fmt.Sprintf("exited with code %d", event.ExitCode))
				}
				if cascadeStop {
					if !aborting {
						aborting = true
						fmt.Println("Aborting on container exit...")
						err := stopFn()
						if err != nil {
							return 0, err
						}
					}
					if event.Type == api.ContainerEventExit {
						if exitCodeFrom == "" {
							exitCodeFrom = event.Service
						}
						if exitCodeFrom == event.Service {
							exitCode = event.ExitCode
						}
					}
				}
				if len(containers) == 0 {
					// Last container terminated, done
					return exitCode, nil
				}
			case api.ContainerEventLog:
				if !aborting {
					p.consumer.Log(container, event.Service, event.Line)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io"
	"os"
	"strconv"

	"github.com/spf13/cobra"
	"k8s.io/kubernetes/pkg/api"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	libutil "k8s.io/kubernetes/pkg/util"
)

const (
	log_example = `# Returns snapshot of ruby-container logs from pod 123456-7890.
$ kubectl logs 123456-7890 ruby-container

# Returns snapshot of previous terminated ruby-container logs from pod 123456-7890.
$ kubectl logs -p 123456-7890 ruby-container

# Starts streaming of ruby-container logs from pod 123456-7890.
$ kubectl logs -f 123456-7890 ruby-container`
)

func selectContainer(pod *api.Pod, in io.Reader, out io.Writer) string {
	fmt.Fprintf(out, "Please select a container:\n")
	options := libutil.StringSet{}
	for ix := range pod.Spec.Containers {
		fmt.Fprintf(out, "[%d] %s\n", ix+1, pod.Spec.Containers[ix].Name)
		options.Insert(pod.Spec.Containers[ix].Name)
	}
	for {
		var input string
		fmt.Fprintf(out, "> ")
		fmt.Fscanln(in, &input)
		if options.Has(input) {
			return input
		}
		ix, err := strconv.Atoi(input)
		if err == nil && ix > 0 && ix <= len(pod.Spec.Containers) {
			return pod.Spec.Containers[ix-1].Name
		}
		fmt.Fprintf(out, "Invalid input: %s", input)
	}
}

type logParams struct {
	containerName string
}

// NewCmdLog creates a new pod log command
func NewCmdLog(f *cmdutil.Factory, out io.Writer) *cobra.Command {
	params := &logParams{}
	cmd := &cobra.Command{
		Use:     "logs [-f] [-p] POD [-c CONTAINER]",
		Short:   "Print the logs for a container in a pod.",
		Long:    "Print the logs for a container in a pod. If the pod has only one container, the container name is optional.",
		Example: log_example,
		Run: func(cmd *cobra.Command, args []string) {
			err := RunLog(f, out, cmd, args, params)
			cmdutil.CheckErr(err)
		},
		Aliases: []string{"log"},
	}
	cmd.Flags().BoolP("follow", "f", false, "Specify if the logs should be streamed.")
	cmd.Flags().Bool("interactive", true, "If true, prompt the user for input when required. Default true.")
	cmd.Flags().BoolP("previous", "p", false, "If true, print the logs for the previous instance of the container in a pod if it exists.")
	cmd.Flags().StringVarP(&params.containerName, "container", "c", "", "Container name")
	return cmd
}

// RunLog retrieves a pod log
func RunLog(f *cmdutil.Factory, out io.Writer, cmd *cobra.Command, args []string, p *logParams) error {
	if len(os.Args) > 1 && os.Args[1] == "log" {
		printDeprecationWarning("logs", "log")
	}

	if len(args) == 0 {
		return cmdutil.UsageError(cmd, "POD is required for log")
	}

	if len(args) > 2 {
		return cmdutil.UsageError(cmd, "log POD [CONTAINER]")
	}

	namespace, _, err := f.DefaultNamespace()
	if err != nil {
		return err
	}
	client, err := f.Client()
	if err != nil {
		return err
	}

	podID := args[0]

	pod, err := client.Pods(namespace).Get(podID)
	if err != nil {
		return err
	}

	var container string
	if cmdutil.GetFlagString(cmd, "container") != "" {
		// [-c CONTAINER]
		container = p.containerName
	} else {
		// [CONTAINER] (container as arg not flag) is supported as legacy behavior. See PR #10519 for more details.
		if len(args) == 1 {
			if len(pod.Spec.Containers) != 1 {
				return fmt.Errorf("POD %s has more than one container; please specify the container to print logs for", pod.ObjectMeta.Name)
			}
			container = pod.Spec.Containers[0].Name
		} else {
			container = args[1]
		}
	}

	follow := false
	if cmdutil.GetFlagBool(cmd, "follow") {
		follow = true
	}

	previous := false
	if cmdutil.GetFlagBool(cmd, "previous") {
		previous = true
	}

	readCloser, err := client.RESTClient.Get().
		Namespace(namespace).
		Name(podID).
		Resource("pods").
		SubResource("log").
		Param("follow", strconv.FormatBool(follow)).
		Param("container", container).
		Param("previous", strconv.FormatBool(previous)).
		Stream()
	if err != nil {
		return err
	}

	defer readCloser.Close()
	_, err = io.Copy(out, readCloser)
	return err
}
<commit_msg>Print valid container names when the command not specify the container<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"

	"github.com/spf13/cobra"
	"k8s.io/kubernetes/pkg/api"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	libutil "k8s.io/kubernetes/pkg/util"
)

const (
	log_example = `# Returns snapshot of ruby-container logs from pod 123456-7890.
$ kubectl logs 123456-7890 ruby-container

# Returns snapshot of previous terminated ruby-container logs from pod 123456-7890.
$ kubectl logs -p 123456-7890 ruby-container

# Starts streaming of ruby-container logs from pod 123456-7890.
$ kubectl logs -f 123456-7890 ruby-container`
)

func selectContainer(pod *api.Pod, in io.Reader, out io.Writer) string {
	fmt.Fprintf(out, "Please select a container:\n")
	options := libutil.StringSet{}
	for ix := range pod.Spec.Containers {
		fmt.Fprintf(out, "[%d] %s\n", ix+1, pod.Spec.Containers[ix].Name)
		options.Insert(pod.Spec.Containers[ix].Name)
	}
	for {
		var input string
		fmt.Fprintf(out, "> ")
		fmt.Fscanln(in, &input)
		if options.Has(input) {
			return input
		}
		ix, err := strconv.Atoi(input)
		if err == nil && ix > 0 && ix <= len(pod.Spec.Containers) {
			return pod.Spec.Containers[ix-1].Name
		}
		fmt.Fprintf(out, "Invalid input: %s", input)
	}
}

type logParams struct {
	containerName string
}

// NewCmdLog creates a new pod log command
func NewCmdLog(f *cmdutil.Factory, out io.Writer) *cobra.Command {
	params := &logParams{}
	cmd := &cobra.Command{
		Use:     "logs [-f] [-p] POD [-c CONTAINER]",
		Short:   "Print the logs for a container in a pod.",
		Long:    "Print the logs for a container in a pod. If the pod has only one container, the container name is optional.",
		Example: log_example,
		Run: func(cmd *cobra.Command, args []string) {
			err := RunLog(f, out, cmd, args, params)
			cmdutil.CheckErr(err)
		},
		Aliases: []string{"log"},
	}
	cmd.Flags().BoolP("follow", "f", false, "Specify if the logs should be streamed.")
	cmd.Flags().Bool("interactive", true, "If true, prompt the user for input when required. Default true.")
	cmd.Flags().BoolP("previous", "p", false, "If true, print the logs for the previous instance of the container in a pod if it exists.")
	cmd.Flags().StringVarP(&params.containerName, "container", "c", "", "Container name")
	return cmd
}

// RunLog retrieves a pod log
func RunLog(f *cmdutil.Factory, out io.Writer, cmd *cobra.Command, args []string, p *logParams) error {
	if len(os.Args) > 1 && os.Args[1] == "log" {
		printDeprecationWarning("logs", "log")
	}

	if len(args) == 0 {
		return cmdutil.UsageError(cmd, "POD is required for log")
	}

	if len(args) > 2 {
		return cmdutil.UsageError(cmd, "log POD [CONTAINER]")
	}

	namespace, _, err := f.DefaultNamespace()
	if err != nil {
		return err
	}
	client, err := f.Client()
	if err != nil {
		return err
	}

	podID := args[0]

	pod, err := client.Pods(namespace).Get(podID)
	if err != nil {
		return err
	}

	// [-c CONTAINER]
	container := p.containerName
	if len(container) == 0 {
		// [CONTAINER] (container as arg not flag) is supported as legacy behavior. See PR #10519 for more details.
		if len(args) == 1 {
			if len(pod.Spec.Containers) != 1 {
				podContainersNames := []string{}
				for _, container := range pod.Spec.Containers {
					podContainersNames = append(podContainersNames, container.Name)
				}

				return fmt.Errorf("Pod %s has the following containers: %s; please specify the container to print logs for with -c", pod.ObjectMeta.Name, strings.Join(podContainersNames, ", "))
			}
			container = pod.Spec.Containers[0].Name
		} else {
			container = args[1]
		}
	}

	follow := false
	if cmdutil.GetFlagBool(cmd, "follow") {
		follow = true
	}

	previous := false
	if cmdutil.GetFlagBool(cmd, "previous") {
		previous = true
	}

	readCloser, err := client.RESTClient.Get().
		Namespace(namespace).
		Name(podID).
		Resource("pods").
		SubResource("log").
		Param("follow", strconv.FormatBool(follow)).
		Param("container", container).
		Param("previous", strconv.FormatBool(previous)).
		Stream()
	if err != nil {
		return err
	}

	defer readCloser.Close()
	_, err = io.Copy(out, readCloser)
	return err
}
<|endoftext|>


<commit_before>// Golang API for cex.io trading
// Author: julien@hautefeuille.eu
// BTC: 1Q2BDvd5zWW8znK7cBdaRTA9hBSLJJZzoy
// Date: 09/03/2014
// Version: 0.2

package cexio

import (
    "net/http"
    "net/url"
    "io/ioutil"
    "log"
    "time"
    "strconv"
    "strings"
    "crypto/hmac"
    "crypto/sha256"
    "encoding/hex"
)

type CexKey struct {
    username string
    api_key string
    api_secret string
}

func (cexapi CexKey) Nonce() string {
    return strconv.FormatInt(time.Now().UnixNano(), 10)
}

func (cexapi CexKey) ToHmac256(message string, secret string) string {
    key := []byte(secret)
    h := hmac.New(sha256.New, key)
    h.Write([]byte(message))
    return strings.ToUpper(hex.EncodeToString(h.Sum(nil)))
}

func (cexapi CexKey) Signature() (string, string) {
    nonce := cexapi.Nonce()
    message := nonce + cexapi.username + cexapi.api_key
    signature := cexapi.ToHmac256(message, cexapi.api_secret)
    return signature, nonce
}

func (cexapi CexKey) GetMethod(u string) []byte{
    res, err := http.Get(u)
    if err != nil {log.Fatal(err)}
    data, err := ioutil.ReadAll(res.Body)
    res.Body.Close()
    if err != nil {log.Fatal(err)}
    return data
}

func (cexapi CexKey) PostMethod(u string, v url.Values) []byte{
    res, err := http.PostForm(u, v)
    if err != nil {log.Fatal(err)}
    data, err := ioutil.ReadAll(res.Body)
    res.Body.Close()
    if err != nil {log.Fatal(err)}
    return data
}

func (cexapi CexKey) ApiCall(method string, id string, param map[string]string, private bool, opt string) []byte {
    var data []byte
    u := "https://cex.io/api/" + method + "/"
    w := "https://cex.io/api/ghash.io/" + method
    if len(opt) != 0 {
        u = u + opt + "/"
    }
    if private {
        // Post method for private method
        signature, nonce := cexapi.Signature()
        v := url.Values{}
        v.Set("key", cexapi.api_key)
        v.Add("signature", signature)
        v.Add("nonce", nonce)
        // Place order param
        if len(param) != 0 {
            v.Add("type", param["ordertype"])
            v.Add("amount", param["amount"])
            v.Add("price", param["price"])
        }
        // Cancel order id
        if len(id) != 0 {
            v.Add("id", id)
        }
        v.Encode()
        if method == "workers" || method == "hashrate" {
            // Ghash.io post method
            data = cexapi.PostMethod(w, v) // url ghash.io , param
        } else {
            // Cex.io post method
            data = cexapi.PostMethod(u, v) // url cex.io, param
        }
    } else {
        // Get method for public method
        data = cexapi.GetMethod(u)
    }
    return data
}
// Public functions
func (cexapi CexKey) Ticker(opt string) []byte {
    return cexapi.ApiCall("ticker", "", map[string]string{}, false, opt)
}

func (cexapi CexKey) OrderBook(opt string) []byte {
    return cexapi.ApiCall("order_book", "", map[string]string{}, false, opt)
}

func (cexapi CexKey) TradeHistory(opt string) []byte {
    return cexapi.ApiCall("trade_history", "", map[string]string{}, false, opt)
}

// Private functions
func (cexapi CexKey) Balance() []byte {
    return cexapi.ApiCall("balance", "", map[string]string{}, true, "")
}

func (cexapi CexKey) OpenOrders(opt string) []byte {
    return cexapi.ApiCall("open_orders", "", map[string]string{}, true, opt)
}

// Orders functions
func (cexapi CexKey) PlaceOrder(ordertype string, amount string, price string, opt string) []byte {
    var param = map[string]string {
        "ordertype": ordertype,
        "amount": amount,
        "price" : price}
    return cexapi.ApiCall("place_order", "", param, true, opt)
}

func (cexapi CexKey) CancelOrder(id string) []byte {
    return cexapi.ApiCall("cancel_order", id, map[string]string{}, true, "")
}

// Workers functions
func (cexapi CexKey) Hashrate() []byte {
    return cexapi.ApiCall("hashrate", "", map[string]string{}, true, "")
}

func (cexapi CexKey) Workers() []byte {
    return cexapi.ApiCall("workers", "", map[string]string{}, true, "")
}
<commit_msg>Cexkey exported available field<commit_after>// Golang API for cex.io trading
// Author: julien@hautefeuille.eu
// BTC: 1Q2BDvd5zWW8znK7cBdaRTA9hBSLJJZzoy
// Date: 09/03/2014
// Version: 0.2

package cexio

import (
    "net/http"
    "net/url"
    "io/ioutil"
    "log"
    "time"
    "strconv"
    "strings"
    "crypto/hmac"
    "crypto/sha256"
    "encoding/hex"
)

type CexKey struct {
    Username string
    Api_key string
    Api_secret string
}

func (cexapi CexKey) Nonce() string {
    return strconv.FormatInt(time.Now().UnixNano(), 10)
}

func (cexapi CexKey) ToHmac256(message string, secret string) string {
    key := []byte(secret)
    h := hmac.New(sha256.New, key)
    h.Write([]byte(message))
    return strings.ToUpper(hex.EncodeToString(h.Sum(nil)))
}

func (cexapi CexKey) Signature() (string, string) {
    nonce := cexapi.Nonce()
    message := nonce + cexapi.Username + cexapi.Api_key
    signature := cexapi.ToHmac256(message, cexapi.Api_secret)
    return signature, nonce
}

func (cexapi CexKey) GetMethod(u string) []byte{
    res, err := http.Get(u)
    if err != nil {log.Fatal(err)}
    data, err := ioutil.ReadAll(res.Body)
    res.Body.Close()
    if err != nil {log.Fatal(err)}
    return data
}

func (cexapi CexKey) PostMethod(u string, v url.Values) []byte{
    res, err := http.PostForm(u, v)
    if err != nil {log.Fatal(err)}
    data, err := ioutil.ReadAll(res.Body)
    res.Body.Close()
    if err != nil {log.Fatal(err)}
    return data
}

func (cexapi CexKey) ApiCall(method string, id string, param map[string]string, private bool, opt string) []byte {
    var data []byte
    u := "https://cex.io/api/" + method + "/"
    w := "https://cex.io/api/ghash.io/" + method
    if len(opt) != 0 {
        u = u + opt + "/"
    }
    if private {
        // Post method for private method
        signature, nonce := cexapi.Signature()
        v := url.Values{}
        v.Set("key", cexapi.Api_key)
        v.Add("signature", signature)
        v.Add("nonce", nonce)
        // Place order param
        if len(param) != 0 {
            v.Add("type", param["ordertype"])
            v.Add("amount", param["amount"])
            v.Add("price", param["price"])
        }
        // Cancel order id
        if len(id) != 0 {
            v.Add("id", id)
        }
        v.Encode()
        if method == "workers" || method == "hashrate" {
            // Ghash.io post method
            data = cexapi.PostMethod(w, v) // url ghash.io , param
        } else {
            // Cex.io post method
            data = cexapi.PostMethod(u, v) // url cex.io, param
        }
    } else {
        // Get method for public method
        data = cexapi.GetMethod(u)
    }
    return data
}
// Public functions
func (cexapi CexKey) Ticker(opt string) []byte {
    return cexapi.ApiCall("ticker", "", map[string]string{}, false, opt)
}

func (cexapi CexKey) OrderBook(opt string) []byte {
    return cexapi.ApiCall("order_book", "", map[string]string{}, false, opt)
}

func (cexapi CexKey) TradeHistory(opt string) []byte {
    return cexapi.ApiCall("trade_history", "", map[string]string{}, false, opt)
}

// Private functions
func (cexapi CexKey) Balance() []byte {
    return cexapi.ApiCall("balance", "", map[string]string{}, true, "")
}

func (cexapi CexKey) OpenOrders(opt string) []byte {
    return cexapi.ApiCall("open_orders", "", map[string]string{}, true, opt)
}

// Orders functions
func (cexapi CexKey) PlaceOrder(ordertype string, amount string, price string, opt string) []byte {
    var param = map[string]string {
        "ordertype": ordertype,
        "amount": amount,
        "price" : price}
    return cexapi.ApiCall("place_order", "", param, true, opt)
}

func (cexapi CexKey) CancelOrder(id string) []byte {
    return cexapi.ApiCall("cancel_order", id, map[string]string{}, true, "")
}

// Workers functions
func (cexapi CexKey) Hashrate() []byte {
    return cexapi.ApiCall("hashrate", "", map[string]string{}, true, "")
}

func (cexapi CexKey) Workers() []byte {
    return cexapi.ApiCall("workers", "", map[string]string{}, true, "")
}
<|endoftext|>


<commit_before>package locator

import (
	"bufio"
	"context"
	"fmt"
	"io/ioutil"
	"math/big"
	"net/http"
	"regexp"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/prometheus/client_golang/api/prometheus"
)

var (
	splitter = regexp.MustCompile("\r?\n")
)

// Locator is a pluggable interface for locating prometheus endpoints
type Locator interface {

	// Endpoints provides a list of candidate prometheus endpoints
	Endpoints() ([]*PrometheusEndpoint, error)
}

// PrometheusEndpoint encapsulates a QueryAPI instance and its associated address
type PrometheusEndpoint struct {
	QueryAPI              prometheus.QueryAPI
	Error                 error
	Uptime                time.Duration
	Selected              bool
	Address               string
	ComparisonMetricValue interface{}
}

func (pe *PrometheusEndpoint) String() string {
	return pe.Address
}

// Viable returns true if the endpoint is able to
// respond to basic query API requests within a reasonable time
func (pe *PrometheusEndpoint) Viable() bool {
	ctx, cancel := context.WithTimeout(context.TODO(), time.Second)
	_, err := pe.QueryAPI.Query(ctx, "up", time.Now())
	defer cancel()
	if err != nil {
		log.Warnf("Endpoint %v is not viable, based on returned error: %v", pe, err)
		return false
	}
	return true
}

type staticLocator struct {
	endpointsFile string
}

// NewEndpointsFileLocator returns a new Locator which reads
// a set of endpoints from a file path, one endpoint per line
func NewEndpointsFileLocator(endpointsFile string) Locator {
	return &staticLocator{endpointsFile: endpointsFile}
}

// Endpoints provides a list of candidate prometheus endpoints
func (sl *staticLocator) Endpoints() ([]*PrometheusEndpoint, error) {
	b, err := ioutil.ReadFile(sl.endpointsFile)
	if err != nil {
		return nil, err
	}
	return ToPrometheusClients(splitter.Split(strings.Trim(string(b), "\n"), -1))
}

// ToPrometheusClients generates prometheus Client objects from a provided list of URLs
func ToPrometheusClients(endpointURLs []string) ([]*PrometheusEndpoint, error) {
	endpoints := make([]*PrometheusEndpoint, 0, len(endpointURLs))
	for _, endpoint := range endpointURLs {
		addr := strings.Trim(endpoint, " ")
		if len(addr) > 0 {
			var uptime time.Duration
			var queryAPI prometheus.QueryAPI
			client, err := prometheus.New(prometheus.Config{
				Address: addr,
			})
			if err == nil {
				// Scape the /metrics endpoint of the individual prometheus instance, since
				// self-scaping of prometheus' own metrics might not be configured
				if log.GetLevel() >= log.DebugLevel {
					log.Debugf("Testing %s/metrics", addr)
				}
				scraped, err := ScrapeMetric(addr, "process_start_time_seconds")
				if err == nil && scraped != nil {
					processStartTimeSeconds := scraped.Value
					uptime = time.Duration(time.Now().UTC().Unix()-int64(processStartTimeSeconds)) * time.Second
					if log.GetLevel() >= log.DebugLevel {
						log.Debugf("Parsed current uptime for %s: %s", addr, uptime)
					}
					queryAPI = prometheus.NewQueryAPI(client)
					_, err = queryAPI.Query(context.TODO(), "up", time.Now())
					if err != nil && log.GetLevel() >= log.DebugLevel {
						log.Debugf("Query 'up' returned error: %v", err)
					}
				}
			}

			if err == nil {
				endpoints = append(endpoints, &PrometheusEndpoint{QueryAPI: queryAPI, Address: addr, Uptime: uptime})
			} else {
				log.Errorf("Failed to resolve build_info and uptime for %v: %v", addr, err)
				endpoints = append(endpoints, &PrometheusEndpoint{Address: addr, Uptime: time.Duration(0), Error: err})
			}
		}
	}
	if len(endpoints) == 0 {
		return nil, fmt.Errorf("Unable to locate any potential endpoints")
	}
	return endpoints, nil
}

// LabeledValue represents a persed metric instance
type LabeledValue struct {
	Name   string
	Labels string
	Value  float64
}

func (lv *LabeledValue) String() string {
	return fmt.Sprintf("%s%s %f", lv.Name, lv.Labels, lv.Value)
}

// ScrapeMetric parses metrics in a simple fashion, returning
// the first instance of each metric for a given name; results may be unexpected
// for metrics with multiple instances
func ScrapeMetric(addr string, name string) (*LabeledValue, error) {

	resp, err := http.Get(fmt.Sprintf("%s/metrics", addr))
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("%s/metrics returned %d", addr, resp.StatusCode)
	}

	defer resp.Body.Close()
	scanner := bufio.NewScanner(resp.Body)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		if !strings.HasPrefix(line, "#") {
			parts := strings.Split(line, " ")
			nameParts := strings.Split(parts[0], "{")
			if nameParts[0] == name {
				f := new(big.Float)
				_, err := fmt.Sscan(parts[1], f)
				if err == nil {
					v := &LabeledValue{Name: nameParts[0]}
					v.Value, _ = f.Float64()
					if len(nameParts) > 1 {
						v.Labels = "{" + nameParts[1]
					}
					return v, nil
				}
				return nil, fmt.Errorf("Failed to parse value for metric %s", line)
			}
		}
	}
	return nil, nil
}
<commit_msg>add base timeouts for testing viability<commit_after>package locator

import (
	"bufio"
	"context"
	"fmt"
	"io/ioutil"
	"math/big"
	"net"
	"net/http"
	"regexp"
	"strings"
	"time"

	log "github.com/sirupsen/logrus"

	"github.com/prometheus/client_golang/api/prometheus"
)

var (
	splitter = regexp.MustCompile("\r?\n")
)

// Locator is a pluggable interface for locating prometheus endpoints
type Locator interface {

	// Endpoints provides a list of candidate prometheus endpoints
	Endpoints() ([]*PrometheusEndpoint, error)
}

// PrometheusEndpoint encapsulates a QueryAPI instance and its associated address
type PrometheusEndpoint struct {
	QueryAPI              prometheus.QueryAPI
	Error                 error
	Uptime                time.Duration
	Selected              bool
	Address               string
	ComparisonMetricValue interface{}
}

func (pe *PrometheusEndpoint) String() string {
	return pe.Address
}

const connectionTimetout = 1 * time.Second
const readTimeout = 3 * time.Second

func timeoutDialer(netw, addr string) (net.Conn, error) {
	conn, err := net.DialTimeout(netw, addr, connectionTimetout)
	if err != nil {
		return nil, err
	}
	conn.SetDeadline(time.Now().Add(readTimeout))
	return conn, nil
}

var httpClient = &http.Client{
	Transport: &http.Transport{
		Dial: timeoutDialer,
	},
}

// Viable returns true if the endpoint is able to
// respond to basic query API requests within a reasonable time
func (pe *PrometheusEndpoint) Viable() bool {
	ctx, cancel := context.WithTimeout(context.TODO(), time.Second)
	_, err := pe.QueryAPI.Query(ctx, "up", time.Now())
	defer cancel()
	if err != nil {
		log.Warnf("Endpoint %v is not viable, based on returned error: %v", pe, err)
		return false
	}
	return true
}

type staticLocator struct {
	endpointsFile string
}

// NewEndpointsFileLocator returns a new Locator which reads
// a set of endpoints from a file path, one endpoint per line
func NewEndpointsFileLocator(endpointsFile string) Locator {
	return &staticLocator{endpointsFile: endpointsFile}
}

// Endpoints provides a list of candidate prometheus endpoints
func (sl *staticLocator) Endpoints() ([]*PrometheusEndpoint, error) {
	b, err := ioutil.ReadFile(sl.endpointsFile)
	if err != nil {
		return nil, err
	}
	return ToPrometheusClients(splitter.Split(strings.Trim(string(b), "\n"), -1))
}

// ToPrometheusClients generates prometheus Client objects from a provided list of URLs
func ToPrometheusClients(endpointURLs []string) ([]*PrometheusEndpoint, error) {
	endpoints := make([]*PrometheusEndpoint, 0, len(endpointURLs))
	for _, endpoint := range endpointURLs {
		addr := strings.Trim(endpoint, " ")
		if len(addr) > 0 {
			var uptime time.Duration
			var queryAPI prometheus.QueryAPI
			client, err := prometheus.New(prometheus.Config{
				Address: addr,
			})
			if err == nil {
				// Scape the /metrics endpoint of the individual prometheus instance, since
				// self-scaping of prometheus' own metrics might not be configured
				if log.GetLevel() >= log.DebugLevel {
					log.Debugf("Testing %s/metrics", addr)
				}
				scraped, err := ScrapeMetric(addr, "process_start_time_seconds")
				if err == nil && scraped != nil {
					processStartTimeSeconds := scraped.Value
					uptime = time.Duration(time.Now().UTC().Unix()-int64(processStartTimeSeconds)) * time.Second
					if log.GetLevel() >= log.DebugLevel {
						log.Debugf("Parsed current uptime for %s: %s", addr, uptime)
					}
					queryAPI = prometheus.NewQueryAPI(client)
					_, err = queryAPI.Query(context.TODO(), "up", time.Now())
					if err != nil && log.GetLevel() >= log.DebugLevel {
						log.Debugf("Query 'up' returned error: %v", err)
					}
				}
			}

			if err == nil {
				endpoints = append(endpoints, &PrometheusEndpoint{QueryAPI: queryAPI, Address: addr, Uptime: uptime})
			} else {
				log.Errorf("Failed to resolve build_info and uptime for %v: %v", addr, err)
				endpoints = append(endpoints, &PrometheusEndpoint{Address: addr, Uptime: time.Duration(0), Error: err})
			}
		}
	}
	if len(endpoints) == 0 {
		return nil, fmt.Errorf("Unable to locate any potential endpoints")
	}
	return endpoints, nil
}

// LabeledValue represents a persed metric instance
type LabeledValue struct {
	Name   string
	Labels string
	Value  float64
}

func (lv *LabeledValue) String() string {
	return fmt.Sprintf("%s%s %f", lv.Name, lv.Labels, lv.Value)
}

// ScrapeMetric parses metrics in a simple fashion, returning
// the first instance of each metric for a given name; results may be unexpected
// for metrics with multiple instances
func ScrapeMetric(addr string, name string) (*LabeledValue, error) {

	resp, err := httpClient.Get(fmt.Sprintf("%s/metrics", addr))
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("%s/metrics returned %d", addr, resp.StatusCode)
	}

	defer resp.Body.Close()
	scanner := bufio.NewScanner(resp.Body)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		if !strings.HasPrefix(line, "#") {
			parts := strings.Split(line, " ")
			nameParts := strings.Split(parts[0], "{")
			if nameParts[0] == name {
				f := new(big.Float)
				_, err := fmt.Sscan(parts[1], f)
				if err == nil {
					v := &LabeledValue{Name: nameParts[0]}
					v.Value, _ = f.Float64()
					if len(nameParts) > 1 {
						v.Labels = "{" + nameParts[1]
					}
					return v, nil
				}
				return nil, fmt.Errorf("Failed to parse value for metric %s", line)
			}
		}
	}
	return nil, nil
}
<|endoftext|>


<commit_before>package cairo

//#cgo pkg-config: cairo
//#include <stdlib.h>
//#include <cairo/cairo.h>
import "C"

import (
	"runtime"
	"unsafe"
)

//An ImageSurface is an in-memory surface.
type ImageSurface struct {
	*XtensionSurface
	format                Format
	width, height, stride int
}

func newImg(s *C.cairo_surface_t, format Format, width, height, stride int) (ImageSurface, error) {
	S := ImageSurface{
		XtensionSurface: NewXtensionSurface(s),
		format:          format,
		width:           width,
		height:          height,
		stride:          stride,
	}
	return S, S.Err()
}

//NewImageSurface creates an image surface of the given width, height,
//and format.
//
//Originally cairo_image_surface_create.
func NewImageSurface(format Format, width, height int) (ImageSurface, error) {
	is := C.cairo_image_surface_create(format.c(), C.int(width), C.int(height))
	stride := int(C.cairo_image_surface_get_stride(is))
	return newImg(is, format, width, height, stride)
}

func cNewImageSurface(s *C.cairo_surface_t) (Surface, error) {
	format := Format(C.cairo_image_surface_get_format(s))
	width := int(C.cairo_image_surface_get_width(s))
	height := int(C.cairo_image_surface_get_height(s))
	stride := int(C.cairo_image_surface_get_stride(s))

	return newImg(s, format, width, height, stride)
}

//NewImageSurfaceFromPNG creates a new image surface and initalizes
//it with the given PNG file.
//
//Originally cairo_image_surface_create_from_png_stream.
func NewImageSurfaceFromPNG(r Reader) (ImageSurface, error) {
	rp := wrapReader(r)
	is := C.cairo_image_surface_create_from_png_stream(cairoreadfunct, rp)
	s, err := cNewImageSurface(is)
	S := s.(ImageSurface)
	if err != nil {
		return S, err
	}
	S.registerReader(rp)
	return S, nil
}

//NewImageSurfaceFromPNGFile creates a new image surface and initializes
//it with the contents of the given PNG file.
//
//Originally cairo_image_surface_create_from_png.
func NewImageSurfaceFromPNGFile(filename string) (ImageSurface, error) {
	f := C.CString(filename)
	is := C.cairo_image_surface_create_from_png(f)
	C.free(unsafe.Pointer(f))
	s, err := cNewImageSurface(is)
	return s.(ImageSurface), err
}

//BUG(jmf): ImageSurface: need safe wrapper around get_data

//BUG(jmf): ImageSurface: need image_surface_create_for_data analog(s)

//Format reports the format of the surface.
//
//Originally cairo_image_surface_get_format.
func (i ImageSurface) Format() Format {
	return i.format
}

//Width reports the width of the surface in pixels.
//
//Originally cairo_image_surface_get_width.
func (i ImageSurface) Width() int {
	return i.width
}

//Height reports the height of the surface in pixels.
//
//Originally cairo_image_surface_get_height.
func (i ImageSurface) Height() int {
	return i.height
}

//Stride reports the stride of the image surface in number of bytes.
//
//Originally cairo_image_surface_get_stride.
func (i ImageSurface) Stride() int {
	return i.stride
}

type mappedImageSurface struct {
	ImageSurface
	from *C.cairo_surface_t
}

func newMappedImageSurface(s, from *C.cairo_surface_t) (m mappedImageSurface, err error) {
	im, err1 := cNewImageSurface(s)
	if err1 != nil {
		err = err1
		return
	}
	//Clear default finalizer so GC doesn't call surface_destroy.
	//We do not set a new finalizer on mappedImageSurface.Close,
	//because that would not do the right thing and the user is expected to close
	//manually when done.
	runtime.SetFinalizer(m.XtensionSurface, nil)
	m = mappedImageSurface{
		ImageSurface: im.(ImageSurface),
		from:         from,
	}
	err = m.Err()
	if err != nil {
		m.s = nil
	}
	return
}

func (m mappedImageSurface) Close() error {
	err := m.Err()
	C.cairo_surface_unmap_image(m.from, m.s)
	m.from = nil
	m.s = nil
	return err
}
<commit_msg>removed NewImageSurfaceFromPNGFile and renamed NewImageSurfaceFromPNG to ReadPNG<commit_after>package cairo

//#cgo pkg-config: cairo
//#include <stdlib.h>
//#include <cairo/cairo.h>
import "C"

import "runtime"

//An ImageSurface is an in-memory surface.
type ImageSurface struct {
	*XtensionSurface
	format                Format
	width, height, stride int
}

func newImg(s *C.cairo_surface_t, format Format, width, height, stride int) (ImageSurface, error) {
	S := ImageSurface{
		XtensionSurface: NewXtensionSurface(s),
		format:          format,
		width:           width,
		height:          height,
		stride:          stride,
	}
	return S, S.Err()
}

//NewImageSurface creates an image surface of the given width, height,
//and format.
//
//Originally cairo_image_surface_create.
func NewImageSurface(format Format, width, height int) (ImageSurface, error) {
	is := C.cairo_image_surface_create(format.c(), C.int(width), C.int(height))
	stride := int(C.cairo_image_surface_get_stride(is))
	return newImg(is, format, width, height, stride)
}

func cNewImageSurface(s *C.cairo_surface_t) (Surface, error) {
	format := Format(C.cairo_image_surface_get_format(s))
	width := int(C.cairo_image_surface_get_width(s))
	height := int(C.cairo_image_surface_get_height(s))
	stride := int(C.cairo_image_surface_get_stride(s))

	return newImg(s, format, width, height, stride)
}

//ReadPNG creates a new image surface and initalizes
//it with the given PNG file.
//
//Originally cairo_image_surface_create_from_png_stream.
func ReadPNG(r Reader) (ImageSurface, error) {
	rp := wrapReader(r)
	is := C.cairo_image_surface_create_from_png_stream(cairoreadfunct, rp)
	s, err := cNewImageSurface(is)
	S := s.(ImageSurface)
	if err != nil {
		return S, err
	}
	S.registerReader(rp)
	return S, nil
}

//BUG(jmf): ImageSurface: need safe wrapper around get_data

//BUG(jmf): ImageSurface: need image_surface_create_for_data analog(s)

//Format reports the format of the surface.
//
//Originally cairo_image_surface_get_format.
func (i ImageSurface) Format() Format {
	return i.format
}

//Width reports the width of the surface in pixels.
//
//Originally cairo_image_surface_get_width.
func (i ImageSurface) Width() int {
	return i.width
}

//Height reports the height of the surface in pixels.
//
//Originally cairo_image_surface_get_height.
func (i ImageSurface) Height() int {
	return i.height
}

//Stride reports the stride of the image surface in number of bytes.
//
//Originally cairo_image_surface_get_stride.
func (i ImageSurface) Stride() int {
	return i.stride
}

type mappedImageSurface struct {
	ImageSurface
	from *C.cairo_surface_t
}

func newMappedImageSurface(s, from *C.cairo_surface_t) (m mappedImageSurface, err error) {
	im, err1 := cNewImageSurface(s)
	if err1 != nil {
		err = err1
		return
	}
	//Clear default finalizer so GC doesn't call surface_destroy.
	//We do not set a new finalizer on mappedImageSurface.Close,
	//because that would not do the right thing and the user is expected to close
	//manually when done.
	runtime.SetFinalizer(m.XtensionSurface, nil)
	m = mappedImageSurface{
		ImageSurface: im.(ImageSurface),
		from:         from,
	}
	err = m.Err()
	if err != nil {
		m.s = nil
	}
	return
}

func (m mappedImageSurface) Close() error {
	err := m.Err()
	C.cairo_surface_unmap_image(m.from, m.s)
	m.from = nil
	m.s = nil
	return err
}
<|endoftext|>


<commit_before>package project

var (
	description        = "The aws-operator manages Kubernetes clusters running on AWS."
	gitSHA             = "n/a"
	name        string = "aws-operator"
	source      string = "https://github.com/giantswarm/aws-operator"
	version            = "11.14.0"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<commit_msg>Bump version to 11.14.1-dev (#3351)<commit_after>package project

var (
	description        = "The aws-operator manages Kubernetes clusters running on AWS."
	gitSHA             = "n/a"
	name        string = "aws-operator"
	source      string = "https://github.com/giantswarm/aws-operator"
	version            = "11.14.1-dev"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<|endoftext|>


<commit_before>//TODO make it a key value storage
package server

import (
	"bufio"
	"fmt"
	"log"
	"math/rand"
	"net/rpc"
	"os"
	"sync"
	"time"

	"github.com/mateusbraga/freestore/pkg/view"
)

var register Value

//  ---------- RPC Requests -------------
type ClientRequest int

func (r *ClientRequest) Read(clientView *view.View, reply *Value) error {
	if !clientView.Equal(currentView.View()) {
		log.Printf("Got old view: %v, sending new one: %v\n", clientView, currentView.View())
		reply.Err = view.OldViewError{NewView: currentView.View()}
		return nil
	}

	register.mu.RLock()
	defer register.mu.RUnlock()

	reply.Value = register.Value
	reply.Timestamp = register.Timestamp

	throughput++

	return nil
}

func (r *ClientRequest) Write(value Value, reply *Value) error {
	if !value.View.Equal(currentView.View()) {
		log.Printf("Got old view: %v, sending new one: %v\n", value.View, currentView.View())
		reply.Err = view.OldViewError{NewView: currentView.View()}
		return nil
	}

	register.mu.Lock()
	defer register.mu.Unlock()

	// Two writes with the same timestamp -> give preference to first one. This makes the Write operation idempotent and still read/write coherent.
	if value.Timestamp > register.Timestamp {
		register.Value = value.Value
		register.Timestamp = value.Timestamp
	}

	return nil
}

func (r *ClientRequest) GetCurrentView(value int, reply *view.View) error {
	*reply = *currentView.View()
	log.Println("Done GetCurrentView request")
	return nil
}

// --------- Init ---------
func init() {
	register.mu.Lock() // The register starts locked
	register.Value = nil
	register.Timestamp = 0
}

func init() {
	rpc.Register(new(ClientRequest))
}

// --------- Types ---------
type Value struct {
	Value     interface{}
	Timestamp int

	View *view.View
	Err  error

	mu sync.RWMutex
}

var throughput uint64
var throughputBuffer = make(map[time.Time]uint64, 70)

func collectThroughputWorker() {
	writeLength := rand.Intn(20)
	var lastThroughput uint64

	for now := range time.Tick(time.Second) {
		aux := throughput
		throughputBuffer[now] = aux - lastThroughput
		lastThroughput = aux

		if len(throughputBuffer) == writeLength {
			writeLength = rand.Intn(20)
			saveThroughput()
		}
	}
}

func saveThroughput() {
	filename := fmt.Sprintf("/proj/freestore/throughputs_%v.txt", currentView.View().GetProcessPosition(thisProcess))
	file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0664)
	if err != nil {
		//log.Println(err)
		for loopTime, loopThroughput := range throughputBuffer {
			fmt.Printf("%v %v %v\n", thisProcess, loopThroughput, loopTime.Format(time.RFC3339))
			delete(throughputBuffer, loopTime)
		}
		return
	}
	defer file.Close()

	w := bufio.NewWriter(file)
	defer w.Flush()

	for loopTime, loopThroughput := range throughputBuffer {
		if _, err = w.Write([]byte(fmt.Sprintf("%v %v %v\n", thisProcess, loopThroughput, loopTime.Format(time.RFC3339)))); err != nil {
			log.Fatalln(err)
		}
		delete(throughputBuffer, loopTime)
	}
}

func init() {
	go collectThroughputWorker()
}
<commit_msg>Change throughput output file<commit_after>//TODO make it a key value storage
package server

import (
	"bufio"
	"fmt"
	"log"
	"math/rand"
	"net/rpc"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/mateusbraga/freestore/pkg/view"
)

var register Value

//  ---------- RPC Requests -------------
type ClientRequest int

func (r *ClientRequest) Read(clientView *view.View, reply *Value) error {
	if !clientView.Equal(currentView.View()) {
		log.Printf("Got old view: %v, sending new one: %v\n", clientView, currentView.View())
		reply.Err = view.OldViewError{NewView: currentView.View()}
		return nil
	}

	register.mu.RLock()
	defer register.mu.RUnlock()

	reply.Value = register.Value
	reply.Timestamp = register.Timestamp

	throughput++

	return nil
}

func (r *ClientRequest) Write(value Value, reply *Value) error {
	if !value.View.Equal(currentView.View()) {
		log.Printf("Got old view: %v, sending new one: %v\n", value.View, currentView.View())
		reply.Err = view.OldViewError{NewView: currentView.View()}
		return nil
	}

	register.mu.Lock()
	defer register.mu.Unlock()

	// Two writes with the same timestamp -> give preference to first one. This makes the Write operation idempotent and still read/write coherent.
	if value.Timestamp > register.Timestamp {
		register.Value = value.Value
		register.Timestamp = value.Timestamp
	}

	return nil
}

func (r *ClientRequest) GetCurrentView(value int, reply *view.View) error {
	*reply = *currentView.View()
	log.Println("Done GetCurrentView request")
	return nil
}

// --------- Init ---------
func init() {
	register.mu.Lock() // The register starts locked
	register.Value = nil
	register.Timestamp = 0
}

func init() {
	rpc.Register(new(ClientRequest))
}

// --------- Types ---------
type Value struct {
	Value     interface{}
	Timestamp int

	View *view.View
	Err  error

	mu sync.RWMutex
}

var throughput uint64
var throughputBuffer = make(map[time.Time]uint64, 70)

func collectThroughputWorker() {
	writeLength := rand.Intn(20)
	var lastThroughput uint64

	for now := range time.Tick(time.Second) {
		aux := throughput
		throughputBuffer[now] = aux - lastThroughput
		lastThroughput = aux

		if len(throughputBuffer) == writeLength {
			writeLength = rand.Intn(20)
			saveThroughput()
		}
	}
}

func saveThroughput() {
	filename := fmt.Sprintf("/proj/freestore/throughputs_%v.txt", strings.Split(thisProcess.Addr, ":")[0])
	file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0664)
	if err != nil {
		//log.Println(err)
		for loopTime, loopThroughput := range throughputBuffer {
			fmt.Printf("%v %v %v\n", thisProcess, loopThroughput, loopTime.Format(time.RFC3339))
			delete(throughputBuffer, loopTime)
		}
		return
	}
	defer file.Close()

	w := bufio.NewWriter(file)
	defer w.Flush()

	for loopTime, loopThroughput := range throughputBuffer {
		if _, err = w.Write([]byte(fmt.Sprintf("%v %v %v\n", thisProcess, loopThroughput, loopTime.Format(time.RFC3339)))); err != nil {
			log.Fatalln(err)
		}
		delete(throughputBuffer, loopTime)
	}
}

func init() {
	go collectThroughputWorker()
}
<|endoftext|>


<commit_before>package sync

import (
	"testing"
)

func TestanyExecutableBitSet(t *testing.T) {
	if anyExecutableBitSet(0666) {
		t.Error("executable bits detected")
	}
	if !anyExecutableBitSet(0766) {
		t.Error("user executable bit not detected")
	}
	if !anyExecutableBitSet(0676) {
		t.Error("group executable bit not detected")
	}
	if !anyExecutableBitSet(0667) {
		t.Error("others executable bit not detected")
	}
}

func TeststripExecutableBits(t *testing.T) {
	if stripExecutableBits(0777) != 0666 {
		t.Error("executable bits not stripped")
	}
	if stripExecutableBits(0766) != 0666 {
		t.Error("user executable bit not stripped")
	}
	if stripExecutableBits(0676) != 0666 {
		t.Error("group executable bit not stripped")
	}
	if stripExecutableBits(0667) != 0666 {
		t.Error("others executable bit not stripped")
	}
}

func TestmarkExecutableForReaders(t *testing.T) {
	if markExecutableForReaders(0222) != 0222 {
		t.Error("erroneous executable bits added")
	}
	if markExecutableForReaders(0622) != 0722 {
		t.Error("incorrect executable bits added for user-readable file")
	}
	if markExecutableForReaders(0262) != 0272 {
		t.Error("incorrect executable bits added for group-readable file")
	}
	if markExecutableForReaders(0226) != 0227 {
		t.Error("incorrect executable bits added for others-readable file")
	}
}
<commit_msg>Fixed test name casing.<commit_after>package sync

import (
	"testing"
)

func TestAnyExecutableBitSet(t *testing.T) {
	if anyExecutableBitSet(0666) {
		t.Error("executable bits detected")
	}
	if !anyExecutableBitSet(0766) {
		t.Error("user executable bit not detected")
	}
	if !anyExecutableBitSet(0676) {
		t.Error("group executable bit not detected")
	}
	if !anyExecutableBitSet(0667) {
		t.Error("others executable bit not detected")
	}
}

func TestStripExecutableBits(t *testing.T) {
	if stripExecutableBits(0777) != 0666 {
		t.Error("executable bits not stripped")
	}
	if stripExecutableBits(0766) != 0666 {
		t.Error("user executable bit not stripped")
	}
	if stripExecutableBits(0676) != 0666 {
		t.Error("group executable bit not stripped")
	}
	if stripExecutableBits(0667) != 0666 {
		t.Error("others executable bit not stripped")
	}
}

func TestMarkExecutableForReaders(t *testing.T) {
	if markExecutableForReaders(0222) != 0222 {
		t.Error("erroneous executable bits added")
	}
	if markExecutableForReaders(0622) != 0722 {
		t.Error("incorrect executable bits added for user-readable file")
	}
	if markExecutableForReaders(0262) != 0272 {
		t.Error("incorrect executable bits added for group-readable file")
	}
	if markExecutableForReaders(0226) != 0227 {
		t.Error("incorrect executable bits added for others-readable file")
	}
}
<|endoftext|>


<commit_before>package immap

import (
	"context"
	"log"
)

const (
	// DELETED is a delete marker used by ImutMap.
	DELETED = "marked as deleted"
)

// NewImutMapper returns a new instance of implementing ImutMapper interface.
func NewImutMapper(ctx context.Context) (ImutMapper, context.CancelFunc) {
	canCtx, terminate := context.WithCancel(ctx)
	iPack := &ImutMap{canCtx, make(chan *mapPack, 1)}

	go iPack.runLoop()
	return iPack, terminate
}

// RunLoop is the ImutMapper's map requests processing loop.
func (imap *ImutMap) runLoop() {

	mapList := make([]IntfMap, 0)

	for {
	SelAgain:
		select {
		case <-imap.Done():
			return
		case opMsg := <-imap.cChan:
			switch opMsg.op {
			case ADD_KEY:
				for counter := 0; counter <= len(mapList)-1; counter++ {
					indMap := mapList[counter]
					value, exists := indMap[opMsg.key]
					if !exists || (exists && value == DELETED) {
						indMap[opMsg.key] = opMsg.value
						opMsg.ret <- retPack{nil, indMap}
						break SelAgain
					}
				}
				mapList = append(mapList, make(IntfMap))
				lpage := mapList[len(mapList)-1]
				lpage[opMsg.key] = opMsg.value
				opMsg.ret <- retPack{nil, lpage}
			case CHECK_KEY:
				for counter := len(mapList) - 1; counter >= 0; counter-- {
					indMap := mapList[counter]
					if value, exists := indMap[opMsg.key]; exists {
						if value == DELETED {
							opMsg.ret <- retPack{nil, nil}
							break SelAgain
						} else {
							opMsg.ret <- retPack{value, indMap}
							break SelAgain
						}
					}
				}
				opMsg.ret <- retPack{nil, nil}
			case DEL_KEY:
				for counter := len(mapList) - 1; counter >= 0; counter-- {
					indMap := mapList[counter]
					value, exists := indMap[opMsg.key]
					if exists {
						if value == DELETED {
							// Do nothing
							log.Printf("Key %+v already deleted", opMsg.key)
							break SelAgain

						} else {
							indMap[opMsg.key] = DELETED
							break SelAgain
						}
					}
				}

			}
		}
	}

}

// Add method allows one to add new keys.
// Returns a reference to IntfMap
func (imap *ImutMap) Add(key, value interface{}) IntfMap {
	iPack := &mapPack{ADD_KEY, key, value, make(chan retPack, 1)}
	imap.cChan <- iPack

	pack := <-iPack.ret

	return pack.mapRef
}

// Exists method allows to check and return the key.
// Also, returns a reference to IntfMap.
func (imap *ImutMap) Exists(key interface{}) (interface{}, bool, IntfMap) {
	iPack := &mapPack{CHECK_KEY, key, nil, make(chan retPack, 1)}
	imap.cChan <- iPack
	val := <-iPack.ret

	if val.value == nil {
		return nil, false, nil
	}
	return val.value, true, val.mapRef
}

// Delete method allows to delete keys.
func (imap *ImutMap) Delete(key interface{}) {
	iPack := &mapPack{DEL_KEY, key, nil, nil}
	imap.cChan <- iPack
}
<commit_msg>Remove the extraneous looping from immut map<commit_after>package immap

import (
	"context"
)

const (
	// DELETED is a delete marker used by ImutMap.
	DELETED = "marked as deleted"
)

// NewImutMapper returns a new instance of implementing ImutMapper interface.
func NewImutMapper(ctx context.Context) (ImutMapper, context.CancelFunc) {
	canCtx, terminate := context.WithCancel(ctx)
	iPack := &ImutMap{canCtx, make(chan *mapPack, 1)}

	go iPack.runLoop()
	return iPack, terminate
}

// RunLoop is the ImutMapper's map requests processing loop.
func (imap *ImutMap) runLoop() {

	mapList := make([]IntfMap, 0)

	intMap := make(map[interface{}][]int)
	var lastInd, lastMapInd int
	var iPoint int

	for {
	SelAgain:
		select {
		case <-imap.Done():
			return
		case opMsg := <-imap.cChan:
			startSlice, exists := intMap[opMsg.key]

			if !exists {
				lastInd = -1
			} else {
				lastInd = startSlice[len(startSlice)-1]
			}
			switch opMsg.op {
			case ADD_KEY:

				expand := false

				if len(mapList) > 0 {
					lastMapInd = len(mapList) - 1
				} else {
					expand = true
				}

				lastInd = lastInd + 1

				if lastInd > lastMapInd || expand {
					mapList = append(mapList, make(IntfMap))
					iPoint = len(mapList) - 1
				} else {
					iPoint = lastInd
				}

				lpage := mapList[iPoint]
				lpage[opMsg.key] = opMsg.value

				intMap[opMsg.key] = append(intMap[opMsg.key], lastInd)

				opMsg.ret <- retPack{nil, lpage}
			case CHECK_KEY:
				indMap := mapList[lastInd]
				if value, exists := indMap[opMsg.key]; exists {
					if value == DELETED {
						panic("DELETED value should not be here")
					} else {
						opMsg.ret <- retPack{value, indMap}
						break SelAgain
					}
				}
				opMsg.ret <- retPack{nil, nil}
			case DEL_KEY:
				indMap := mapList[lastInd]
				value, exists := indMap[opMsg.key]
				if exists {
					if value == DELETED {
						panic("DELETED value should not be here")

					} else {
						indMap[opMsg.key] = DELETED
						intMap[opMsg.key] = intMap[opMsg.key][:len(intMap[opMsg.key])-1]
						break SelAgain
					}
				}

			}
		}
	}

}

// Add method allows one to add new keys.
// Returns a reference to IntfMap
func (imap *ImutMap) Add(key, value interface{}) IntfMap {
	iPack := &mapPack{ADD_KEY, key, value, make(chan retPack, 1)}
	imap.cChan <- iPack

	pack := <-iPack.ret

	return pack.mapRef
}

// Exists method allows to check and return the key.
// Also, returns a reference to IntfMap.
func (imap *ImutMap) Exists(key interface{}) (interface{}, bool, IntfMap) {
	iPack := &mapPack{CHECK_KEY, key, nil, make(chan retPack, 1)}
	imap.cChan <- iPack
	val := <-iPack.ret

	if val.value == nil {
		return nil, false, nil
	}
	return val.value, true, val.mapRef
}

// Delete method allows to delete keys.
func (imap *ImutMap) Delete(key interface{}) {
	iPack := &mapPack{DEL_KEY, key, nil, nil}
	imap.cChan <- iPack
}
<|endoftext|>


<commit_before><commit_msg>AWS: Enable ICMP Type 3 Code 4 for ELBs<commit_after><|endoftext|>


<commit_before>package gorobdd

import (
	"fmt"
	"reflect"
)

type NodeLabel string

// Bdd, or Binary Decision Diagram is a balanced binary tree, with branches labelled True and False.
// The leafs are also labeled True and Fal
// The tree is |vocabulary| deep, and each "ply" is corresponds to a name in the vocabulary.
// The order in which vocabulary names match the plys can be changed.
type BDD struct {
	// Names for all the plys.
	Vocabulary []NodeLabel
	*node
}

func (b BDD) String() string {
	return b.node.String(b.Vocabulary)
}

func FromTuples(voc []NodeLabel, tuples [][]bool)  (*BDD, error) {
	bdd := False(voc)
	for _, t:= range tuples {
		if len(t) != len(voc) {
			return nil, fmt.Errorf("Length of tuple %v does not match vocabulary (len: %d)", t, len(voc))
		}
		p := bdd.node
		for _, b := range t {
			if b {
				p = p.True
			} else {
				p = p.False
			}
		}
		p.Value = true
	}
	return bdd, nil
}

func False(voc []NodeLabel) (*BDD) {
	return &BDD{voc, uniform(len(voc), false)}
}

func True(voc []NodeLabel) (*BDD) {
	return &BDD{voc, uniform(len(voc), true)}
}

func Equal(a *BDD, b *BDD) (bool, error) {
	return a.Equal(b)
}

func And(a *BDD, b *BDD) (*BDD, error) {
	return a.And(b)
}

func Or(a *BDD, b *BDD) (*BDD, error) {
	return a.Or(b)
}

func Not(a *BDD) (*BDD, error) {
	return a.Not()
}

func (a *BDD) Equal(b *BDD) (bool, error) {
	if ! reflect.DeepEqual(a.Vocabulary, b.Vocabulary) {
		return false, fmt.Errorf("Mismatched vocabularies in And: %v, %v", a.Vocabulary, b.Vocabulary)
	}
	return reflect.DeepEqual(a, b), nil
}

func (a *BDD) And(b *BDD) (*BDD, error) {
	if ! reflect.DeepEqual(a.Vocabulary, b.Vocabulary) {
		return nil, fmt.Errorf("Mismatched vocabularies in And: %v, %v", a.Vocabulary, b.Vocabulary)
	}
	return &BDD{a.Vocabulary, a.node.And(b.node)}, nil
}

func (a *BDD) Or(b *BDD) (*BDD, error) {
	if ! reflect.DeepEqual(a.Vocabulary, b.Vocabulary) {
		return nil, fmt.Errorf("Mismatched vocabularies in And: %v, %v", a.Vocabulary, b.Vocabulary)
	}
	return &BDD{a.Vocabulary, a.node.Or(b.node)}, nil
}

func (a *BDD) Not() (*BDD, error) {
	return &BDD{a.Vocabulary, a.node.Not()}, nil
}

func uniform(depth int, v bool) *node {
	if depth == 0 {
		return &node {
			Type: leafNodeType,
			leafNode: leafNode { v },
		}
	}
	return &node {
		Type: internalNodeType,
		internalNode: internalNode{True: uniform(depth - 1, v), False: uniform(depth - 1, v)},
	}
}

func (a *node) And(b*node) *node {
	return &node{
		Type: leafNodeType,
		leafNode: leafNode{ a.Value && b.Value },
	}
}

func (a *node) Or(b*node) *node {
	return &node{
		Type: leafNodeType,
		leafNode: leafNode{ a.Value || b.Value },
	}
}

func (a *node) Not() *node {
	return &node {
		Type: leafNodeType,
		leafNode: leafNode{ ! a.Value },
	}
}

type nodeType int

const (
	internalNodeType nodeType = iota
	leafNodeType
)

type node struct {
	Type nodeType
	internalNode
	leafNode
}

type internalNode struct {
	True *node
	False *node
}

type leafNode struct {
	Value bool
}

func (n leafNode) String() string {
	if n.Value {
		return "T"
	} else {
		return "F"
	}
}

func (n internalNode) String(v []NodeLabel) string {
	return fmt.Sprintf("(%v/T: %v, %v/F: %v)", v[0], n.True.String(v[1:]), v[0], n.False.String(v[1:]))
}

func (n node) String(v []NodeLabel) string {
	switch(n.Type) {
	case internalNodeType:
		return n.internalNode.String(v)
	case leafNodeType:
		return n.leafNode.String()
	default:
		return fmt.Sprintf("Garbage node type: %v", n.Type)
	}
}
<commit_msg>Minor error message fixes<commit_after>package gorobdd

import (
	"fmt"
	"reflect"
)

type NodeLabel string

// Bdd, or Binary Decision Diagram is a balanced binary tree, with branches labelled True and False.
// The leafs are also labeled True and Fal
// The tree is |vocabulary| deep, and each "ply" is corresponds to a name in the vocabulary.
// The order in which vocabulary names match the plys can be changed.
type BDD struct {
	// Names for all the plys.
	Vocabulary []NodeLabel
	*node
}

func (b BDD) String() string {
	return b.node.String(b.Vocabulary)
}

func FromTuples(voc []NodeLabel, tuples [][]bool)  (*BDD, error) {
	bdd := False(voc)
	for _, t:= range tuples {
		if len(t) != len(voc) {
			return nil, fmt.Errorf("Length of tuple %v does not match vocabulary (len: %d)", t, len(voc))
		}
		p := bdd.node
		for _, b := range t {
			if b {
				p = p.True
			} else {
				p = p.False
			}
		}
		p.Value = true
	}
	return bdd, nil
}

func False(voc []NodeLabel) (*BDD) {
	return &BDD{voc, uniform(len(voc), false)}
}

func True(voc []NodeLabel) (*BDD) {
	return &BDD{voc, uniform(len(voc), true)}
}

func Equal(a *BDD, b *BDD) (bool, error) {
	return a.Equal(b)
}

func And(a *BDD, b *BDD) (*BDD, error) {
	return a.And(b)
}

func Or(a *BDD, b *BDD) (*BDD, error) {
	return a.Or(b)
}

func Not(a *BDD) (*BDD, error) {
	return a.Not()
}

func (a *BDD) Equal(b *BDD) (bool, error) {
	if ! reflect.DeepEqual(a.Vocabulary, b.Vocabulary) {
		return false, fmt.Errorf("Mismatched vocabularies in Equal: %v, %v", a.Vocabulary, b.Vocabulary)
	}
	return reflect.DeepEqual(a, b), nil
}

func (a *BDD) And(b *BDD) (*BDD, error) {
	if ! reflect.DeepEqual(a.Vocabulary, b.Vocabulary) {
		return nil, fmt.Errorf("Mismatched vocabularies in And: %v, %v", a.Vocabulary, b.Vocabulary)
	}
	return &BDD{a.Vocabulary, a.node.And(b.node)}, nil
}

func (a *BDD) Or(b *BDD) (*BDD, error) {
	if ! reflect.DeepEqual(a.Vocabulary, b.Vocabulary) {
		return nil, fmt.Errorf("Mismatched vocabularies in Or: %v, %v", a.Vocabulary, b.Vocabulary)
	}
	return &BDD{a.Vocabulary, a.node.Or(b.node)}, nil
}

func (a *BDD) Not() (*BDD, error) {
	return &BDD{a.Vocabulary, a.node.Not()}, nil
}

func uniform(depth int, v bool) *node {
	if depth == 0 {
		return &node {
			Type: leafNodeType,
			leafNode: leafNode { v },
		}
	}
	return &node {
		Type: internalNodeType,
		internalNode: internalNode{True: uniform(depth - 1, v), False: uniform(depth - 1, v)},
	}
}

func (a *node) And(b*node) *node {
	return &node{
		Type: leafNodeType,
		leafNode: leafNode{ a.Value && b.Value },
	}
}

func (a *node) Or(b*node) *node {
	return &node{
		Type: leafNodeType,
		leafNode: leafNode{ a.Value || b.Value },
	}
}

func (a *node) Not() *node {
	return &node {
		Type: leafNodeType,
		leafNode: leafNode{ ! a.Value },
	}
}

type nodeType int

const (
	internalNodeType nodeType = iota
	leafNodeType
)

type node struct {
	Type nodeType
	internalNode
	leafNode
}

type internalNode struct {
	True *node
	False *node
}

type leafNode struct {
	Value bool
}

func (n leafNode) String() string {
	if n.Value {
		return "T"
	} else {
		return "F"
	}
}

func (n internalNode) String(v []NodeLabel) string {
	return fmt.Sprintf("(%v/T: %v, %v/F: %v)", v[0], n.True.String(v[1:]), v[0], n.False.String(v[1:]))
}

func (n node) String(v []NodeLabel) string {
	switch(n.Type) {
	case internalNodeType:
		return n.internalNode.String(v)
	case leafNodeType:
		return n.leafNode.String()
	default:
		return fmt.Sprintf("Garbage node type: %v", n.Type)
	}
}
<|endoftext|>


<commit_before>package opencl

import (
	"fmt"
	"image"
	"image/png"
	"math"
	"os"
	"time"

	"github.com/achilleasa/go-pathtrace/tracer"
	"github.com/achilleasa/go-pathtrace/types"
)

func appendPipeline(stage pipelineStage) tracer.Stage {
	return func(t tracer.Tracer) error {
		tr, ok := t.(*clTracer)
		if !ok {
			return ErrInvalidOption
		}

		tr.pipeline = append(tr.pipeline, stage)
		return nil
	}
}

// Clear the accumulator buffer.
func ClearAccumulator() tracer.Stage {
	return appendPipeline(
		func(tr *clTracer, blockReq *tracer.BlockRequest) (time.Duration, error) {
			return tr.resources.ClearAccumulator(blockReq)
		},
	)
}

// Use a perspective camera for the primary ray generation stage.
func PerspectiveCamera() tracer.Stage {
	return appendPipeline(
		func(tr *clTracer, blockReq *tracer.BlockRequest) (time.Duration, error) {
			return tr.resources.GeneratePrimaryRays(blockReq, tr.sceneData.Camera.Position, tr.sceneData.Camera.Frustrum)
		},
	)
}

// Apply simple Reinhard tone-mapping.
func TonemapSimpleReinhard(exposure float32) tracer.Stage {
	return appendPipeline(
		func(tr *clTracer, blockReq *tracer.BlockRequest) (time.Duration, error) {
			return tr.resources.TonemapSimpleReinhard(blockReq, exposure)
		},
	)
}

// Use a montecarlo pathtracer implementation.
func MonteCarloIntegrator(numBounces uint32) tracer.Stage {
	return appendPipeline(
		func(tr *clTracer, blockReq *tracer.BlockRequest) (time.Duration, error) {
			var err error

			start := time.Now()
			numPixels := int(blockReq.FrameW * blockReq.BlockH)
			numEmissives := uint32(len(tr.sceneData.EmissivePrimitives))

			var activeRayBuf uint32 = 0

			// Intersect primary rays outside of the loop
			// TODO: Use packet query
			_, err = tr.resources.RayIntersectionQuery(activeRayBuf, numPixels)
			if err != nil {
				return time.Since(start), err
			}

			_, err = debugPrimaryRayIntersections(tr.resources, blockReq, "debug-00-primary-intersections.png")
			if err != nil {
				return time.Since(start), err
			}

			var bounce uint32
			for bounce = 0; bounce < numBounces; bounce++ {
				tr.logger.Noticef("[bounce %02d] intersected rays: %d", bounce, readCounter(tr.resources, activeRayBuf))

				// Shade hits
				_, err = tr.resources.ShadeHits(bounce, 0, numEmissives, activeRayBuf, numPixels)
				if err != nil {
					return time.Since(start), err
				}

				// Shade misses on first bounce
				if bounce == 0 {
					_, err = tr.resources.ShadeMisses(tr.sceneData.SceneDiffuseMatIndex, activeRayBuf, numPixels)
					if err != nil {
						return time.Since(start), err
					}
				}

				_, err = debugThroughput(tr.resources, blockReq, fmt.Sprintf("debug-01-throughput-bounce-%d.png", bounce))
				if err != nil {
					return time.Since(start), err
				}

				_, err = debugEmissiveSamples(tr.resources, blockReq, fmt.Sprintf("debug-01-emissive-samples-bounce-%d.png", bounce))
				if err != nil {
					return time.Since(start), err
				}

				tr.logger.Noticef("[bounce %02d] generated indirect rays: %d", bounce, readCounter(tr.resources, 1-activeRayBuf))
				tr.logger.Noticef("[bounce %02d] generated occlusion rays: %d", bounce, readCounter(tr.resources, 2))

				// Process intersections for occlusion rays and accumulate emissive samples for non occluded paths
				_, err = tr.resources.RayIntersectionTest(2, numPixels)
				if err != nil {
					return time.Since(start), err
				}
				_, err = tr.resources.AccumulateEmissiveSamples(2, numPixels)
				if err != nil {
					return time.Since(start), err
				}

				// Process intersections for indirect rays
				activeRayBuf = 1 - activeRayBuf
				_, err = tr.resources.RayIntersectionQuery(activeRayBuf, numPixels)
				if err != nil {
					return time.Since(start), err
				}
			}

			return time.Since(start), nil
		},
	)
}

type ray struct {
	origin types.Vec4
	dir    types.Vec4
}

type intersection struct {
	wuvt         types.Vec4
	meshInstance uint32
	triIndex     uint32
	_padding1    uint32
	_padding2    uint32
}

type rayPath struct {
	throughput types.Vec4
	status     uint32
	pixelIndex uint32
	reserved   [2]uint32
}

// Debug primary ray intersections.
func debugPrimaryRayIntersections(dr *deviceResources, blockReq *tracer.BlockRequest, imgFile string) (time.Duration, error) {
	start := time.Now()

	f, err := os.Create(imgFile)
	if err != nil {
		return time.Since(start), err
	}
	defer f.Close()

	im := image.NewRGBA(image.Rect(0, 0, int(blockReq.FrameW), int(blockReq.FrameH)))

	data, err := dr.buffers.Intersections.ReadDataIntoSlice(make([]intersection, 0))
	if err != nil {
		return time.Since(start), err
	}
	intersections := data.([]intersection)

	data, err = dr.buffers.Paths.ReadDataIntoSlice(make([]rayPath, 0))
	if err != nil {
		return time.Since(start), err
	}
	paths := data.([]rayPath)

	// Find max depth
	var maxDepth float32 = 0
	for _, hit := range intersections {
		if hit.wuvt[3] < math.MaxFloat32 && hit.wuvt[3] > maxDepth {
			maxDepth = hit.wuvt[3]
		}
	}
	maxDepth++

	// convert depth to 0-255 range
	for hitIndex, hit := range intersections {
		if hit.wuvt[3] == math.MaxFloat32 {
			continue
		}

		normDepth := uint8(255.0 * (1.0 - hit.wuvt[3]/maxDepth))
		offset := paths[hitIndex].pixelIndex << 2
		im.Pix[offset] = normDepth
		im.Pix[offset+1] = normDepth
		im.Pix[offset+2] = normDepth
		im.Pix[offset+3] = 255 // alpha
	}

	return time.Since(start), png.Encode(f, im)
}

// Debug path throughput.
func debugThroughput(dr *deviceResources, blockReq *tracer.BlockRequest, imgFile string) (time.Duration, error) {
	start := time.Now()

	f, err := os.Create(imgFile)
	if err != nil {
		return time.Since(start), err
	}
	defer f.Close()

	im := image.NewRGBA(image.Rect(0, 0, int(blockReq.FrameW), int(blockReq.FrameH)))

	data, err := dr.buffers.Paths.ReadDataIntoSlice(make([]rayPath, 0))
	if err != nil {
		return time.Since(start), err
	}
	paths := data.([]rayPath)

	for _, path := range paths {
		offset := path.pixelIndex << 2
		scaledThroughput := path.throughput.Mul(255.0)
		im.Pix[offset] = uint8(scaledThroughput[0])
		im.Pix[offset+1] = uint8(scaledThroughput[1])
		im.Pix[offset+2] = uint8(scaledThroughput[2])
		im.Pix[offset+3] = 255 // alpha
	}

	return time.Since(start), png.Encode(f, im)
}

// Debug emissive samples.
func debugEmissiveSamples(dr *deviceResources, blockReq *tracer.BlockRequest, imgFile string) (time.Duration, error) {
	start := time.Now()

	f, err := os.Create(imgFile)
	if err != nil {
		return time.Since(start), err
	}
	defer f.Close()

	im := image.NewRGBA(image.Rect(0, 0, int(blockReq.FrameW), int(blockReq.FrameH)))

	data, err := dr.buffers.Rays[2].ReadDataIntoSlice(make([]ray, 0))
	if err != nil {
		return time.Since(start), err
	}
	rays := data.([]ray)

	data, err = dr.buffers.EmissiveSamples.ReadDataIntoSlice(make([]types.Vec4, 0))
	if err != nil {
		return time.Since(start), err
	}
	emissiveSamples := data.([]types.Vec4)

	data, err = dr.buffers.Paths.ReadDataIntoSlice(make([]rayPath, 0))
	if err != nil {
		return time.Since(start), err
	}
	paths := data.([]rayPath)

	numRays := int(readCounter(dr, 2))
	for rayIndex := 0; rayIndex < numRays; rayIndex++ {
		pathIndex := int(rays[rayIndex].dir[3])
		offset := paths[pathIndex].pixelIndex << 2
		scaledThroughput := types.MinVec3(emissiveSamples[rayIndex].Mul(255.0).Vec3(), types.Vec3{255.0, 255.0, 255.0})
		im.Pix[offset] = uint8(scaledThroughput[0])
		im.Pix[offset+1] = uint8(scaledThroughput[1])
		im.Pix[offset+2] = uint8(scaledThroughput[2])
		im.Pix[offset+3] = 255 // alpha
	}

	return time.Since(start), png.Encode(f, im)
}

// Dump a copy of the RGBA framebuffer.
func DebugFrameBuffer(imgFile string) tracer.Stage {
	return appendPipeline(
		func(tr *clTracer, blockReq *tracer.BlockRequest) (time.Duration, error) {
			start := time.Now()
			f, err := os.Create(imgFile)
			if err != nil {
				return time.Since(start), err
			}
			defer f.Close()

			im := image.NewRGBA(image.Rect(0, 0, int(blockReq.FrameW), int(blockReq.FrameH)))

			pix, err := tr.resources.buffers.FrameBuffer.ReadDataIntoSlice(make([]uint8, 0))
			if err != nil {
				return time.Since(start), err
			}
			im.Pix = pix.([]uint8)

			return time.Since(start), png.Encode(f, im)
		},
	)
}

func readCounter(dr *deviceResources, counterIndex uint32) uint32 {
	out := make([]uint32, 1)
	dr.buffers.RayCounters[counterIndex].ReadData(0, 0, 4, out)
	return out[0]
}
<commit_msg>Use a PRNG to pass seed values to the ShadeHits kernel<commit_after>package opencl

import (
	"fmt"
	"image"
	"image/png"
	"math"
	"math/rand"
	"os"
	"time"

	"github.com/achilleasa/go-pathtrace/tracer"
	"github.com/achilleasa/go-pathtrace/types"
)

func appendPipeline(stage pipelineStage) tracer.Stage {
	return func(t tracer.Tracer) error {
		tr, ok := t.(*clTracer)
		if !ok {
			return ErrInvalidOption
		}

		tr.pipeline = append(tr.pipeline, stage)
		return nil
	}
}

// Clear the accumulator buffer.
func ClearAccumulator() tracer.Stage {
	return appendPipeline(
		func(tr *clTracer, blockReq *tracer.BlockRequest) (time.Duration, error) {
			return tr.resources.ClearAccumulator(blockReq)
		},
	)
}

// Use a perspective camera for the primary ray generation stage.
func PerspectiveCamera() tracer.Stage {
	return appendPipeline(
		func(tr *clTracer, blockReq *tracer.BlockRequest) (time.Duration, error) {
			return tr.resources.GeneratePrimaryRays(blockReq, tr.sceneData.Camera.Position, tr.sceneData.Camera.Frustrum)
		},
	)
}

// Apply simple Reinhard tone-mapping.
func TonemapSimpleReinhard(exposure float32) tracer.Stage {
	return appendPipeline(
		func(tr *clTracer, blockReq *tracer.BlockRequest) (time.Duration, error) {
			return tr.resources.TonemapSimpleReinhard(blockReq, exposure)
		},
	)
}

// Use a montecarlo pathtracer implementation.
func MonteCarloIntegrator(numBounces uint32) tracer.Stage {
	return appendPipeline(
		func(tr *clTracer, blockReq *tracer.BlockRequest) (time.Duration, error) {
			var err error

			start := time.Now()
			numPixels := int(blockReq.FrameW * blockReq.BlockH)
			numEmissives := uint32(len(tr.sceneData.EmissivePrimitives))

			var activeRayBuf uint32 = 0

			// Intersect primary rays outside of the loop
			// TODO: Use packet query
			_, err = tr.resources.RayIntersectionQuery(activeRayBuf, numPixels)
			if err != nil {
				return time.Since(start), err
			}

			_, err = debugPrimaryRayIntersections(tr.resources, blockReq, "debug-00-primary-intersections.png")
			if err != nil {
				return time.Since(start), err
			}

			var bounce uint32
			for bounce = 0; bounce < numBounces; bounce++ {
				tr.logger.Noticef("[bounce %02d] intersected rays: %d", bounce, readCounter(tr.resources, activeRayBuf))

				// Shade hits
				_, err = tr.resources.ShadeHits(bounce, rand.Uint32(), numEmissives, activeRayBuf, numPixels)
				if err != nil {
					return time.Since(start), err
				}

				// Shade misses on first bounce
				if bounce == 0 {
					_, err = tr.resources.ShadeMisses(tr.sceneData.SceneDiffuseMatIndex, activeRayBuf, numPixels)
					if err != nil {
						return time.Since(start), err
					}
				}

				_, err = debugThroughput(tr.resources, blockReq, fmt.Sprintf("debug-01-throughput-bounce-%d.png", bounce))
				if err != nil {
					return time.Since(start), err
				}

				_, err = debugEmissiveSamples(tr.resources, blockReq, fmt.Sprintf("debug-01-emissive-samples-bounce-%d.png", bounce))
				if err != nil {
					return time.Since(start), err
				}

				tr.logger.Noticef("[bounce %02d] generated indirect rays: %d", bounce, readCounter(tr.resources, 1-activeRayBuf))
				tr.logger.Noticef("[bounce %02d] generated occlusion rays: %d", bounce, readCounter(tr.resources, 2))

				// Process intersections for occlusion rays and accumulate emissive samples for non occluded paths
				_, err = tr.resources.RayIntersectionTest(2, numPixels)
				if err != nil {
					return time.Since(start), err
				}
				_, err = tr.resources.AccumulateEmissiveSamples(2, numPixels)
				if err != nil {
					return time.Since(start), err
				}

				// Process intersections for indirect rays
				activeRayBuf = 1 - activeRayBuf
				_, err = tr.resources.RayIntersectionQuery(activeRayBuf, numPixels)
				if err != nil {
					return time.Since(start), err
				}
			}

			return time.Since(start), nil
		},
	)
}

type ray struct {
	origin types.Vec4
	dir    types.Vec4
}

type intersection struct {
	wuvt         types.Vec4
	meshInstance uint32
	triIndex     uint32
	_padding1    uint32
	_padding2    uint32
}

type rayPath struct {
	throughput types.Vec4
	status     uint32
	pixelIndex uint32
	reserved   [2]uint32
}

// Debug primary ray intersections.
func debugPrimaryRayIntersections(dr *deviceResources, blockReq *tracer.BlockRequest, imgFile string) (time.Duration, error) {
	start := time.Now()

	f, err := os.Create(imgFile)
	if err != nil {
		return time.Since(start), err
	}
	defer f.Close()

	im := image.NewRGBA(image.Rect(0, 0, int(blockReq.FrameW), int(blockReq.FrameH)))

	data, err := dr.buffers.Intersections.ReadDataIntoSlice(make([]intersection, 0))
	if err != nil {
		return time.Since(start), err
	}
	intersections := data.([]intersection)

	data, err = dr.buffers.Paths.ReadDataIntoSlice(make([]rayPath, 0))
	if err != nil {
		return time.Since(start), err
	}
	paths := data.([]rayPath)

	// Find max depth
	var maxDepth float32 = 0
	for _, hit := range intersections {
		if hit.wuvt[3] < math.MaxFloat32 && hit.wuvt[3] > maxDepth {
			maxDepth = hit.wuvt[3]
		}
	}
	maxDepth++

	// convert depth to 0-255 range
	for hitIndex, hit := range intersections {
		if hit.wuvt[3] == math.MaxFloat32 {
			continue
		}

		normDepth := uint8(255.0 * (1.0 - hit.wuvt[3]/maxDepth))
		offset := paths[hitIndex].pixelIndex << 2
		im.Pix[offset] = normDepth
		im.Pix[offset+1] = normDepth
		im.Pix[offset+2] = normDepth
		im.Pix[offset+3] = 255 // alpha
	}

	return time.Since(start), png.Encode(f, im)
}

// Debug path throughput.
func debugThroughput(dr *deviceResources, blockReq *tracer.BlockRequest, imgFile string) (time.Duration, error) {
	start := time.Now()

	f, err := os.Create(imgFile)
	if err != nil {
		return time.Since(start), err
	}
	defer f.Close()

	im := image.NewRGBA(image.Rect(0, 0, int(blockReq.FrameW), int(blockReq.FrameH)))

	data, err := dr.buffers.Paths.ReadDataIntoSlice(make([]rayPath, 0))
	if err != nil {
		return time.Since(start), err
	}
	paths := data.([]rayPath)

	for _, path := range paths {
		offset := path.pixelIndex << 2
		scaledThroughput := path.throughput.Mul(255.0)
		im.Pix[offset] = uint8(scaledThroughput[0])
		im.Pix[offset+1] = uint8(scaledThroughput[1])
		im.Pix[offset+2] = uint8(scaledThroughput[2])
		im.Pix[offset+3] = 255 // alpha
	}

	return time.Since(start), png.Encode(f, im)
}

// Debug emissive samples.
func debugEmissiveSamples(dr *deviceResources, blockReq *tracer.BlockRequest, imgFile string) (time.Duration, error) {
	start := time.Now()

	f, err := os.Create(imgFile)
	if err != nil {
		return time.Since(start), err
	}
	defer f.Close()

	im := image.NewRGBA(image.Rect(0, 0, int(blockReq.FrameW), int(blockReq.FrameH)))

	data, err := dr.buffers.Rays[2].ReadDataIntoSlice(make([]ray, 0))
	if err != nil {
		return time.Since(start), err
	}
	rays := data.([]ray)

	data, err = dr.buffers.EmissiveSamples.ReadDataIntoSlice(make([]types.Vec4, 0))
	if err != nil {
		return time.Since(start), err
	}
	emissiveSamples := data.([]types.Vec4)

	data, err = dr.buffers.Paths.ReadDataIntoSlice(make([]rayPath, 0))
	if err != nil {
		return time.Since(start), err
	}
	paths := data.([]rayPath)

	// Set alpha channel for all pixels
	for i := 3; i < len(im.Pix); i += 4 {
		im.Pix[i] = 255
	}

	numRays := int(readCounter(dr, 2))
	for rayIndex := 0; rayIndex < numRays; rayIndex++ {
		pathIndex := int(rays[rayIndex].dir[3])
		offset := paths[pathIndex].pixelIndex << 2
		scaledThroughput := types.MinVec3(emissiveSamples[rayIndex].Mul(255.0).Vec3(), types.Vec3{255.0, 255.0, 255.0})
		im.Pix[offset] = uint8(scaledThroughput[0])
		im.Pix[offset+1] = uint8(scaledThroughput[1])
		im.Pix[offset+2] = uint8(scaledThroughput[2])
	}

	return time.Since(start), png.Encode(f, im)
}

// Dump a copy of the RGBA framebuffer.
func DebugFrameBuffer(imgFile string) tracer.Stage {
	return appendPipeline(
		func(tr *clTracer, blockReq *tracer.BlockRequest) (time.Duration, error) {
			start := time.Now()
			f, err := os.Create(imgFile)
			if err != nil {
				return time.Since(start), err
			}
			defer f.Close()

			im := image.NewRGBA(image.Rect(0, 0, int(blockReq.FrameW), int(blockReq.FrameH)))

			pix, err := tr.resources.buffers.FrameBuffer.ReadDataIntoSlice(make([]uint8, 0))
			if err != nil {
				return time.Since(start), err
			}
			im.Pix = pix.([]uint8)

			return time.Since(start), png.Encode(f, im)
		},
	)
}

func readCounter(dr *deviceResources, counterIndex uint32) uint32 {
	out := make([]uint32, 1)
	dr.buffers.RayCounters[counterIndex].ReadData(0, 0, 4, out)
	return out[0]
}
<|endoftext|>