<commit_before>package server

import (
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/server/pkg/obj"
)

// Valid object storage backends
const (
	MinioBackendEnvVar     = "MINIO"
	AmazonBackendEnvVar    = "AMAZON"
	GoogleBackendEnvVar    = "GOOGLE"
	MicrosoftBackendEnvVar = "MICROSOFT"
)

var (
	blockSize = 8 * 1024 * 1024 // 8 Megabytes
	// maxBlockSize specifies the maximum block size for any data type
	maxBlockSize = 100 * 1024 * 1024 // 100 MB
)

// APIServer represents and api server.
type APIServer interface {
	pfsclient.APIServer
}

// BlockAPIServer combines BlockAPIServer and ObjectAPIServer.
type BlockAPIServer interface {
	pfsclient.ObjectAPIServer
}

// NewAPIServer creates an APIServer.
func NewAPIServer(address string, etcdAddresses []string, etcdPrefix string, cacheSize int64) (APIServer, error) {
	return newAPIServer(address, etcdAddresses, etcdPrefix, cacheSize)
}

// NewHTTPServer creates an APIServer.
func NewHTTPServer(address string, etcdAddresses []string, etcdPrefix string, cacheSize int64) (*HTTPServer, error) {
	return newHTTPServer(address, etcdAddresses, etcdPrefix, cacheSize)
}

// NewLocalBlockAPIServer creates a BlockAPIServer.
func NewLocalBlockAPIServer(dir string) (BlockAPIServer, error) {
	return newLocalBlockAPIServer(dir)
}

// NewObjBlockAPIServer create a BlockAPIServer from an obj.Client.
func NewObjBlockAPIServer(dir string, cacheBytes int64, etcdAddress string, objClient obj.Client) (BlockAPIServer, error) {
	return newObjBlockAPIServer(dir, cacheBytes, etcdAddress, objClient)
}

// NewBlockAPIServer creates a BlockAPIServer using the credentials it finds in
// the environment
func NewBlockAPIServer(dir string, cacheBytes int64, backend string, etcdAddress string) (BlockAPIServer, error) {
	switch backend {
	case MinioBackendEnvVar:
		// S3 compatible doesn't like leading slashes
		if len(dir) > 0 && dir[0] == '/' {
			dir = dir[1:]
		}
		blockAPIServer, err := newMinioBlockAPIServer(dir, cacheBytes, etcdAddress)
		if err != nil {
			return nil, err
		}
		return blockAPIServer, nil
	case AmazonBackendEnvVar:
		// amazon doesn't like leading slashes
		if len(dir) > 0 && dir[0] == '/' {
			dir = dir[1:]
		}
		blockAPIServer, err := newAmazonBlockAPIServer(dir, cacheBytes, etcdAddress)
		if err != nil {
			return nil, err
		}
		return blockAPIServer, nil
	case GoogleBackendEnvVar:
		// TODO figure out if google likes leading slashses
		blockAPIServer, err := newGoogleBlockAPIServer(dir, cacheBytes, etcdAddress)
		if err != nil {
			return nil, err
		}
		return blockAPIServer, nil
	case MicrosoftBackendEnvVar:
		blockAPIServer, err := newMicrosoftBlockAPIServer(dir, cacheBytes, etcdAddress)
		if err != nil {
			return nil, err
		}
		return blockAPIServer, nil
	default:
		return NewLocalBlockAPIServer(dir)
	}
}
<commit_msg>Adds a comment about cacheSize.<commit_after>package server

import (
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/server/pkg/obj"
)

// Valid object storage backends
const (
	MinioBackendEnvVar     = "MINIO"
	AmazonBackendEnvVar    = "AMAZON"
	GoogleBackendEnvVar    = "GOOGLE"
	MicrosoftBackendEnvVar = "MICROSOFT"
)

var (
	blockSize = 8 * 1024 * 1024 // 8 Megabytes
	// maxBlockSize specifies the maximum block size for any data type
	maxBlockSize = 100 * 1024 * 1024 // 100 MB
)

// APIServer represents and api server.
type APIServer interface {
	pfsclient.APIServer
}

// BlockAPIServer combines BlockAPIServer and ObjectAPIServer.
type BlockAPIServer interface {
	pfsclient.ObjectAPIServer
}

// NewAPIServer creates an APIServer.
// cacheSize is the number of commit trees which will be cached in the server.
func NewAPIServer(address string, etcdAddresses []string, etcdPrefix string, cacheSize int64) (APIServer, error) {
	return newAPIServer(address, etcdAddresses, etcdPrefix, cacheSize)
}

// NewHTTPServer creates an APIServer.
// cacheSize is the number of commit trees which will be cached in the server.
func NewHTTPServer(address string, etcdAddresses []string, etcdPrefix string, cacheSize int64) (*HTTPServer, error) {
	return newHTTPServer(address, etcdAddresses, etcdPrefix, cacheSize)
}

// NewLocalBlockAPIServer creates a BlockAPIServer.
func NewLocalBlockAPIServer(dir string) (BlockAPIServer, error) {
	return newLocalBlockAPIServer(dir)
}

// NewObjBlockAPIServer create a BlockAPIServer from an obj.Client.
func NewObjBlockAPIServer(dir string, cacheBytes int64, etcdAddress string, objClient obj.Client) (BlockAPIServer, error) {
	return newObjBlockAPIServer(dir, cacheBytes, etcdAddress, objClient)
}

// NewBlockAPIServer creates a BlockAPIServer using the credentials it finds in
// the environment
func NewBlockAPIServer(dir string, cacheBytes int64, backend string, etcdAddress string) (BlockAPIServer, error) {
	switch backend {
	case MinioBackendEnvVar:
		// S3 compatible doesn't like leading slashes
		if len(dir) > 0 && dir[0] == '/' {
			dir = dir[1:]
		}
		blockAPIServer, err := newMinioBlockAPIServer(dir, cacheBytes, etcdAddress)
		if err != nil {
			return nil, err
		}
		return blockAPIServer, nil
	case AmazonBackendEnvVar:
		// amazon doesn't like leading slashes
		if len(dir) > 0 && dir[0] == '/' {
			dir = dir[1:]
		}
		blockAPIServer, err := newAmazonBlockAPIServer(dir, cacheBytes, etcdAddress)
		if err != nil {
			return nil, err
		}
		return blockAPIServer, nil
	case GoogleBackendEnvVar:
		// TODO figure out if google likes leading slashses
		blockAPIServer, err := newGoogleBlockAPIServer(dir, cacheBytes, etcdAddress)
		if err != nil {
			return nil, err
		}
		return blockAPIServer, nil
	case MicrosoftBackendEnvVar:
		blockAPIServer, err := newMicrosoftBlockAPIServer(dir, cacheBytes, etcdAddress)
		if err != nil {
			return nil, err
		}
		return blockAPIServer, nil
	default:
		return NewLocalBlockAPIServer(dir)
	}
}
<|endoftext|>


<commit_before>package dns

import (
	"context"
	"net"
	"strings"

	"github.com/miekg/dns"

	"github.com/datawire/dlib/dcontext"
	"github.com/datawire/dlib/dgroup"
	"github.com/datawire/dlib/dlog"
)

// Server is a DNS server which implements the github.com/miekg/dns Handler interface
type Server struct {
	ctx       context.Context // necessary to make logging work in ServeDNS function
	listeners []net.PacketConn
	fallback  string
	resolve   func(string) []string
}

// NewServer returns a new dns.Server
func NewServer(c context.Context, listeners []net.PacketConn, fallback string, resolve func(string) []string) *Server {
	return &Server{
		ctx:       c,
		listeners: listeners,
		fallback:  fallback,
		resolve:   resolve,
	}
}

// ServeDNS is an implementation of github.com/miekg/dns Handler.ServeDNS.
func (s *Server) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
	c := s.ctx
	defer func() {
		// Closing the response tells the DNS service to terminate
		if c.Err() != nil {
			_ = w.Close()
		}
	}()

	domain := strings.ToLower(r.Question[0].Name)
	switch r.Question[0].Qtype {
	case dns.TypeA:
		var ips []string
		if domain == "localhost." {
			// BUG(lukeshu): I have no idea why a lookup
			// for localhost even makes it to here on my
			// home WiFi when connecting to a k3sctl
			// cluster (but not a kubernaut.io cluster).
			// But it does, so I need this in order to be
			// productive at home.  We should really
			// root-cause this, because it's weird.
			ips = []string{"127.0.0.1"}
		} else {
			ips = s.resolve(domain)
		}
		if len(ips) > 0 {
			msg := dns.Msg{}
			msg.SetReply(r)
			msg.Authoritative = true
			// mac dns seems to fallback if you don't
			// support recursion, if you have more than a
			// single dns server, this will prevent us
			// from intercepting all queries
			msg.RecursionAvailable = true
			for _, ip := range ips {
				dlog.Debugf(c, "QUERY %s -> %s", domain, ip)
				// if we don't give back the same domain
				// requested, then mac dns seems to return an
				// nxdomain
				msg.Answer = append(msg.Answer, &dns.A{
					Hdr: dns.RR_Header{Name: r.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60},
					A:   net.ParseIP(ip),
				})
			}
			_ = w.WriteMsg(&msg)
			return
		}
	default:
		ips := s.resolve(domain)
		if len(ips) > 0 {
			dlog.Debugf(c, "QTYPE[%v] %s -> EMPTY", r.Question[0].Qtype, domain)
			msg := dns.Msg{}
			msg.SetReply(r)
			msg.Authoritative = true
			msg.RecursionAvailable = true
			_ = w.WriteMsg(&msg)
			return
		}
	}
	if s.fallback != "" {
		dlog.Debugf(c, "QTYPE[%v] %s -> FALLBACK", r.Question[0].Qtype, domain)
		in, err := dns.Exchange(r, s.fallback)
		if err != nil {
			dlog.Error(c, err)
			return
		}
		_ = w.WriteMsg(in)
	} else {
		dlog.Debugf(c, "QTYPE[%v] %s -> NOT FOUND", r.Question[0].Qtype, domain)
		m := new(dns.Msg)
		m.SetRcode(r, dns.RcodeNameError)
		_ = w.WriteMsg(m)
	}
}

// Start starts the DNS server
func (s *Server) Run(c context.Context) error {
	g := dgroup.NewGroup(c, dgroup.GroupConfig{})
	for _, listener := range s.listeners {
		srv := &dns.Server{PacketConn: listener, Handler: s}
		g.Go(listener.LocalAddr().String(), func(c context.Context) error {
			go func() {
				<-c.Done()
				_ = srv.ShutdownContext(dcontext.HardContext(c))
			}()
			return srv.ActivateAndServe()
		})
	}
	return g.Wait()
}
<commit_msg>Make local DNS resolver keep track of number of requests.<commit_after>package dns

import (
	"context"
	"net"
	"strings"
	"sync/atomic"

	"github.com/miekg/dns"

	"github.com/datawire/dlib/dcontext"
	"github.com/datawire/dlib/dgroup"
	"github.com/datawire/dlib/dlog"
)

// Server is a DNS server which implements the github.com/miekg/dns Handler interface
type Server struct {
	ctx          context.Context // necessary to make logging work in ServeDNS function
	listeners    []net.PacketConn
	fallback     string
	resolve      func(string) []string
	requestCount int64
}

// NewServer returns a new dns.Server
func NewServer(c context.Context, listeners []net.PacketConn, fallback string, resolve func(string) []string) *Server {
	return &Server{
		ctx:       c,
		listeners: listeners,
		fallback:  fallback,
		resolve:   resolve,
	}
}

// RequestCount returns the number of requests that this server has received.
func (s *Server) RequestCount() int {
	return int(atomic.LoadInt64(&s.requestCount))
}

// ServeDNS is an implementation of github.com/miekg/dns Handler.ServeDNS.
func (s *Server) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
	c := s.ctx
	defer func() {
		// Closing the response tells the DNS service to terminate
		if c.Err() != nil {
			_ = w.Close()
		}
	}()

	atomic.AddInt64(&s.requestCount, 1)

	domain := strings.ToLower(r.Question[0].Name)
	switch r.Question[0].Qtype {
	case dns.TypeA:
		var ips []string
		if domain == "localhost." {
			// BUG(lukeshu): I have no idea why a lookup
			// for localhost even makes it to here on my
			// home WiFi when connecting to a k3sctl
			// cluster (but not a kubernaut.io cluster).
			// But it does, so I need this in order to be
			// productive at home.  We should really
			// root-cause this, because it's weird.
			ips = []string{"127.0.0.1"}
		} else {
			ips = s.resolve(domain)
		}
		if len(ips) > 0 {
			msg := dns.Msg{}
			msg.SetReply(r)
			msg.Authoritative = true
			// mac dns seems to fallback if you don't
			// support recursion, if you have more than a
			// single dns server, this will prevent us
			// from intercepting all queries
			msg.RecursionAvailable = true
			for _, ip := range ips {
				dlog.Debugf(c, "QUERY %s -> %s", domain, ip)
				// if we don't give back the same domain
				// requested, then mac dns seems to return an
				// nxdomain
				msg.Answer = append(msg.Answer, &dns.A{
					Hdr: dns.RR_Header{Name: r.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60},
					A:   net.ParseIP(ip),
				})
			}
			_ = w.WriteMsg(&msg)
			return
		}
	default:
		ips := s.resolve(domain)
		if len(ips) > 0 {
			dlog.Debugf(c, "QTYPE[%v] %s -> EMPTY", r.Question[0].Qtype, domain)
			msg := dns.Msg{}
			msg.SetReply(r)
			msg.Authoritative = true
			msg.RecursionAvailable = true
			_ = w.WriteMsg(&msg)
			return
		}
	}
	if s.fallback != "" {
		dlog.Debugf(c, "QTYPE[%v] %s -> FALLBACK", r.Question[0].Qtype, domain)
		in, err := dns.Exchange(r, s.fallback)
		if err != nil {
			dlog.Error(c, err)
			return
		}
		_ = w.WriteMsg(in)
	} else {
		dlog.Debugf(c, "QTYPE[%v] %s -> NOT FOUND", r.Question[0].Qtype, domain)
		m := new(dns.Msg)
		m.SetRcode(r, dns.RcodeNameError)
		_ = w.WriteMsg(m)
	}
}

// Start starts the DNS server
func (s *Server) Run(c context.Context) error {
	g := dgroup.NewGroup(c, dgroup.GroupConfig{})
	for _, listener := range s.listeners {
		srv := &dns.Server{PacketConn: listener, Handler: s}
		g.Go(listener.LocalAddr().String(), func(c context.Context) error {
			go func() {
				<-c.Done()
				_ = srv.ShutdownContext(dcontext.HardContext(c))
			}()
			return srv.ActivateAndServe()
		})
	}
	return g.Wait()
}
<|endoftext|>


<commit_before>package stripper_test

import (
	"io"
	"os"
	"strings"

	"github.com/docker-library/go-dockerlibrary/pkg/stripper"
)

func ExampleCommentStripper() {
	r := strings.NewReader(`
# opening comment
a: b
# comment!
c: d # not a comment

# another cheeky comment
e: f
`)

	comStrip := stripper.NewCommentStripper(r)

	io.Copy(os.Stdout, comStrip)

	// Output:
	// a: b
	// c: d # not a comment
	//
	// e: f
}
<commit_msg>Update "pkg/stripper" coverage to 100% with a smaller buffer size<commit_after>package stripper_test

import (
	"io"
	"os"
	"strings"

	"github.com/docker-library/go-dockerlibrary/pkg/stripper"
)

func ExampleCommentStripper() {
	r := strings.NewReader(`
# opening comment
a: b
# comment!
c: d # not a comment

# another cheeky comment
e: f
`)

	comStrip := stripper.NewCommentStripper(r)

	// using CopyBuffer to force smaller Read sizes (better testing coverage that way)
	io.CopyBuffer(os.Stdout, comStrip, make([]byte, 32))

	// Output:
	// a: b
	// c: d # not a comment
	//
	// e: f
}
<|endoftext|>


<commit_before>package kafka

import (
	"runtime/debug"
	"sync"

	"github.com/Shopify/sarama"
	"github.com/funkygao/dbus/pkg/batcher"
	log "github.com/funkygao/log4go"
)

// Producer is a uniform kafka producer that is transparent for sync/async mode.
type Producer struct {
	cf      *Config
	name    string
	brokers []string
	stopper chan struct{}
	wg      sync.WaitGroup
	b       *batcher.Batcher
	m       *producerMetrics

	p  sarama.SyncProducer
	ap sarama.AsyncProducer

	// Send will send a kafka message.
	Send func(*sarama.ProducerMessage) error

	onError   func(*sarama.ProducerError)
	onSuccess func(*sarama.ProducerMessage)
}

// NewProducer creates a uniform kafka producer.
func NewProducer(name string, brokers []string, cf *Config) *Producer {
	if !cf.dryrun && len(brokers) == 0 {
		panic(name + " empty brokers")
	}

	p := &Producer{
		name:    name,
		brokers: brokers,
		cf:      cf,
		stopper: make(chan struct{}),
	}

	return p
}

// Start is REQUIRED before the producer is able to produce.
func (p *Producer) Start() error {
	p.m = newMetrics(p.name)

	var err error
	if p.cf.dryrun {
		// dryrun mode
		p.b = batcher.NewBatcher(p.cf.Sarama.Producer.Flush.Messages)
		p.Send = p.dryrunSend
		return nil
	}

	if !p.cf.async {
		// sync mode
		p.p, err = sarama.NewSyncProducer(p.brokers, p.cf.Sarama)
		p.Send = p.syncSend
		return err
	}

	// async mode
	if p.onError == nil || p.onSuccess == nil {
		return ErrNotReady
	}

	p.b = batcher.NewBatcher(p.cf.Sarama.Producer.Flush.Messages)
	if p.ap, err = sarama.NewAsyncProducer(p.brokers, p.cf.Sarama); err != nil {
		return err
	}

	p.Send = p.asyncSend

	p.wg.Add(1)
	go p.dispatchCallbacks()
	p.wg.Add(1)
	go p.asyncSendWorker()

	return nil
}

// Close will drain and close the Producer.
func (p *Producer) Close() error {
	close(p.stopper)

	if p.cf.dryrun {
		return nil
	}

	if p.cf.async {
		p.ap.AsyncClose()
		p.b.Close()
		p.wg.Wait()
		return nil
	}

	return p.p.Close()
}

// ClientID returns the client id for the kafka connection.
func (p *Producer) ClientID() string {
	return p.cf.Sarama.ClientID
}

// SetErrorHandler setup the async producer unretriable errors, e.g:
// ErrInvalidPartition, ErrMessageSizeTooLarge, ErrIncompleteResponse
// ErrBreakerOpen(e,g. update leader fails).
// And it is *REQUIRED* for async producer.
// For sync producer it is not allowed.
func (p *Producer) SetErrorHandler(f func(err *sarama.ProducerError)) error {
	if !p.cf.async {
		return ErrNotAllowed
	}

	if f == nil {
		p.cf.Sarama.Producer.Return.Errors = false
	}
	if p.onError != nil {
		return ErrNotAllowed
	}
	p.onError = f
	return nil
}

// SetSuccessHandler sets the success produced message callback for async producer.
// And it is *REQUIRED* for async producer.
// For sync producer it is not allowed.
func (p *Producer) SetSuccessHandler(f func(err *sarama.ProducerMessage)) error {
	if !p.cf.async {
		return ErrNotAllowed
	}

	if f == nil {
		p.cf.Sarama.Producer.Return.Successes = false
	}
	if p.onSuccess != nil {
		return ErrNotAllowed
	}
	p.onSuccess = f
	return nil
}

func (p *Producer) syncSend(m *sarama.ProducerMessage) error {
	_, _, err := p.p.SendMessage(m)
	if err != nil {
		p.m.syncFail.Mark(1)
	} else {
		p.m.syncOk.Mark(1)
	}
	return err
}

func (p *Producer) dryrunSend(m *sarama.ProducerMessage) error {
	p.b.Put(m)
	p.b.Succeed() // i,e. onSuccess called silently
	return nil
}

func (p *Producer) asyncSend(m *sarama.ProducerMessage) error {
	p.b.Put(m)
	return nil
}

func (p *Producer) asyncSendWorker() {
	defer p.wg.Done()

	for {
		select {
		case <-p.stopper:
			return

		default:
			if msg, err := p.b.Get(); err == nil {
				// FIXME what if msg is nil
				// FIXME a batch of 10, msg7 is too big message size, lead to dead loop
				pm := msg.(*sarama.ProducerMessage)
				p.ap.Input() <- pm
				p.m.asyncSend.Mark(1)
			} else {
				log.Trace("[%s] batcher closed", p.name)
				return
			}
		}
	}
}

func (p *Producer) dispatchCallbacks() {
	defer func() {
		if err := recover(); err != nil {
			log.Critical("[%s] %v\n%s", p.name, err, string(debug.Stack()))
		}

		p.wg.Done()
	}()

	errChan := p.ap.Errors()
	okChan := p.ap.Successes()
	for {
		select {
		case msg, ok := <-okChan:
			if !ok {
				okChan = nil
			} else {
				p.b.Succeed()
				p.m.asyncOk.Mark(1)
				p.onSuccess(msg)
			}

		case err, ok := <-errChan:
			if !ok {
				errChan = nil
			} else {
				p.b.Fail()
				p.m.asyncFail.Mark(1)
				p.onError(err)
			}
		}

		if okChan == nil && errChan == nil {
			log.Trace("[%s] success & err chan both closed", p.name)
			return
		}
	}
}
<commit_msg>bug fix: dry run kafka producer need recycle packet<commit_after>package kafka

import (
	"runtime/debug"
	"sync"

	"github.com/Shopify/sarama"
	"github.com/funkygao/dbus/engine"
	"github.com/funkygao/dbus/pkg/batcher"
	log "github.com/funkygao/log4go"
)

// Producer is a uniform kafka producer that is transparent for sync/async mode.
type Producer struct {
	cf      *Config
	name    string
	brokers []string
	stopper chan struct{}
	wg      sync.WaitGroup
	b       *batcher.Batcher
	m       *producerMetrics

	p  sarama.SyncProducer
	ap sarama.AsyncProducer

	// Send will send a kafka message.
	Send func(*sarama.ProducerMessage) error

	onError   func(*sarama.ProducerError)
	onSuccess func(*sarama.ProducerMessage)
}

// NewProducer creates a uniform kafka producer.
func NewProducer(name string, brokers []string, cf *Config) *Producer {
	if !cf.dryrun && len(brokers) == 0 {
		panic(name + " empty brokers")
	}

	p := &Producer{
		name:    name,
		brokers: brokers,
		cf:      cf,
		stopper: make(chan struct{}),
	}

	return p
}

// Start is REQUIRED before the producer is able to produce.
func (p *Producer) Start() error {
	p.m = newMetrics(p.name)

	var err error
	if p.cf.dryrun {
		// dryrun mode
		p.b = batcher.NewBatcher(p.cf.Sarama.Producer.Flush.Messages)
		p.Send = p.dryrunSend
		return nil
	}

	if !p.cf.async {
		// sync mode
		p.p, err = sarama.NewSyncProducer(p.brokers, p.cf.Sarama)
		p.Send = p.syncSend
		return err
	}

	// async mode
	if p.onError == nil || p.onSuccess == nil {
		return ErrNotReady
	}

	p.b = batcher.NewBatcher(p.cf.Sarama.Producer.Flush.Messages)
	if p.ap, err = sarama.NewAsyncProducer(p.brokers, p.cf.Sarama); err != nil {
		return err
	}

	p.Send = p.asyncSend

	p.wg.Add(1)
	go p.dispatchCallbacks()
	p.wg.Add(1)
	go p.asyncSendWorker()

	return nil
}

// Close will drain and close the Producer.
func (p *Producer) Close() error {
	close(p.stopper)

	if p.cf.dryrun {
		return nil
	}

	if p.cf.async {
		p.ap.AsyncClose()
		p.b.Close()
		p.wg.Wait()
		return nil
	}

	return p.p.Close()
}

// ClientID returns the client id for the kafka connection.
func (p *Producer) ClientID() string {
	return p.cf.Sarama.ClientID
}

// SetErrorHandler setup the async producer unretriable errors, e.g:
// ErrInvalidPartition, ErrMessageSizeTooLarge, ErrIncompleteResponse
// ErrBreakerOpen(e,g. update leader fails).
// And it is *REQUIRED* for async producer.
// For sync producer it is not allowed.
func (p *Producer) SetErrorHandler(f func(err *sarama.ProducerError)) error {
	if !p.cf.async {
		return ErrNotAllowed
	}

	if f == nil {
		p.cf.Sarama.Producer.Return.Errors = false
	}
	if p.onError != nil {
		return ErrNotAllowed
	}
	p.onError = f
	return nil
}

// SetSuccessHandler sets the success produced message callback for async producer.
// And it is *REQUIRED* for async producer.
// For sync producer it is not allowed.
func (p *Producer) SetSuccessHandler(f func(err *sarama.ProducerMessage)) error {
	if !p.cf.async {
		return ErrNotAllowed
	}

	if f == nil {
		p.cf.Sarama.Producer.Return.Successes = false
	}
	if p.onSuccess != nil {
		return ErrNotAllowed
	}
	p.onSuccess = f
	return nil
}

func (p *Producer) syncSend(m *sarama.ProducerMessage) error {
	_, _, err := p.p.SendMessage(m)
	if err != nil {
		p.m.syncFail.Mark(1)
	} else {
		p.m.syncOk.Mark(1)
	}
	return err
}

func (p *Producer) dryrunSend(m *sarama.ProducerMessage) error {
	p.b.Put(m)
	p.b.Succeed() // i,e. onSuccess called silently
	pack := m.Metadata.(*engine.Packet)
	pack.Recycle()
	return nil
}

func (p *Producer) asyncSend(m *sarama.ProducerMessage) error {
	p.b.Put(m)
	return nil
}

func (p *Producer) asyncSendWorker() {
	defer p.wg.Done()

	for {
		select {
		case <-p.stopper:
			return

		default:
			if msg, err := p.b.Get(); err == nil {
				// FIXME what if msg is nil
				// FIXME a batch of 10, msg7 is too big message size, lead to dead loop
				pm := msg.(*sarama.ProducerMessage)
				p.ap.Input() <- pm
				p.m.asyncSend.Mark(1)
			} else {
				log.Trace("[%s] batcher closed", p.name)
				return
			}
		}
	}
}

func (p *Producer) dispatchCallbacks() {
	defer func() {
		if err := recover(); err != nil {
			log.Critical("[%s] %v\n%s", p.name, err, string(debug.Stack()))
		}

		p.wg.Done()
	}()

	errChan := p.ap.Errors()
	okChan := p.ap.Successes()
	for {
		select {
		case msg, ok := <-okChan:
			if !ok {
				okChan = nil
			} else {
				p.b.Succeed()
				p.m.asyncOk.Mark(1)
				p.onSuccess(msg)
			}

		case err, ok := <-errChan:
			if !ok {
				errChan = nil
			} else {
				p.b.Fail()
				p.m.asyncFail.Mark(1)
				p.onError(err)
			}
		}

		if okChan == nil && errChan == nil {
			log.Trace("[%s] success & err chan both closed", p.name)
			return
		}
	}
}
<|endoftext|>


<commit_before>package redis_backend

import (
	"github.com/go-errors/errors"
	"github.com/leanovate/microzon-auth-go/common"
	"github.com/leanovate/microzon-auth-go/logging"
	"gopkg.in/redis.v3"
	"strconv"
	"strings"
	"sync/atomic"
	"time"
)

const keyRevocationsPublish = "revocations"
const keyRevocationVersionCounter = "revocations:version"

type redisRevocationsListener struct {
	lastVersion uint64
	connector   redisConnector
	listener    common.RevocationsListener
	logger      logging.Logger
}

func newRedisRevocationsListener(connector redisConnector, listener common.RevocationsListener, logger logging.Logger) (*redisRevocationsListener, error) {
	redisListener := &redisRevocationsListener{
		lastVersion: 0,
		connector:      connector,
		listener:       listener,
		logger:         logger,
	}

	if err := redisListener.fetchLastVersion(); err != nil {
		return nil, err
	}

	go redisListener.startListenRevocationUpdates()

	if err := redisListener.scanRevocations(); err != nil {
		return nil, err
	}

	return redisListener, nil
}

func (r *redisRevocationsListener) fetchLastVersion() error {
	client, err := r.connector.getClient("")
	if err != nil {
		return errors.Wrap(err, 0)
	}
	value, err := client.Get(keyRevocationVersionCounter).Result()
	if err == redis.Nil {
		return nil
	} else if err != nil {
		return errors.Wrap(err, 0)
	}
	version, err := strconv.ParseUint(value, 10, 64)
	if err != nil {
		return errors.Wrap(err, 0)
	}
	r.lastVersion = version
	return nil
}

func (r *redisRevocationsListener) scanRevocations() error {
	client, err := r.connector.getClient("")
	if err != nil {
		return errors.Wrap(err, 0)
	}
	var cursor int64 = 0
	first := true
	for first || cursor != 0 {
		first = false
		nextCursor, keys, err := client.Scan(cursor, revocationKey("*"), 0).Result()
		if err != nil {
			return errors.Wrap(err, 0)
		}
		cursor = nextCursor
		if len(keys) > 0 {
			values, err := client.MGet(keys...).Result()
			if err != nil {
				return errors.Wrap(err, 0)
			}
			for _, value := range values {
				if encodedRevocation, ok := value.(string); ok {
					if err := r.decodeAndAddRevocation(encodedRevocation); err != nil {
						return err
					}
				}
			}
		}
	}
	return nil
}

func (r *redisRevocationsListener) decodeAndFillGaps(encoded string) error {
	currentVersion := atomic.LoadUint64(&r.lastVersion)

	parts := strings.Split(encoded, ";")
	newVersion, err := strconv.ParseUint(parts[2], 10, 64)
	if err != nil {
		return err
	}

	client, err := r.connector.getClient(keyRevocationVersionCounter)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	for version := currentVersion + 1; version < newVersion; version++ {
		r.logger.Debugf("Fetch gaps from %d", version)

		encoded, err := client.Get(revocationKey(strconv.FormatUint(version, 10))).Result()
		if err == redis.Nil {
			r.logger.Debugf("Version %d does not exists in redis", version)
		} else if err != nil {
			r.logger.ErrorErr(err)
		} else {
			if err := r.decodeAndAddRevocation(encoded); err != nil {
				r.logger.ErrorErr(err)
			}
		}
	}

	return r.decodeAndAddRevocation(encoded)
}

func (r *redisRevocationsListener) decodeAndAddRevocation(encoded string) error {
	parts := strings.Split(encoded, ";")
	if len(parts) != 3 {
		return errors.Errorf("Invalid entry: %s", encoded)
	}
	sha256, err := common.RawSha256FromBase64(parts[0])
	if err != nil {
		return err
	}
	expiresAt, err := strconv.ParseInt(parts[1], 10, 64)
	if err != nil {
		return err
	}
	version, err := strconv.ParseUint(parts[2], 10, 64)
	if err != nil {
		return err
	}

	currentVersion := atomic.LoadUint64(&r.lastVersion)
	if version > currentVersion {
		atomic.CompareAndSwapUint64(&r.lastVersion, currentVersion, version)
	}
	r.listener(version, sha256, time.Unix(expiresAt, 0))
	return nil
}

func (r *redisRevocationsListener) listenRevocationUpdates() error {
	client, err := r.connector.getClient(keyRevocationVersionCounter)
	if err != nil {
		return err
	}
	subscription, err := client.Subscribe(keyRevocationsPublish)
	if err != nil {
		return err
	}

	for {
		if message, err := subscription.ReceiveMessage(); err == nil {
			r.logger.Debugf("Received revocation update: %s", message.Payload)
			if err := r.decodeAndFillGaps(message.Payload); err != nil {
				r.logger.ErrorErr(err)
			}
		} else {
			return err
		}
	}
}

func (r *redisRevocationsListener) startListenRevocationUpdates() {
	for {
		r.logger.Info("Connect to revocation subscription")

		if err := r.listenRevocationUpdates(); err != nil {
			r.logger.ErrorErr(err)
		}
		r.logger.Info("Wait 1 second before reconnecting")
		time.Sleep(1 * time.Second)
	}
}
<commit_msg>Minor refactor<commit_after>package redis_backend

import (
	"github.com/go-errors/errors"
	"github.com/leanovate/microzon-auth-go/common"
	"github.com/leanovate/microzon-auth-go/logging"
	"gopkg.in/redis.v3"
	"strconv"
	"strings"
	"sync/atomic"
	"time"
)

const keyRevocationsPublish = "revocations"
const keyRevocationVersionCounter = "revocations:version"

type redisRevocationsListener struct {
	lastVersion uint64
	connector   redisConnector
	listener    common.RevocationsListener
	logger      logging.Logger
}

func newRedisRevocationsListener(connector redisConnector, listener common.RevocationsListener, logger logging.Logger) (*redisRevocationsListener, error) {
	redisListener := &redisRevocationsListener{
		lastVersion: 0,
		connector:   connector,
		listener:    listener,
		logger:      logger,
	}

	if err := redisListener.fetchLastVersion(); err != nil {
		return nil, err
	}

	go redisListener.startListenRevocationUpdates()

	if err := redisListener.scanRevocations(); err != nil {
		return nil, err
	}

	return redisListener, nil
}

func (r *redisRevocationsListener) fetchLastVersion() error {
	client, err := r.connector.getClient("")
	if err != nil {
		return errors.Wrap(err, 0)
	}
	value, err := client.Get(keyRevocationVersionCounter).Result()
	if err == redis.Nil {
		return nil
	} else if err != nil {
		return errors.Wrap(err, 0)
	}
	version, err := strconv.ParseUint(value, 10, 64)
	if err != nil {
		return errors.Wrap(err, 0)
	}
	r.lastVersion = version
	return nil
}

func (r *redisRevocationsListener) scanRevocations() error {
	client, err := r.connector.getClient("")
	if err != nil {
		return errors.Wrap(err, 0)
	}
	var cursor int64 = 0
	first := true
	for first || cursor != 0 {
		first = false
		nextCursor, keys, err := client.Scan(cursor, revocationKey("*"), 0).Result()
		if err != nil {
			return errors.Wrap(err, 0)
		}
		cursor = nextCursor
		if len(keys) > 0 {
			values, err := client.MGet(keys...).Result()
			if err != nil {
				return errors.Wrap(err, 0)
			}
			for _, value := range values {
				if encodedRevocation, ok := value.(string); ok {
					if err := r.decodeAndAddRevocation(encodedRevocation); err != nil {
						return err
					}
				}
			}
		}
	}
	return nil
}

func (r *redisRevocationsListener) decodeAndFillGaps(encoded string) error {
	currentVersion := atomic.LoadUint64(&r.lastVersion)

	parts := strings.Split(encoded, ";")
	newVersion, err := strconv.ParseUint(parts[2], 10, 64)
	if err != nil {
		return err
	}

	client, err := r.connector.getClient(keyRevocationVersionCounter)
	if err != nil {
		return errors.Wrap(err, 0)
	}

	for version := currentVersion + 1; version < newVersion; version++ {
		r.logger.Debugf("Fetch gaps from %d", version)

		encoded, err := client.Get(revocationKey(strconv.FormatUint(version, 10))).Result()
		if err == redis.Nil {
			r.logger.Debugf("Version %d does not exists in redis", version)
		} else if err != nil {
			r.logger.ErrorErr(err)
		} else {
			if err := r.decodeAndAddRevocation(encoded); err != nil {
				r.logger.ErrorErr(err)
			}
		}
	}

	return r.decodeAndAddRevocation(encoded)
}

func (r *redisRevocationsListener) decodeAndAddRevocation(encoded string) error {
	parts := strings.Split(encoded, ";")
	if len(parts) != 3 {
		return errors.Errorf("Invalid entry: %s", encoded)
	}
	sha256, err := common.RawSha256FromBase64(parts[0])
	if err != nil {
		return err
	}
	expiresAt, err := strconv.ParseInt(parts[1], 10, 64)
	if err != nil {
		return err
	}
	version, err := strconv.ParseUint(parts[2], 10, 64)
	if err != nil {
		return err
	}

	currentVersion := atomic.LoadUint64(&r.lastVersion)
	if version > currentVersion {
		atomic.CompareAndSwapUint64(&r.lastVersion, currentVersion, version)
	}
	r.listener(version, sha256, time.Unix(expiresAt, 0))
	return nil
}

func (r *redisRevocationsListener) listenRevocationUpdates() error {
	client, err := r.connector.getClient(keyRevocationVersionCounter)
	if err != nil {
		return err
	}
	subscription, err := client.Subscribe(keyRevocationsPublish)
	if err != nil {
		return err
	}

	for {
		if message, err := subscription.ReceiveMessage(); err == nil {
			r.logger.Debugf("Received revocation update: %s", message.Payload)
			if err := r.decodeAndFillGaps(message.Payload); err != nil {
				r.logger.ErrorErr(err)
			}
		} else {
			return err
		}
	}
}

func (r *redisRevocationsListener) startListenRevocationUpdates() {
	for {
		r.logger.Info("Connect to revocation subscription")

		if err := r.listenRevocationUpdates(); err != nil {
			r.logger.ErrorErr(err)
		}
		r.logger.Info("Wait 1 second before reconnecting")
		time.Sleep(1 * time.Second)
	}
}
<|endoftext|>


<commit_before>// Original license //
// ---------------- //

/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// All other modifications and improvements //
// ---------------------------------------- //

/*
 * Minimalist Object Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package s3

import (
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/minio-io/mc/pkg/client"
)

// Date format
const (
	iso8601Format = "2006-01-02T15:04:05.000Z"
)

var tc *s3Client

func TestParseBuckets(t *testing.T) {
	res := "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Owner><ID>ownerIDField</ID><DisplayName>bobDisplayName</DisplayName></Owner><Buckets><Bucket><Name>bucketOne</Name><CreationDate>2006-06-21T07:04:31.000Z</CreationDate></Bucket><Bucket><Name>bucketTwo</Name><CreationDate>2006-06-21T07:04:32.000Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>"
	buckets, err := parseListAllMyBuckets(strings.NewReader(res))
	if err != nil {
		t.Fatal(err)
	}
	if g, w := len(buckets), 2; g != w {
		t.Errorf("num parsed buckets = %d; want %d", g, w)
	}

	t1, _ := time.Parse(iso8601Format, "2006-06-21T07:04:31.000Z")
	t2, _ := time.Parse(iso8601Format, "2006-06-21T07:04:32.000Z")
	want := []*client.Bucket{
		{Name: "bucketOne", CreationDate: t1},
		{Name: "bucketTwo", CreationDate: t2},
	}
	dump := func(v []*client.Bucket) {
		for i, b := range v {
			t.Logf("Bucket #%d: %#v", i, b)
		}
	}
	if !reflect.DeepEqual(buckets, want) {
		t.Error("mismatch; GOT:")
		dump(buckets)
		t.Error("WANT:")
		dump(want)
	}
}

func TestValidBucketNames(t *testing.T) {
	m := []struct {
		in   string
		want bool
	}{
		{"myawsbucket", true},
		{"myaws-bucket", true},
		{"my-aws-bucket", true},
		{"my.aws.bucket", false},
		{"my-aws-bucket.1", false},
		{"my---bucket.1", false},
		{".myawsbucket", false},
		{"-myawsbucket", false},
		{"myawsbucket.", false},
		{"myawsbucket-", false},
		{"my..awsbucket", false},
	}

	for _, bt := range m {
		got := IsValidBucketName(bt.in)
		if got != bt.want {
			t.Errorf("func(%q) = %v; want %v", bt.in, got, bt.want)
		}
	}
}
<commit_msg>Fix typo<commit_after>// Original license //
// ---------------- //

/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// All other modifications and improvements //
// ---------------------------------------- //

/*
 * Minimalist Object Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package s3

import (
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/minio-io/mc/pkg/client"
)

// Date format
const (
	iso8601Format = "2006-01-02T15:04:05.000Z"
)

var tc *s3Client

func TestParseBuckets(t *testing.T) {
	res := "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Owner><ID>ownerIDField</ID><DisplayName>bobDisplayName</DisplayName></Owner><Buckets><Bucket><Name>bucketOne</Name><CreationDate>2006-06-21T07:04:31.000Z</CreationDate></Bucket><Bucket><Name>bucketTwo</Name><CreationDate>2006-06-21T07:04:32.000Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>"
	buckets, err := listAllMyBuckets(strings.NewReader(res))
	if err != nil {
		t.Fatal(err)
	}
	if g, w := len(buckets), 2; g != w {
		t.Errorf("num parsed buckets = %d; want %d", g, w)
	}

	t1, _ := time.Parse(iso8601Format, "2006-06-21T07:04:31.000Z")
	t2, _ := time.Parse(iso8601Format, "2006-06-21T07:04:32.000Z")
	want := []*client.Bucket{
		{Name: "bucketOne", CreationDate: t1},
		{Name: "bucketTwo", CreationDate: t2},
	}
	dump := func(v []*client.Bucket) {
		for i, b := range v {
			t.Logf("Bucket #%d: %#v", i, b)
		}
	}
	if !reflect.DeepEqual(buckets, want) {
		t.Error("mismatch; GOT:")
		dump(buckets)
		t.Error("WANT:")
		dump(want)
	}
}

func TestValidBucketNames(t *testing.T) {
	m := []struct {
		in   string
		want bool
	}{
		{"myawsbucket", true},
		{"myaws-bucket", true},
		{"my-aws-bucket", true},
		{"my.aws.bucket", false},
		{"my-aws-bucket.1", false},
		{"my---bucket.1", false},
		{".myawsbucket", false},
		{"-myawsbucket", false},
		{"myawsbucket.", false},
		{"myawsbucket-", false},
		{"my..awsbucket", false},
	}

	for _, bt := range m {
		got := IsValidBucketName(bt.in)
		if got != bt.want {
			t.Errorf("func(%q) = %v; want %v", bt.in, got, bt.want)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kafka

import (
	"encoding/binary"
	"encoding/json"
	"fmt"

	"github.com/optiopay/kafka/proto"
	"io"
)

// ResponseMessage represents a Kafka response message.
type ResponseMessage struct {
	rawMsg   []byte
	response interface{}
}

// GetCorrelationID returns the Kafka request correlationID
func (res *ResponseMessage) GetCorrelationID() CorrelationID {
	if len(res.rawMsg) >= 8 {
		return CorrelationID(binary.BigEndian.Uint32(res.rawMsg[4:8]))
	}

	return CorrelationID(0)
}

// SetCorrelationID modified the correlation ID of the Kafka request
func (res *ResponseMessage) SetCorrelationID(id CorrelationID) {
	if len(res.rawMsg) >= 8 {
		binary.BigEndian.PutUint32(res.rawMsg[4:8], uint32(id))
	}
}

// GetRaw returns the raw Kafka response
func (res *ResponseMessage) GetRaw() []byte {
	return res.rawMsg
}

// String returns a human readable representation of the response message
func (res *ResponseMessage) String() string {
	b, err := json.Marshal(res.response)
	if err != nil {
		return err.Error()
	}
	return string(b)
}

// ReadResponse will read a Kafka response from an io.Reader and return the
// message or an error.
func ReadResponse(reader io.Reader) (*ResponseMessage, error) {
	rsp := &ResponseMessage{}
	var err error

	_, rsp.rawMsg, err = proto.ReadResp(reader)
	if err != nil {
		return nil, err
	}

	if len(rsp.rawMsg) < 6 {
		return nil,
			fmt.Errorf("unexpected end of response (length < 6 bytes)")
	}

	return rsp, nil
}

func createProduceResponse(req *proto.ProduceReq, err error) (*ResponseMessage, error) {
	if req == nil {
		return nil, fmt.Errorf("request is nil")
	}

	resp := &proto.ProduceResp{
		CorrelationID: req.CorrelationID,
		Topics:        make([]proto.ProduceRespTopic, len(req.Topics)),
	}

	for k, topic := range req.Topics {
		resp.Topics[k] = proto.ProduceRespTopic{
			Name:       topic.Name,
			Partitions: make([]proto.ProduceRespPartition, len(topic.Partitions)),
		}

		for k2, partition := range topic.Partitions {
			resp.Topics[k].Partitions[k2] = proto.ProduceRespPartition{
				ID:  partition.ID,
				Err: err,
			}
		}
	}

	b, err := resp.Bytes(req.Version)
	if err != nil {
		return nil, err
	}

	return &ResponseMessage{
		response: resp,
		rawMsg:   b,
	}, nil
}

func createFetchResponse(req *proto.FetchReq, err error) (*ResponseMessage, error) {
	if req == nil {
		return nil, fmt.Errorf("request is nil")
	}

	resp := &proto.FetchResp{
		CorrelationID: req.CorrelationID,
		Topics:        make([]proto.FetchRespTopic, len(req.Topics)),
	}

	for k, topic := range req.Topics {
		resp.Topics[k] = proto.FetchRespTopic{
			Name:       topic.Name,
			Partitions: make([]proto.FetchRespPartition, len(topic.Partitions)),
		}

		for k2, partition := range topic.Partitions {
			resp.Topics[k].Partitions[k2] = proto.FetchRespPartition{
				ID:  partition.ID,
				Err: err,
			}
		}
	}

	b, err := resp.Bytes(req.Version)
	if err != nil {
		return nil, err
	}

	return &ResponseMessage{
		response: resp,
		rawMsg:   b,
	}, nil
}

func createOffsetResponse(req *proto.OffsetReq, err error) (*ResponseMessage, error) {
	if req == nil {
		return nil, fmt.Errorf("request is nil")
	}

	resp := &proto.OffsetResp{
		CorrelationID: req.CorrelationID,
		Topics:        make([]proto.OffsetRespTopic, len(req.Topics)),
	}

	for k, topic := range req.Topics {
		resp.Topics[k] = proto.OffsetRespTopic{
			Name:       topic.Name,
			Partitions: make([]proto.OffsetRespPartition, len(topic.Partitions)),
		}

		for k2, partition := range topic.Partitions {
			resp.Topics[k].Partitions[k2] = proto.OffsetRespPartition{
				ID:  partition.ID,
				Err: err,
			}
		}
	}

	b, err := resp.Bytes(req.Version)
	if err != nil {
		return nil, err
	}

	return &ResponseMessage{
		response: resp,
		rawMsg:   b,
	}, nil
}

func createMetadataResponse(req *proto.MetadataReq, err error) (*ResponseMessage, error) {
	if req == nil {
		return nil, fmt.Errorf("request is nil")
	}

	resp := &proto.MetadataResp{
		CorrelationID: req.CorrelationID,
		Topics:        make([]proto.MetadataRespTopic, len(req.Topics)),
	}

	for k, topic := range req.Topics {
		resp.Topics[k] = proto.MetadataRespTopic{
			Name: topic,
			Err:  err,
		}
	}

	b, err := resp.Bytes(req.Version)
	if err != nil {
		return nil, err
	}

	return &ResponseMessage{
		response: resp,
		rawMsg:   b,
	}, nil
}

func createConsumerMetadataResponse(req *proto.ConsumerMetadataReq, err error) (*ResponseMessage, error) {
	if req == nil {
		return nil, fmt.Errorf("request is nil")
	}

	resp := &proto.ConsumerMetadataResp{
		CorrelationID: req.CorrelationID,
		Err:           err,
	}

	b, err := resp.Bytes(req.Version)
	if err != nil {
		return nil, err
	}

	return &ResponseMessage{
		response: resp,
		rawMsg:   b,
	}, nil
}

func createOffsetCommitResponse(req *proto.OffsetCommitReq, err error) (*ResponseMessage, error) {
	if req == nil {
		return nil, fmt.Errorf("request is nil")
	}

	resp := &proto.OffsetCommitResp{
		CorrelationID: req.CorrelationID,
		Topics:        make([]proto.OffsetCommitRespTopic, len(req.Topics)),
	}

	for k, topic := range req.Topics {
		resp.Topics[k] = proto.OffsetCommitRespTopic{
			Name:       topic.Name,
			Partitions: make([]proto.OffsetCommitRespPartition, len(topic.Partitions)),
		}

		for k2, partition := range topic.Partitions {
			resp.Topics[k].Partitions[k2] = proto.OffsetCommitRespPartition{
				ID:  partition.ID,
				Err: err,
			}
		}
	}

	b, err := resp.Bytes(req.Version)
	if err != nil {
		return nil, err
	}

	return &ResponseMessage{
		response: resp,
		rawMsg:   b,
	}, nil
}

func createOffsetFetchResponse(req *proto.OffsetFetchReq, err error) (*ResponseMessage, error) {
	if req == nil {
		return nil, fmt.Errorf("request is nil")
	}

	resp := &proto.OffsetFetchResp{
		CorrelationID: req.CorrelationID,
		Topics:        make([]proto.OffsetFetchRespTopic, len(req.Topics)),
	}

	for k, topic := range req.Topics {
		resp.Topics[k] = proto.OffsetFetchRespTopic{
			Name:       topic.Name,
			Partitions: make([]proto.OffsetFetchRespPartition, len(topic.Partitions)),
		}

		for k2, partition := range topic.Partitions {
			resp.Topics[k].Partitions[k2] = proto.OffsetFetchRespPartition{
				ID:  partition,
				Err: err,
			}
		}
	}

	b, err := resp.Bytes(req.Version)
	if err != nil {
		return nil, err
	}

	return &ResponseMessage{
		response: resp,
		rawMsg:   b,
	}, nil
}
<commit_msg>kafka: Properly set (non-)nullable arrays in synthesized responses<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kafka

import (
	"encoding/binary"
	"encoding/json"
	"fmt"

	"github.com/optiopay/kafka/proto"
	"io"
)

// ResponseMessage represents a Kafka response message.
type ResponseMessage struct {
	rawMsg   []byte
	response interface{}
}

// GetCorrelationID returns the Kafka request correlationID
func (res *ResponseMessage) GetCorrelationID() CorrelationID {
	if len(res.rawMsg) >= 8 {
		return CorrelationID(binary.BigEndian.Uint32(res.rawMsg[4:8]))
	}

	return CorrelationID(0)
}

// SetCorrelationID modified the correlation ID of the Kafka request
func (res *ResponseMessage) SetCorrelationID(id CorrelationID) {
	if len(res.rawMsg) >= 8 {
		binary.BigEndian.PutUint32(res.rawMsg[4:8], uint32(id))
	}
}

// GetRaw returns the raw Kafka response
func (res *ResponseMessage) GetRaw() []byte {
	return res.rawMsg
}

// String returns a human readable representation of the response message
func (res *ResponseMessage) String() string {
	b, err := json.Marshal(res.response)
	if err != nil {
		return err.Error()
	}
	return string(b)
}

// ReadResponse will read a Kafka response from an io.Reader and return the
// message or an error.
func ReadResponse(reader io.Reader) (*ResponseMessage, error) {
	rsp := &ResponseMessage{}
	var err error

	_, rsp.rawMsg, err = proto.ReadResp(reader)
	if err != nil {
		return nil, err
	}

	if len(rsp.rawMsg) < 6 {
		return nil,
			fmt.Errorf("unexpected end of response (length < 6 bytes)")
	}

	return rsp, nil
}

func createProduceResponse(req *proto.ProduceReq, err error) (*ResponseMessage, error) {
	if req == nil {
		return nil, fmt.Errorf("request is nil")
	}

	resp := &proto.ProduceResp{
		CorrelationID: req.CorrelationID,
		Topics:        make([]proto.ProduceRespTopic, len(req.Topics)),
	}

	for k, topic := range req.Topics {
		resp.Topics[k] = proto.ProduceRespTopic{
			Name:       topic.Name,
			Partitions: make([]proto.ProduceRespPartition, len(topic.Partitions)),
		}

		for k2, partition := range topic.Partitions {
			resp.Topics[k].Partitions[k2] = proto.ProduceRespPartition{
				ID:  partition.ID,
				Err: err,
			}
		}
	}

	b, err := resp.Bytes(req.Version)
	if err != nil {
		return nil, err
	}

	return &ResponseMessage{
		response: resp,
		rawMsg:   b,
	}, nil
}

func createFetchResponse(req *proto.FetchReq, err error) (*ResponseMessage, error) {
	if req == nil {
		return nil, fmt.Errorf("request is nil")
	}

	resp := &proto.FetchResp{
		CorrelationID: req.CorrelationID,
		Topics:        make([]proto.FetchRespTopic, len(req.Topics)),
	}

	for k, topic := range req.Topics {
		resp.Topics[k] = proto.FetchRespTopic{
			Name:       topic.Name,
			Partitions: make([]proto.FetchRespPartition, len(topic.Partitions)),
		}

		for k2, partition := range topic.Partitions {
			resp.Topics[k].Partitions[k2] = proto.FetchRespPartition{
				ID:                  partition.ID,
				Err:                 err,
				AbortedTransactions: nil, // nullable
			}
		}
	}

	b, err := resp.Bytes(req.Version)
	if err != nil {
		return nil, err
	}

	return &ResponseMessage{
		response: resp,
		rawMsg:   b,
	}, nil
}

func createOffsetResponse(req *proto.OffsetReq, err error) (*ResponseMessage, error) {
	if req == nil {
		return nil, fmt.Errorf("request is nil")
	}

	resp := &proto.OffsetResp{
		CorrelationID: req.CorrelationID,
		Topics:        make([]proto.OffsetRespTopic, len(req.Topics)),
	}

	for k, topic := range req.Topics {
		resp.Topics[k] = proto.OffsetRespTopic{
			Name:       topic.Name,
			Partitions: make([]proto.OffsetRespPartition, len(topic.Partitions)),
		}

		for k2, partition := range topic.Partitions {
			resp.Topics[k].Partitions[k2] = proto.OffsetRespPartition{
				ID:      partition.ID,
				Err:     err,
				Offsets: make([]int64, 0), // Not nullable, so must never be nil.
			}
		}
	}

	b, err := resp.Bytes(req.Version)
	if err != nil {
		return nil, err
	}

	return &ResponseMessage{
		response: resp,
		rawMsg:   b,
	}, nil
}

func createMetadataResponse(req *proto.MetadataReq, err error) (*ResponseMessage, error) {
	if req == nil {
		return nil, fmt.Errorf("request is nil")
	}

	var topics []proto.MetadataRespTopic
	if req.Topics != nil {
		topics = make([]proto.MetadataRespTopic, len(req.Topics))
	}
	resp := &proto.MetadataResp{
		CorrelationID: req.CorrelationID,
		Brokers:       make([]proto.MetadataRespBroker, 0), // Not nullable, so must never be nil.
		Topics:        topics,
	}

	for k, topic := range req.Topics {
		resp.Topics[k] = proto.MetadataRespTopic{
			Name:       topic,
			Err:        err,
			Partitions: make([]proto.MetadataRespPartition, 0), // Not nullable, so must never be nil.
		}
	}

	b, err := resp.Bytes(req.Version)
	if err != nil {
		return nil, err
	}

	return &ResponseMessage{
		response: resp,
		rawMsg:   b,
	}, nil
}

func createConsumerMetadataResponse(req *proto.ConsumerMetadataReq, err error) (*ResponseMessage, error) {
	if req == nil {
		return nil, fmt.Errorf("request is nil")
	}

	resp := &proto.ConsumerMetadataResp{
		CorrelationID: req.CorrelationID,
		Err:           err,
	}

	b, err := resp.Bytes(req.Version)
	if err != nil {
		return nil, err
	}

	return &ResponseMessage{
		response: resp,
		rawMsg:   b,
	}, nil
}

func createOffsetCommitResponse(req *proto.OffsetCommitReq, err error) (*ResponseMessage, error) {
	if req == nil {
		return nil, fmt.Errorf("request is nil")
	}

	resp := &proto.OffsetCommitResp{
		CorrelationID: req.CorrelationID,
		Topics:        make([]proto.OffsetCommitRespTopic, len(req.Topics)),
	}

	for k, topic := range req.Topics {
		resp.Topics[k] = proto.OffsetCommitRespTopic{
			Name:       topic.Name,
			Partitions: make([]proto.OffsetCommitRespPartition, len(topic.Partitions)),
		}

		for k2, partition := range topic.Partitions {
			resp.Topics[k].Partitions[k2] = proto.OffsetCommitRespPartition{
				ID:  partition.ID,
				Err: err,
			}
		}
	}

	b, err := resp.Bytes(req.Version)
	if err != nil {
		return nil, err
	}

	return &ResponseMessage{
		response: resp,
		rawMsg:   b,
	}, nil
}

func createOffsetFetchResponse(req *proto.OffsetFetchReq, err error) (*ResponseMessage, error) {
	if req == nil {
		return nil, fmt.Errorf("request is nil")
	}

	var topics []proto.OffsetFetchRespTopic
	if req.Topics != nil {
		topics = make([]proto.OffsetFetchRespTopic, len(req.Topics))
	}
	resp := &proto.OffsetFetchResp{
		CorrelationID: req.CorrelationID,
		Topics:        topics,
	}

	for k, topic := range req.Topics {
		resp.Topics[k] = proto.OffsetFetchRespTopic{
			Name:       topic.Name,
			Partitions: make([]proto.OffsetFetchRespPartition, len(topic.Partitions)),
		}

		for k2, partition := range topic.Partitions {
			resp.Topics[k].Partitions[k2] = proto.OffsetFetchRespPartition{
				ID:  partition,
				Err: err,
			}
		}
	}

	b, err := resp.Bytes(req.Version)
	if err != nil {
		return nil, err
	}

	return &ResponseMessage{
		response: resp,
		rawMsg:   b,
	}, nil
}
<|endoftext|>


<commit_before>package irc

import (
	"bufio"
	"crypto/tls"
	"fmt"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/jriddick/geoffrey/msg"
)

// IRC client
type IRC struct {
	sync.WaitGroup
	conn         net.Conn
	get          chan *msg.Message
	put          chan string
	end          chan struct{}
	err          chan error
	config       Config
	reconnecting bool
}

// NewIRC returns a new IRC client
func NewIRC(config Config) *IRC {
	return &IRC{
		config: config,
	}
}

func (m *IRC) loopPut() {
	defer m.Done()

	for {
		select {
		case <-m.end:
			return
		case msg, ok := <-m.put:
			// Make sure we received a value
			if !ok {
				m.err <- fmt.Errorf("send channel is invalid")
				return
			}

			if m.conn == nil {
				m.err <- fmt.Errorf("no connection active open")
				return
			}

			// We do not send any empty values
			if msg == "" {
				m.err <- fmt.Errorf("tried to send empty message")
				continue
			}

			// Make sure the suffix is correct
			if !strings.HasSuffix(msg, "\r\n") {
				msg = msg + "\r\n"
			}

			// Set the timeout
			m.conn.SetWriteDeadline(time.Now().Add(time.Second * 30))

			// Send the message to the server
			_, err := m.conn.Write([]byte(msg))

			// Reset the timeout
			m.conn.SetWriteDeadline(time.Time{})

			// Make sure we did not get any errors
			if err != nil {
				m.err <- err
				return
			}
		}
	}
}

func (m *IRC) loopGet() {
	defer m.Done()

	// Reader for the connection
	reader := bufio.NewReaderSize(m.conn, 1024)

	for {
		select {
		case <-m.end:
			return
		default:
			if m.conn != nil {
				// Set the read timeout
				m.conn.SetReadDeadline(time.Now().Add(time.Second * 30))
			}

			// Fetch the message from the server
			raw, err := reader.ReadString('\n')

			if err != nil {
				// Ignore any errors during reconnect
				if !m.reconnecting {
					m.err <- err
				}

				return
			}

			// Reset the timeout
			if m.conn != nil {
				m.conn.SetReadDeadline(time.Time{})
			}

			// Parse the message
			msg, err := msg.ParseMessage(raw)

			if err != nil {
				m.err <- fmt.Errorf("%v [%s]", err, raw)
				return
			}

			// Send the parsed message
			m.get <- msg
		}
	}
}

// Disconnect will disconnect the client
func (m *IRC) Disconnect() {
	// Close the channels
	if m.end != nil {
		close(m.end)
	}

	if m.put != nil {
		close(m.put)
	}

	if m.get != nil {
		close(m.get)
	}

	// Close the connection
	if m.conn != nil {
		m.conn.Close()
	}

	// Reset the connection
	m.conn = nil

	// Wait for loops
	m.Wait()
}

// Connect will connect the client, create new channels if needed
// and start the handler loops.
func (m *IRC) Connect() error {
	// Don't connect if we already are connected
	if m.conn != nil {
		return fmt.Errorf("connection already active")
	}

	// Holds any encountered errors
	var err error

	// Get the hostname
	hostname := m.config.GetHostname()

	if hostname == ":0" || hostname[0] == ':' {
		return fmt.Errorf("need hostname and port to connect")
	}

	// Create the connection
	if m.config.Secure {
		m.conn, err = tls.Dial("tcp", hostname, &tls.Config{
			InsecureSkipVerify: m.config.InsecureSkipVerify,
		})
	} else {
		m.conn, err = net.Dial("tcp", hostname)
	}

	// Check for errors
	if err != nil {
		return err
	}

	// Create the input and output channels
	if m.get == nil {
		m.get = make(chan *msg.Message, 100)
	}

	if m.put == nil {
		m.put = make(chan string, 100)
	}

	if m.end == nil {
		m.end = make(chan struct{})
	}

	if m.err == nil {
		m.err = make(chan error, 100)
	}

	// Start the loops
	m.Add(2)
	go m.loopGet()
	go m.loopPut()

	return nil
}

// Reconnect will disconnect, stop the loops and then call Connect()
func (m *IRC) Reconnect() error {
	// Flag as reconnecting
	m.reconnecting = true

	// Close the connection
	if m.conn != nil {
		m.conn.Close()
	}

	// Reset the connection
	m.conn = nil

	// Close the channel
	close(m.end)

	// Wait until loops complete
	m.Wait()

	// Create the end channel
	m.end = make(chan struct{})

	// Remove the flag
	m.reconnecting = false

	// Connect to the server again
	return m.Connect()
}

// Reader returns channel for reading messages
func (m *IRC) Reader() <-chan *msg.Message {
	return m.get
}

// Writer returns channel for sending messages
func (m *IRC) Writer() chan<- string {
	return m.put
}

// Errors returns channel for reading errors
func (m *IRC) Errors() <-chan error {
	return m.err
}
<commit_msg>fixed data races on the client<commit_after>package irc

import (
	"bufio"
	"crypto/tls"
	"fmt"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/jriddick/geoffrey/msg"
)

// IRC client
type IRC struct {
	sync.WaitGroup
	conn         net.Conn
	get          chan *msg.Message
	put          chan string
	end          chan struct{}
	err          chan error
	config       Config
	reconnecting bool
}

// NewIRC returns a new IRC client
func NewIRC(config Config) *IRC {
	return &IRC{
		config: config,
		get:    make(chan *msg.Message, 100),
		put:    make(chan string, 100),
		end:    make(chan struct{}),
		err:    make(chan error, 100),
	}
}

func (m *IRC) loopPut() {
	defer m.Done()

	for {
		select {
		case <-m.end:
			return
		case msg, ok := <-m.put:
			// Make sure we received a value
			if !ok {
				m.err <- fmt.Errorf("send channel is invalid")
				return
			}

			// We do not send any empty values
			if msg == "" {
				m.err <- fmt.Errorf("tried to send empty message")
				continue
			}

			// Make sure the suffix is correct
			if !strings.HasSuffix(msg, "\r\n") {
				msg = msg + "\r\n"
			}

			// Set the timeout
			m.conn.SetWriteDeadline(time.Now().Add(time.Second * 2))

			// Send the message to the server
			_, err := m.conn.Write([]byte(msg))

			// Reset the timeout
			m.conn.SetWriteDeadline(time.Time{})

			// Make sure we did not get any errors
			if err != nil {
				m.err <- err
				return
			}
		}
	}
}

func (m *IRC) loopGet() {
	defer m.Done()

	// Reader for the connection
	reader := bufio.NewReaderSize(m.conn, 1024)

	for {
		select {
		case <-m.end:
			return
		default:
			// Set the read timeout
			m.conn.SetReadDeadline(time.Now().Add(time.Second * 2))

			// Fetch the message from the server
			raw, err := reader.ReadString('\n')

			if err != nil {
				// Ignore any errors during reconnect
				if !m.reconnecting {
					m.err <- err
				}

				return
			}

			// Reset the timeout
			m.conn.SetReadDeadline(time.Time{})

			// Parse the message
			msg, err := msg.ParseMessage(raw)

			if err != nil {
				m.err <- fmt.Errorf("%v [%s]", err, raw)
				return
			}

			// Send the parsed message
			m.get <- msg
		}
	}
}

// Disconnect will disconnect the client
func (m *IRC) Disconnect() {
	// Close the channels
	if m.end != nil {
		close(m.end)
	}

	// Wait for loops
	m.Wait()

	// Close the connection
	if m.conn != nil {
		m.conn.Close()
	}

	// Reset the connection
	m.conn = nil

	// Close the put channel
	if m.put != nil {
		close(m.put)
	}

	// Close the get channel
	if m.get != nil {
		close(m.get)
	}
}

// Connect will connect the client, create new channels if needed
// and start the handler loops.
func (m *IRC) Connect() error {
	// Don't connect if we already are connected
	if m.conn != nil {
		return fmt.Errorf("connection already active")
	}

	// Holds any encountered errors
	var err error

	// Get the hostname
	hostname := m.config.GetHostname()

	if hostname == ":0" || hostname[0] == ':' {
		return fmt.Errorf("need hostname and port to connect")
	}

	// Create the connection
	if m.config.Secure {
		m.conn, err = tls.Dial("tcp", hostname, &tls.Config{
			InsecureSkipVerify: m.config.InsecureSkipVerify,
		})
	} else {
		m.conn, err = net.Dial("tcp", hostname)
	}

	// Check for errors
	if err != nil {
		return err
	}

	// Start the loops
	m.Add(2)
	go m.loopGet()
	go m.loopPut()

	return nil
}

// Reconnect will disconnect, stop the loops and then call Connect()
func (m *IRC) Reconnect() error {
	// Flag as reconnecting
	m.reconnecting = true

	// Close the connection
	if m.conn != nil {
		m.conn.Close()
	}

	// Reset the connection
	m.conn = nil

	// Close the channel
	close(m.end)

	// Wait until loops complete
	m.Wait()

	// Create the end channel
	m.end = make(chan struct{})

	// Remove the flag
	m.reconnecting = false

	// Connect to the server again
	return m.Connect()
}

// Reader returns channel for reading messages
func (m *IRC) Reader() <-chan *msg.Message {
	return m.get
}

// Writer returns channel for sending messages
func (m *IRC) Writer() chan<- string {
	return m.put
}

// Errors returns channel for reading errors
func (m *IRC) Errors() <-chan error {
	return m.err
}
<|endoftext|>


<commit_before>// run

// Copyright 2014 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Scenario that used to leak arbitrarily many SudoG structs.
// See golang.org/issue/9110.

package main

import (
	"runtime"
	"runtime/debug"
	"sync"
	"time"
)

func main() {
	debug.SetGCPercent(1000000) // only GC when we ask for GC

	var stats, stats1, stats2 runtime.MemStats

	release := func() {}
	for i := 0; i < 20; i++ {
		if i == 10 {
			// Should be warmed up by now.
			runtime.ReadMemStats(&stats1)
		}

		c := make(chan int)
		for i := 0; i < 10; i++ {
			go func() {
				select {
				case <-c:
				case <-c:
				case <-c:
				}
			}()
		}
		time.Sleep(1 * time.Millisecond)
		release()

		close(c) // let select put its sudog's into the cache
		time.Sleep(1 * time.Millisecond)

		// pick up top sudog
		var cond1 sync.Cond
		var mu1 sync.Mutex
		cond1.L = &mu1
		go func() {
			mu1.Lock()
			cond1.Wait()
			mu1.Unlock()
		}()
		time.Sleep(1 * time.Millisecond)

		// pick up next sudog
		var cond2 sync.Cond
		var mu2 sync.Mutex
		cond2.L = &mu2
		go func() {
			mu2.Lock()
			cond2.Wait()
			mu2.Unlock()
		}()
		time.Sleep(1 * time.Millisecond)

		// put top sudog back
		cond1.Broadcast()
		time.Sleep(1 * time.Millisecond)

		// drop cache on floor
		runtime.GC()

		// release cond2 after select has gotten to run
		release = func() {
			cond2.Broadcast()
			time.Sleep(1 * time.Millisecond)
		}
	}

	runtime.GC()

	runtime.ReadMemStats(&stats2)

	if int(stats2.HeapObjects)-int(stats1.HeapObjects) > 20 { // normally at most 1 or 2; was 300 with leak
		print("BUG: object leak: ", stats.HeapObjects, " -> ", stats1.HeapObjects, " -> ", stats2.HeapObjects, "\n")
	}
}
<commit_msg>test: set GOMAXPROCS=1 in fixedbugs/issue9110<commit_after>// run

// Copyright 2014 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Scenario that used to leak arbitrarily many SudoG structs.
// See golang.org/issue/9110.

package main

import (
	"runtime"
	"runtime/debug"
	"sync"
	"time"
)

func main() {
	runtime.GOMAXPROCS(1)
	debug.SetGCPercent(1000000) // only GC when we ask for GC

	var stats, stats1, stats2 runtime.MemStats

	release := func() {}
	for i := 0; i < 20; i++ {
		if i == 10 {
			// Should be warmed up by now.
			runtime.ReadMemStats(&stats1)
		}

		c := make(chan int)
		for i := 0; i < 10; i++ {
			go func() {
				select {
				case <-c:
				case <-c:
				case <-c:
				}
			}()
		}
		time.Sleep(1 * time.Millisecond)
		release()

		close(c) // let select put its sudog's into the cache
		time.Sleep(1 * time.Millisecond)

		// pick up top sudog
		var cond1 sync.Cond
		var mu1 sync.Mutex
		cond1.L = &mu1
		go func() {
			mu1.Lock()
			cond1.Wait()
			mu1.Unlock()
		}()
		time.Sleep(1 * time.Millisecond)

		// pick up next sudog
		var cond2 sync.Cond
		var mu2 sync.Mutex
		cond2.L = &mu2
		go func() {
			mu2.Lock()
			cond2.Wait()
			mu2.Unlock()
		}()
		time.Sleep(1 * time.Millisecond)

		// put top sudog back
		cond1.Broadcast()
		time.Sleep(1 * time.Millisecond)

		// drop cache on floor
		runtime.GC()

		// release cond2 after select has gotten to run
		release = func() {
			cond2.Broadcast()
			time.Sleep(1 * time.Millisecond)
		}
	}

	runtime.GC()

	runtime.ReadMemStats(&stats2)

	if int(stats2.HeapObjects)-int(stats1.HeapObjects) > 20 { // normally at most 1 or 2; was 300 with leak
		print("BUG: object leak: ", stats.HeapObjects, " -> ", stats1.HeapObjects, " -> ", stats2.HeapObjects, "\n")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dispatcher

import (
	"fmt"
	"os"
	"syscall"

	"github.com/kubectl-dispatcher/pkg/client"
	"github.com/kubectl-dispatcher/pkg/filepath"
	"github.com/kubectl-dispatcher/pkg/logging"
	"github.com/kubectl-dispatcher/pkg/util"
	"github.com/spf13/pflag"
	"k8s.io/apimachinery/pkg/version"
	"k8s.io/cli-runtime/pkg/genericclioptions"

	// klog calls in this file assume it has been initialized beforehand
	"k8s.io/klog"
)

const requestTimeout = "5s"     // Timeout for server version query
const cacheMaxAge = 2 * 60 * 60 // 2 hours in seconds

type Dispatcher struct {
	args            []string
	env             []string
	clientVersion   *version.Info
	filepathBuilder *filepath.FilepathBuilder
}

// NewDispatcher returns a new pointer to a Dispatcher struct.
func NewDispatcher(args []string, env []string,
	clientVersion *version.Info,
	filepathBuilder *filepath.FilepathBuilder) *Dispatcher {

	return &Dispatcher{
		args:            args,
		env:             env,
		clientVersion:   clientVersion,
		filepathBuilder: filepathBuilder,
	}
}

// GetArgs returns a copy of the slice of strings representing the command line arguments.
func (d *Dispatcher) GetArgs() []string {
	return util.CopyStrSlice(d.args)
}

// GetEnv returns a copy of the slice of environment variables.
func (d *Dispatcher) GetEnv() []string {
	return util.CopyStrSlice(d.env)
}

func (d *Dispatcher) GetClientVersion() *version.Info {
	return d.clientVersion
}

const kubeConfigFlagSetName = "dispatcher-kube-config"

// InitKubeConfigFlags returns the ConfigFlags struct filled in with
// kube config values parsed from command line arguments. These flag values can
// affect the server version query. Therefore, the set of kubeConfigFlags MUST
// match the set used in the regular kubectl binary.
func (d *Dispatcher) InitKubeConfigFlags() (*genericclioptions.ConfigFlags, error) {

	kubeConfigFlagSet := logging.NewFlagSet(kubeConfigFlagSetName)

	unusedParameter := true // Could be either true or false
	kubeConfigFlags := genericclioptions.NewConfigFlags(unusedParameter)
	kubeConfigFlags.AddFlags(kubeConfigFlagSet)

	// Remove help flags, since these are special-cased in pflag.Parse,
	// and handled in the dispatcher instead of passed to versioned binary.
	args := util.FilterList(d.GetArgs(), logging.HelpFlags)
	if err := kubeConfigFlagSet.Parse(args[1:]); err != nil {
		return nil, err
	}
	kubeConfigFlagSet.VisitAll(func(flag *pflag.Flag) {
		klog.Infof("KubeConfig Flag: --%s=%q", flag.Name, flag.Value)
	})

	return kubeConfigFlags, nil
}

// Dispatch attempts to execute a matching version of kubectl based on the
// version of the APIServer. If successful, this method will not return, since
// current process will be overwritten (see execve(2)). Otherwise, this method
// returns an error describing why the execution could not happen.
func (d *Dispatcher) Dispatch() error {
	// Fetch the server version and generate the kubectl binary full file path
	// from this version.
	// Example:
	//   serverVersion=1.11 -> /home/seans/go/bin/kubectl.1.11
	kubeConfigFlags, err := d.InitKubeConfigFlags()
	if err != nil {
		return err
	}
	svclient := client.NewServerVersionClient(kubeConfigFlags)
	svclient.SetRequestTimeout(requestTimeout)
	svclient.SetCacheMaxAge(cacheMaxAge)
	serverVersion, err := svclient.ServerVersion()
	if err != nil {
		return err
	}
	klog.Infof("Server Version: %s", serverVersion.GitVersion)
	klog.Infof("Client Version: %s", d.GetClientVersion().GitVersion)
	if util.VersionMatch(d.GetClientVersion(), serverVersion) {
		// TODO(seans): Consider changing to return a bool as well as error, since
		// this isn't really an error.
		return fmt.Errorf("Client/Server version match--fall through to default")
	}

	kubectlFilepath, err := d.filepathBuilder.VersionedFilePath(serverVersion)
	if err != nil {
		return err
	}
	if err := d.filepathBuilder.ValidateFilepath(kubectlFilepath); err != nil {
		return err
	}

	// Delegate to the versioned kubectl binary. This overwrites the current process
	// (by calling execve(2) system call), and it does not return on success.
	klog.Infof("kubectl dispatching: %s\n", kubectlFilepath)
	return syscall.Exec(kubectlFilepath, d.GetArgs(), d.GetEnv())
}

// Execute is the entry point to the dispatcher. It passes in the current client
// version, which is used to determine if a delegation is necessary. If this function
// successfully delegates, then it will NOT return, since the current process will be
// overwritten (see execve(2)). If this function does not delegate, it merely falls
// through. This function assumes logging has been initialized before it is run;
// otherwise, log statements will not work.
func Execute(clientVersion *version.Info) {
	klog.Info("Starting dispatcher")
	filepathBuilder := filepath.NewFilepathBuilder(&filepath.ExeDirGetter{}, os.Stat)
	dispatcher := NewDispatcher(os.Args, os.Environ(), clientVersion, filepathBuilder)
	if err := dispatcher.Dispatch(); err != nil {
		klog.Warningf("Dispatch error: %v", err)
	}
}
<commit_msg>Validate client version for dispatcher.Execute()<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dispatcher

import (
	"fmt"
	"os"
	"syscall"

	"github.com/kubectl-dispatcher/pkg/client"
	"github.com/kubectl-dispatcher/pkg/filepath"
	"github.com/kubectl-dispatcher/pkg/logging"
	"github.com/kubectl-dispatcher/pkg/util"
	"github.com/spf13/pflag"
	"k8s.io/apimachinery/pkg/version"
	"k8s.io/cli-runtime/pkg/genericclioptions"

	// klog calls in this file assume it has been initialized beforehand
	"k8s.io/klog"
)

const requestTimeout = "5s"     // Timeout for server version query
const cacheMaxAge = 2 * 60 * 60 // 2 hours in seconds

type Dispatcher struct {
	args            []string
	env             []string
	clientVersion   *version.Info
	filepathBuilder *filepath.FilepathBuilder
}

// NewDispatcher returns a new pointer to a Dispatcher struct.
func NewDispatcher(args []string, env []string,
	clientVersion *version.Info,
	filepathBuilder *filepath.FilepathBuilder) *Dispatcher {

	return &Dispatcher{
		args:            args,
		env:             env,
		clientVersion:   clientVersion,
		filepathBuilder: filepathBuilder,
	}
}

// GetArgs returns a copy of the slice of strings representing the command line arguments.
func (d *Dispatcher) GetArgs() []string {
	return util.CopyStrSlice(d.args)
}

// GetEnv returns a copy of the slice of environment variables.
func (d *Dispatcher) GetEnv() []string {
	return util.CopyStrSlice(d.env)
}

func (d *Dispatcher) GetClientVersion() *version.Info {
	return d.clientVersion
}

const kubeConfigFlagSetName = "dispatcher-kube-config"

// InitKubeConfigFlags returns the ConfigFlags struct filled in with
// kube config values parsed from command line arguments. These flag values can
// affect the server version query. Therefore, the set of kubeConfigFlags MUST
// match the set used in the regular kubectl binary.
func (d *Dispatcher) InitKubeConfigFlags() (*genericclioptions.ConfigFlags, error) {

	kubeConfigFlagSet := logging.NewFlagSet(kubeConfigFlagSetName)

	unusedParameter := true // Could be either true or false
	kubeConfigFlags := genericclioptions.NewConfigFlags(unusedParameter)
	kubeConfigFlags.AddFlags(kubeConfigFlagSet)

	// Remove help flags, since these are special-cased in pflag.Parse,
	// and handled in the dispatcher instead of passed to versioned binary.
	args := util.FilterList(d.GetArgs(), logging.HelpFlags)
	if err := kubeConfigFlagSet.Parse(args[1:]); err != nil {
		return nil, err
	}
	kubeConfigFlagSet.VisitAll(func(flag *pflag.Flag) {
		klog.Infof("KubeConfig Flag: --%s=%q", flag.Name, flag.Value)
	})

	return kubeConfigFlags, nil
}

// Dispatch attempts to execute a matching version of kubectl based on the
// version of the APIServer. If successful, this method will not return, since
// current process will be overwritten (see execve(2)). Otherwise, this method
// returns an error describing why the execution could not happen.
func (d *Dispatcher) Dispatch() error {
	// Fetch the server version and generate the kubectl binary full file path
	// from this version.
	// Example:
	//   serverVersion=1.11 -> /home/seans/go/bin/kubectl.1.11
	kubeConfigFlags, err := d.InitKubeConfigFlags()
	if err != nil {
		return err
	}
	svclient := client.NewServerVersionClient(kubeConfigFlags)
	svclient.SetRequestTimeout(requestTimeout)
	svclient.SetCacheMaxAge(cacheMaxAge)
	serverVersion, err := svclient.ServerVersion()
	if err != nil {
		return err
	}
	klog.Infof("Server Version: %s", serverVersion.GitVersion)
	klog.Infof("Client Version: %s", d.GetClientVersion().GitVersion)
	if util.VersionMatch(d.GetClientVersion(), serverVersion) {
		// TODO(seans): Consider changing to return a bool as well as error, since
		// this isn't really an error.
		return fmt.Errorf("Client/Server version match--fall through to default")
	}

	kubectlFilepath, err := d.filepathBuilder.VersionedFilePath(serverVersion)
	if err != nil {
		return err
	}
	if err := d.filepathBuilder.ValidateFilepath(kubectlFilepath); err != nil {
		return err
	}

	// Delegate to the versioned kubectl binary. This overwrites the current process
	// (by calling execve(2) system call), and it does not return on success.
	klog.Infof("kubectl dispatching: %s\n", kubectlFilepath)
	return syscall.Exec(kubectlFilepath, d.GetArgs(), d.GetEnv())
}

// Execute is the entry point to the dispatcher. It passes in the current client
// version, which is used to determine if a delegation is necessary. If this function
// successfully delegates, then it will NOT return, since the current process will be
// overwritten (see execve(2)). If this function does not delegate, it merely falls
// through. This function assumes logging has been initialized before it is run;
// otherwise, log statements will not work.
func Execute(clientVersion *version.Info) {
	klog.Info("Starting dispatcher")
	if clientVersion == nil {
		clientVersion = &version.Info{}
	}
	filepathBuilder := filepath.NewFilepathBuilder(&filepath.ExeDirGetter{}, os.Stat)
	dispatcher := NewDispatcher(os.Args, os.Environ(), clientVersion, filepathBuilder)
	if err := dispatcher.Dispatch(); err != nil {
		klog.Warningf("Dispatch error: %v", err)
	}
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"github.com/hashicorp/go-multierror"
	"github.com/pkg/errors"
	gmailv1 "google.golang.org/api/gmail/v1"

	"github.com/mbrt/gmailctl/pkg/filter"
	"github.com/mbrt/gmailctl/pkg/gmail"
)

// Import exports Gmail filters into Gmail API objects.
//
// If some filter is invalid, the import skips it and returns only the
// valid ones, but records and returns the error in the end.
func Import(filters []*gmailv1.Filter, lmap LabelMap) (filter.Filters, error) {
	res := filter.Filters{}
	var reserr error

	for _, gfilter := range filters {
		impFilter, err := importFilter(gfilter, lmap)
		if err != nil {
			// We don't want to return here, but continue and skip the problematic filter
			err = errors.Wrap(err, fmt.Sprintf("error importing filter '%s'", gfilter.Id))
			reserr = multierror.Append(reserr, err)
		} else {
			res = append(res, impFilter)
		}
	}

	return res, reserr
}

func importFilter(gf *gmailv1.Filter, lmap LabelMap) (filter.Filter, error) {
	action, err := importAction(gf.Action, lmap)
	if err != nil {
		return filter.Filter{}, errors.Wrap(err, "error importing action")
	}
	criteria, err := importCriteria(gf.Criteria)
	if err != nil {
		return filter.Filter{}, errors.Wrap(err, "error importing criteria")
	}
	return filter.Filter{
		ID:       gf.Id,
		Action:   action,
		Criteria: criteria,
	}, nil
}

func importAction(action *gmailv1.FilterAction, lmap LabelMap) (filter.Actions, error) {
	res := filter.Actions{}
	if action == nil {
		return res, errors.New("empty action")
	}
	if err := importAddLabels(&res, action.AddLabelIds, lmap); err != nil {
		return res, err
	}
	err := importRemoveLabels(&res, action.RemoveLabelIds)

	if res.Empty() {
		return res, errors.New("empty or unsupported action")
	}
	return res, err
}

func importAddLabels(res *filter.Actions, addLabelIDs []string, lmap LabelMap) error {
	for _, labelID := range addLabelIDs {
		category := importCategory(labelID)
		if category != "" {
			if res.Category != "" {
				return errors.Errorf("multiple categories: '%s', '%s'", category, res.Category)
			}
			res.Category = category
			continue
		}

		switch labelID {
		case labelIDTrash:
			res.Delete = true
		case labelIDImportant:
			res.MarkImportant = true
		case labelIDStar:
			res.Star = true
		default:
			// it should be a label to add
			labelName, ok := lmap.IDToName(labelID)
			if !ok {
				return errors.Errorf("unknown label ID '%s'", labelID)
			}
			res.AddLabel = labelName
		}
	}
	return nil
}

func importRemoveLabels(res *filter.Actions, removeLabelIDs []string) error {
	for _, labelID := range removeLabelIDs {
		switch labelID {
		case labelIDInbox:
			res.Archive = true
		case labelIDUnread:
			res.MarkRead = true
		case labelIDImportant:
			res.MarkNotImportant = true
		case labelIDSpam:
			res.MarkNotSpam = true
		default:
			// filters not added by us are not supported
			return errors.Errorf("unupported label to remove '%s'", labelID)
		}
	}
	return nil
}

func importCategory(labelID string) gmail.Category {
	switch labelID {
	case labelIDCategoryPersonal:
		return gmail.CategoryPersonal
	case labelIDCategorySocial:
		return gmail.CategorySocial
	case labelIDCategoryUpdates:
		return gmail.CategoryUpdates
	case labelIDCategoryForums:
		return gmail.CategoryForums
	case labelIDCategoryPromotions:
		return gmail.CategoryPromotions
	default:
		return ""
	}
}

func importCriteria(criteria *gmailv1.FilterCriteria) (filter.Criteria, error) {
	if criteria == nil {
		return filter.Criteria{}, errors.New("empty criteria")
	}
	return filter.Criteria{
		From:    criteria.From,
		To:      criteria.To,
		Subject: criteria.Subject,
		Query:   criteria.Query,
	}, nil
}
<commit_msg>Fix import of filters with non-empty negatedQuery.<commit_after>package api

import (
	"fmt"
	"github.com/hashicorp/go-multierror"
	"github.com/pkg/errors"
	gmailv1 "google.golang.org/api/gmail/v1"

	"github.com/mbrt/gmailctl/pkg/filter"
	"github.com/mbrt/gmailctl/pkg/gmail"
)

// Import exports Gmail filters into Gmail API objects.
//
// If some filter is invalid, the import skips it and returns only the
// valid ones, but records and returns the error in the end.
func Import(filters []*gmailv1.Filter, lmap LabelMap) (filter.Filters, error) {
	res := filter.Filters{}
	var reserr error

	for _, gfilter := range filters {
		impFilter, err := importFilter(gfilter, lmap)
		if err != nil {
			// We don't want to return here, but continue and skip the problematic filter
			err = errors.Wrap(err, fmt.Sprintf("error importing filter '%s'", gfilter.Id))
			reserr = multierror.Append(reserr, err)
		} else {
			res = append(res, impFilter)
		}
	}

	return res, reserr
}

func importFilter(gf *gmailv1.Filter, lmap LabelMap) (filter.Filter, error) {
	action, err := importAction(gf.Action, lmap)
	if err != nil {
		return filter.Filter{}, errors.Wrap(err, "error importing action")
	}
	criteria, err := importCriteria(gf.Criteria)
	if err != nil {
		return filter.Filter{}, errors.Wrap(err, "error importing criteria")
	}
	return filter.Filter{
		ID:       gf.Id,
		Action:   action,
		Criteria: criteria,
	}, nil
}

func importAction(action *gmailv1.FilterAction, lmap LabelMap) (filter.Actions, error) {
	res := filter.Actions{}
	if action == nil {
		return res, errors.New("empty action")
	}
	if err := importAddLabels(&res, action.AddLabelIds, lmap); err != nil {
		return res, err
	}
	err := importRemoveLabels(&res, action.RemoveLabelIds)

	if res.Empty() {
		return res, errors.New("empty or unsupported action")
	}
	return res, err
}

func importAddLabels(res *filter.Actions, addLabelIDs []string, lmap LabelMap) error {
	for _, labelID := range addLabelIDs {
		category := importCategory(labelID)
		if category != "" {
			if res.Category != "" {
				return errors.Errorf("multiple categories: '%s', '%s'", category, res.Category)
			}
			res.Category = category
			continue
		}

		switch labelID {
		case labelIDTrash:
			res.Delete = true
		case labelIDImportant:
			res.MarkImportant = true
		case labelIDStar:
			res.Star = true
		default:
			// it should be a label to add
			labelName, ok := lmap.IDToName(labelID)
			if !ok {
				return errors.Errorf("unknown label ID '%s'", labelID)
			}
			res.AddLabel = labelName
		}
	}
	return nil
}

func importRemoveLabels(res *filter.Actions, removeLabelIDs []string) error {
	for _, labelID := range removeLabelIDs {
		switch labelID {
		case labelIDInbox:
			res.Archive = true
		case labelIDUnread:
			res.MarkRead = true
		case labelIDImportant:
			res.MarkNotImportant = true
		case labelIDSpam:
			res.MarkNotSpam = true
		default:
			// filters not added by us are not supported
			return errors.Errorf("unupported label to remove '%s'", labelID)
		}
	}
	return nil
}

func importCategory(labelID string) gmail.Category {
	switch labelID {
	case labelIDCategoryPersonal:
		return gmail.CategoryPersonal
	case labelIDCategorySocial:
		return gmail.CategorySocial
	case labelIDCategoryUpdates:
		return gmail.CategoryUpdates
	case labelIDCategoryForums:
		return gmail.CategoryForums
	case labelIDCategoryPromotions:
		return gmail.CategoryPromotions
	default:
		return ""
	}
}

func importCriteria(criteria *gmailv1.FilterCriteria) (filter.Criteria, error) {
	if criteria == nil {
		return filter.Criteria{}, errors.New("empty criteria")
	}
	query := criteria.Query

	// We don't ever generate negated queries, so supporting them only for the import
	// is not worth the effort. Instead update the regular query field with an
	// equivalent expression.
	// Note that elements in the negated query are by default in OR together, according
	// to GMail behavior.
	if criteria.NegatedQuery != "" {
		query = fmt.Sprintf("%s -{%s}", query, criteria.NegatedQuery)
	}

	return filter.Criteria{
		From:    criteria.From,
		To:      criteria.To,
		Subject: criteria.Subject,
		Query:   query,
	}, nil
}
<|endoftext|>


<commit_before>package builtins

import (
	"testing"

	"github.com/coel-lang/coel/src/lib/core"
	"github.com/coel-lang/coel/src/lib/systemt"
	"github.com/stretchr/testify/assert"
)

func TestY(t *testing.T) {
	for _, n := range []float64{0, 1, 2, 3, 4, 5, 6, 42, 100} {
		n1 := lazyFactorial(core.NewNumber(n))
		n2 := strictFactorial(n)

		t.Logf("%d: %f == %f?\n", int(n), n1, n2)

		assert.Equal(t, n1, n2)
	}
}

func strictFactorial(n float64) float64 {
	if n == 0 {
		return 1
	}

	return n * strictFactorial(n-1)
}

func lazyFactorial(t *core.Thunk) float64 {
	return float64(core.PApp(core.PApp(Y, lazyFactorialImpl), t).Eval().(core.NumberType))
}

var lazyFactorialImpl = core.NewLazyFunction(
	core.NewSignature([]string{"me", "num"}, nil, "", nil, nil, ""),
	func(ts ...*core.Thunk) core.Value {
		return core.PApp(core.If,
			core.PApp(core.Equal, ts[1], core.NewNumber(0)),
			core.NewNumber(1),
			core.PApp(core.Mul,
				ts[1],
				core.PApp(ts[0], append([]*core.Thunk{core.PApp(core.Sub, ts[1], core.NewNumber(1))}, ts[2:]...)...)))
	})

func BenchmarkY(b *testing.B) {
	go systemt.RunDaemons()
	core.PApp(toZero, core.NewNumber(float64(b.N))).Eval()
}

func BenchmarkGoY(b *testing.B) {
	toZeroGo(float64(b.N))
}

var toZero = core.PApp(Y, core.NewLazyFunction(
	core.NewSignature([]string{"me", "num"}, nil, "", nil, nil, ""),
	func(ts ...*core.Thunk) core.Value {
		return core.PApp(core.If,
			core.PApp(core.Equal, ts[1], core.NewNumber(0)),
			core.NewString("Benchmark finished!"),
			core.PApp(ts[0], core.PApp(core.Sub, ts[1], core.NewNumber(1))))
	}))

func toZeroGo(f float64) string {
	t := core.NewNumber(f)
	n := t.Eval().(core.NumberType)

	for n > 0 {
		t = core.PApp(core.Sub, t, core.NewNumber(1))
		n = t.Eval().(core.NumberType)
	}

	return "Benchmark finished!"
}
<commit_msg>Benchmark Y combinator with minimal infinite recursion<commit_after>package builtins

import (
	"testing"

	"github.com/coel-lang/coel/src/lib/core"
	"github.com/coel-lang/coel/src/lib/systemt"
	"github.com/stretchr/testify/assert"
)

func TestY(t *testing.T) {
	for _, n := range []float64{0, 1, 2, 3, 4, 5, 6, 42, 100} {
		n1 := lazyFactorial(core.NewNumber(n))
		n2 := strictFactorial(n)

		t.Logf("%d: %f == %f?\n", int(n), n1, n2)

		assert.Equal(t, n1, n2)
	}
}

func strictFactorial(n float64) float64 {
	if n == 0 {
		return 1
	}

	return n * strictFactorial(n-1)
}

func lazyFactorial(t *core.Thunk) float64 {
	return float64(core.PApp(core.PApp(Y, lazyFactorialImpl), t).Eval().(core.NumberType))
}

var lazyFactorialImpl = core.NewLazyFunction(
	core.NewSignature([]string{"me", "num"}, nil, "", nil, nil, ""),
	func(ts ...*core.Thunk) core.Value {
		return core.PApp(core.If,
			core.PApp(core.Equal, ts[1], core.NewNumber(0)),
			core.NewNumber(1),
			core.PApp(core.Mul,
				ts[1],
				core.PApp(ts[0], append([]*core.Thunk{core.PApp(core.Sub, ts[1], core.NewNumber(1))}, ts[2:]...)...)))
	})

func BenchmarkYInfiniteRecursion(b *testing.B) {
	t := core.PApp(Y, core.NewLazyFunction(
		core.NewSignature([]string{"me"}, nil, "", nil, nil, ""),
		func(ts ...*core.Thunk) core.Value {
			return ts[0]
		}))

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		t.Eval()
		t = core.PApp(t)
	}
}

func BenchmarkY(b *testing.B) {
	go systemt.RunDaemons()
	core.PApp(toZero, core.NewNumber(float64(b.N))).Eval()
}

func BenchmarkGoY(b *testing.B) {
	toZeroGo(float64(b.N))
}

var toZero = core.PApp(Y, core.NewLazyFunction(
	core.NewSignature([]string{"me", "num"}, nil, "", nil, nil, ""),
	func(ts ...*core.Thunk) core.Value {
		return core.PApp(core.If,
			core.PApp(core.Equal, ts[1], core.NewNumber(0)),
			core.NewString("Benchmark finished!"),
			core.PApp(ts[0], core.PApp(core.Sub, ts[1], core.NewNumber(1))))
	}))

func toZeroGo(f float64) string {
	t := core.NewNumber(f)
	n := t.Eval().(core.NumberType)

	for n > 0 {
		t = core.PApp(core.Sub, t, core.NewNumber(1))
		n = t.Eval().(core.NumberType)
	}

	return "Benchmark finished!"
}
<|endoftext|>


<commit_before>package core

import "reflect"

type Object interface{}

type callable interface {
	call(Arguments) Object
}

type equalable interface {
	equal(equalable) Object
}

var Equal = NewStrictFunction(
	NewSignature(
		[]string{"x", "y"}, []OptionalArgument{}, "",
		[]string{}, []OptionalArgument{}, "",
	),
	func(os ...Object) Object {
		var es [2]equalable

		for i, o := range os {
			e, ok := o.(equalable)

			if !ok {
				return TypeError(o, "Equalable")
			}

			es[i] = e
		}

		if !areSameType(es[0], es[1]) {
			return False
		}

		return es[0].equal(es[1])
	})

type listable interface {
	toList() Object
}

var ToList = NewStrictFunction(
	NewSignature(
		[]string{"listLike"}, []OptionalArgument{}, "",
		[]string{}, []OptionalArgument{}, "",
	),
	func(os ...Object) Object {
		o := os[0]
		l, ok := o.(listable)

		if !ok {
			return TypeError(o, "Listable")
		}

		return l.toList()
	})

// This interface should not be used in exported Functions and exists only to
// make keys of DictionaryType and elements of setType ordered.
type ordered interface {
	less(ordered) bool // can panic
}

func less(x1, x2 interface{}) bool {
	if !areSameType(x1, x2) {
		return reflect.TypeOf(x1).Name() < reflect.TypeOf(x2).Name()
	}

	o1, ok := x1.(ordered)

	if !ok {
		panic(notOrderedError(x1))
	}

	o2, ok := x2.(ordered)

	if !ok {
		panic(notOrderedError(x2))
	}

	return o1.less(o2)
}

func areSameType(x1, x2 interface{}) bool {
	return reflect.TypeOf(x1) == reflect.TypeOf(x2)
}

type mergable interface {
	merge(ts ...*Thunk) Object
}

var Merge = NewLazyFunction(
	NewSignature(
		[]string{"x"}, []OptionalArgument{}, "ys",
		[]string{}, []OptionalArgument{}, "",
	),
	func(ts ...*Thunk) Object {
		o := ts[0].Eval()
		m, ok := o.(mergable)

		if !ok {
			return TypeError(o, "Mergable")
		}

		o = ts[1].Eval()
		l, ok := o.(ListType)

		if !ok {
			return notListError(o)
		}

		ts, err := l.ToThunks()

		if err != nil {
			return err
		}

		if len(ts) == 0 {
			return m
		}

		return m.merge(ts...)
	})
<commit_msg>Define Delete function<commit_after>package core

import "reflect"

type Object interface{}

type callable interface {
	call(Arguments) Object
}

type equalable interface {
	equal(equalable) Object
}

var Equal = NewStrictFunction(
	NewSignature(
		[]string{"x", "y"}, []OptionalArgument{}, "",
		[]string{}, []OptionalArgument{}, "",
	),
	func(os ...Object) Object {
		var es [2]equalable

		for i, o := range os {
			e, ok := o.(equalable)

			if !ok {
				return TypeError(o, "Equalable")
			}

			es[i] = e
		}

		if !areSameType(es[0], es[1]) {
			return False
		}

		return es[0].equal(es[1])
	})

type listable interface {
	toList() Object
}

var ToList = NewStrictFunction(
	NewSignature(
		[]string{"listLike"}, []OptionalArgument{}, "",
		[]string{}, []OptionalArgument{}, "",
	),
	func(os ...Object) Object {
		o := os[0]
		l, ok := o.(listable)

		if !ok {
			return TypeError(o, "Listable")
		}

		return l.toList()
	})

// This interface should not be used in exported Functions and exists only to
// make keys of DictionaryType and elements of setType ordered.
type ordered interface {
	less(ordered) bool // can panic
}

func less(x1, x2 interface{}) bool {
	if !areSameType(x1, x2) {
		return reflect.TypeOf(x1).Name() < reflect.TypeOf(x2).Name()
	}

	o1, ok := x1.(ordered)

	if !ok {
		panic(notOrderedError(x1))
	}

	o2, ok := x2.(ordered)

	if !ok {
		panic(notOrderedError(x2))
	}

	return o1.less(o2)
}

func areSameType(x1, x2 interface{}) bool {
	return reflect.TypeOf(x1) == reflect.TypeOf(x2)
}

type mergable interface {
	merge(ts ...*Thunk) Object
}

var Merge = NewLazyFunction(
	NewSignature(
		[]string{"x"}, []OptionalArgument{}, "ys",
		[]string{}, []OptionalArgument{}, "",
	),
	func(ts ...*Thunk) Object {
		o := ts[0].Eval()
		m, ok := o.(mergable)

		if !ok {
			return TypeError(o, "Mergable")
		}

		o = ts[1].Eval()
		l, ok := o.(ListType)

		if !ok {
			return notListError(o)
		}

		ts, err := l.ToThunks()

		if err != nil {
			return err
		}

		if len(ts) == 0 {
			return m
		}

		return m.merge(ts...)
	})

type deletable interface {
	delete(Object) (deletable, Object)
}

var Delete = NewStrictFunction(
	NewSignature(
		[]string{"collection"}, []OptionalArgument{}, "elems",
		[]string{}, []OptionalArgument{}, "",
	),
	func(os ...Object) Object {
		d, ok := os[0].(deletable)

		if !ok {
			return TypeError(os[0], "Deletable")
		}

		l, ok := os[1].(ListType)

		if !ok {
			return notListError(os[1])
		}

		os, err := l.ToObjects()

		if err != nil {
			return err
		}

		for _, o := range os {
			if _, ok := o.(ErrorType); ok {
				return o
			}

			var err Object
			d, err = d.delete(o)

			if err != nil {
				return err
			}
		}

		return d
	})
<|endoftext|>