<commit_before>package firego

import (
	"fmt"
	"strconv"
	"strings"
)

// StartAt creates a new Firebase reference with the
// requested StartAt configuration. The value that is passed in
// is automatically escape if it is a string value.
//
//    StartAt(7)        // -> endAt=7
//    StartAt("foo")    // -> endAt="foo"
//    StartAt(`"foo"`)  // -> endAt="foo"
//
// Reference https://www.firebase.com/docs/rest/guide/retrieving-data.html#section-rest-filtering
func (fb *Firebase) StartAt(value string) *Firebase {
	c := fb.copy()
	if value != "" {
		c.params.Set(startAtParam, escapeString(value))
	} else {
		c.params.Del(startAtParam)
	}
	return c
}

// EndAt creates a new Firebase reference with the
// requested EndAt configuration. The value that is passed in
// is automatically escape if it is a string value.
//
//    EndAt(7)        // -> endAt=7
//    EndAt("foo")    // -> endAt="foo"
//    EndAt(`"foo"`)  // -> endAt="foo"
//
// Reference https://www.firebase.com/docs/rest/guide/retrieving-data.html#section-rest-filtering
func (fb *Firebase) EndAt(value string) *Firebase {
	c := fb.copy()
	if value != "" {
		c.params.Set(endAtParam, escapeString(value))
	} else {
		c.params.Del(endAtParam)
	}
	return c
}

// OrderBy creates a new Firebase reference with the
// requested OrderBy configuration. The value that is passed in
// is automatically escape if it is a string value.
//
//    OrderBy(7)       // -> endAt=7
//    OrderBy("foo")   // -> endAt="foo"
//    OrderBy(`"foo"`) // -> endAt="foo"
//
// Reference https://www.firebase.com/docs/rest/guide/retrieving-data.html#section-rest-filtering
func (fb *Firebase) OrderBy(value string) *Firebase {
	c := fb.copy()
	if value != "" {
		c.params.Set(orderByParam, escapeString(value))
	} else {
		c.params.Del(orderByParam)
	}
	return c
}

// EqualTo sends the query string equalTo so that one can find a single value
//
// Reference https://www.firebase.com/docs/rest/guide/retrieving-data.html#section-rest-filtering
func (fb *Firebase) EqualTo(value string) *Firebase {
	c := fb.copy()
	if value != "" {
		c.params.Set(equalToParam, escapeString(value))
	} else {
		c.params.Del(equalToParam)
	}
	return c
}

func escapeString(s string) string {
	_, errNotInt := strconv.ParseInt(s, 10, 64)
	_, errNotBool := strconv.ParseBool(s)
	if errNotInt == nil || errNotBool == nil {
		// we shouldn't escape bools or ints
		return s
	}
	return fmt.Sprintf(`%q`, strings.Trim(s, `"`))
}

func escapeParameter(s interface{}) string {
	switch s.(type) {
	case string:
		return fmt.Sprintf(`%q`, strings.Trim(s.(string), `"`))
	default:
		return fmt.Sprintf(`%v`, s)
	}
}

// LimitToFirst creates a new Firebase reference with the
// requested limitToFirst configuration.
//
// Reference https://www.firebase.com/docs/rest/api/#section-param-query
func (fb *Firebase) LimitToFirst(value int64) *Firebase {
	c := fb.copy()
	if value > 0 {
		c.params.Set(limitToFirstParam, strconv.FormatInt(value, 10))
	} else {
		c.params.Del(limitToFirstParam)
	}
	return c
}

// LimitToLast creates a new Firebase reference with the
// requested limitToLast configuration.
//
// Reference https://www.firebase.com/docs/rest/api/#section-param-query
func (fb *Firebase) LimitToLast(value int64) *Firebase {
	c := fb.copy()
	if value > 0 {
		c.params.Set(limitToLastParam, strconv.FormatInt(value, 10))
	} else {
		c.params.Del(limitToLastParam)
	}
	return c
}

// Shallow limits the depth of the data returned when calling Value.
// If the data at the location is a JSON primitive (string, number or boolean),
// its value will be returned. If the data is a JSON object, the values
// for each key will be truncated to true.
//
// Reference https://www.firebase.com/docs/rest/api/#section-param-shallow
func (fb *Firebase) Shallow(v bool) {
	if v {
		fb.params.Set(shallowParam, "true")
	} else {
		fb.params.Del(shallowParam)
	}
}

// IncludePriority determines whether or not to ask Firebase
// for the values priority. By default, the priority is not returned.
//
// Reference https://www.firebase.com/docs/rest/api/#section-param-format
func (fb *Firebase) IncludePriority(v bool) {
	if v {
		fb.params.Set(formatParam, formatVal)
	} else {
		fb.params.Del(formatParam)
	}
}
<commit_msg>Update comments about the current behavior, update links<commit_after>package firego

import (
	"fmt"
	"strconv"
	"strings"
)

// StartAt creates a new Firebase reference with the
// requested StartAt configuration. The value that is passed in
// is automatically escaped if it is a string value.
// Numeric strings are automatically converted to numbers.
//
//    StartAt(7)        // -> startAt=7
//    StartAt("7")      // -> startAt=7
//    StartAt("foo")    // -> startAt="foo"
//    StartAt(`"foo"`)  // -> startAt="foo"
//
// Reference https://firebase.google.com/docs/database/rest/retrieve-data#section-rest-filtering
func (fb *Firebase) StartAt(value string) *Firebase {
	c := fb.copy()
	if value != "" {
		c.params.Set(startAtParam, escapeString(value))
	} else {
		c.params.Del(startAtParam)
	}
	return c
}

// EndAt creates a new Firebase reference with the
// requested EndAt configuration. The value that is passed in
// is automatically escaped if it is a string value.
// Numeric strings are automatically converted to numbers.
//
//    EndAt(7)        // -> endAt=7
//    EndAt("7")      // -> endAt=7
//    EndAt("foo")    // -> endAt="foo"
//    EndAt(`"foo"`)  // -> endAt="foo"
//
// Reference https://firebase.google.com/docs/database/rest/retrieve-data#section-rest-filtering
func (fb *Firebase) EndAt(value string) *Firebase {
	c := fb.copy()
	if value != "" {
		c.params.Set(endAtParam, escapeString(value))
	} else {
		c.params.Del(endAtParam)
	}
	return c
}

// OrderBy creates a new Firebase reference with the
// requested OrderBy configuration. The value that is passed in
// is automatically escape if it is a string value.
//
//    OrderBy("foo")   // -> orderBy="foo"
//    OrderBy(`"foo"`) // -> orderBy="foo"
//    OrderBy("$key")  // -> orderBy="$key"
//
// Reference https://firebase.google.com/docs/database/rest/retrieve-data#orderby
func (fb *Firebase) OrderBy(value string) *Firebase {
	c := fb.copy()
	if value != "" {
		c.params.Set(orderByParam, escapeString(value))
	} else {
		c.params.Del(orderByParam)
	}
	return c
}

// EqualTo sends the query string equalTo so that one can find nodes with
// exactly matching values. The value that is passed in is automatically escaped
// if it is a string value.
// Numeric strings are automatically converted to numbers.
//
//    EqualTo(7)        // -> equalTo=7
//    EqualTo("7")      // -> equalTo=7
//    EqualTo("foo")    // -> equalTo="foo"
//    EqualTo(`"foo"`)  // -> equalTo="foo"
//
// Reference https://firebase.google.com/docs/database/rest/retrieve-data#section-rest-filtering
func (fb *Firebase) EqualTo(value string) *Firebase {
	c := fb.copy()
	if value != "" {
		c.params.Set(equalToParam, escapeString(value))
	} else {
		c.params.Del(equalToParam)
	}
	return c
}

func escapeString(s string) string {
	_, errNotInt := strconv.ParseInt(s, 10, 64)
	_, errNotBool := strconv.ParseBool(s)
	if errNotInt == nil || errNotBool == nil {
		// we shouldn't escape bools or ints
		return s
	}
	return fmt.Sprintf(`%q`, strings.Trim(s, `"`))
}

func escapeParameter(s interface{}) string {
	switch s.(type) {
	case string:
		return fmt.Sprintf(`%q`, strings.Trim(s.(string), `"`))
	default:
		return fmt.Sprintf(`%v`, s)
	}
}

// LimitToFirst creates a new Firebase reference with the
// requested limitToFirst configuration.
//
// Reference https://firebase.google.com/docs/database/rest/retrieve-data#limit-queries
func (fb *Firebase) LimitToFirst(value int64) *Firebase {
	c := fb.copy()
	if value > 0 {
		c.params.Set(limitToFirstParam, strconv.FormatInt(value, 10))
	} else {
		c.params.Del(limitToFirstParam)
	}
	return c
}

// LimitToLast creates a new Firebase reference with the
// requested limitToLast configuration.
//
// Reference https://firebase.google.com/docs/database/rest/retrieve-data#limit-queries
func (fb *Firebase) LimitToLast(value int64) *Firebase {
	c := fb.copy()
	if value > 0 {
		c.params.Set(limitToLastParam, strconv.FormatInt(value, 10))
	} else {
		c.params.Del(limitToLastParam)
	}
	return c
}

// Shallow limits the depth of the data returned when calling Value.
// If the data at the location is a JSON primitive (string, number or boolean),
// its value will be returned. If the data is a JSON object, the values
// for each key will be truncated to true.
//
// Reference https://firebase.google.com/docs/database/rest/retrieve-data#shallow
func (fb *Firebase) Shallow(v bool) {
	if v {
		fb.params.Set(shallowParam, "true")
	} else {
		fb.params.Del(shallowParam)
	}
}

// IncludePriority determines whether or not to ask Firebase
// for the values priority. By default, the priority is not returned.
//
// Reference https://www.firebase.com/docs/rest/api/#section-param-format
func (fb *Firebase) IncludePriority(v bool) {
	if v {
		fb.params.Set(formatParam, formatVal)
	} else {
		fb.params.Del(formatParam)
	}
}
<|endoftext|>


<commit_before>package queue

import (
	"sync"
)

type Queue struct {
	push             chan func()
	pop              chan func()
	suspend          chan bool
	close            chan bool
	ConcurrencyLimit int
	wg               sync.WaitGroup
	suspended        bool
}

func Init() (q *Queue) {

	q = &Queue{

		push:    make(chan func()),
		pop:     make(chan func()),
		suspend: make(chan bool),
		close:   make(chan bool),
	}
	go q.run()

	return q

}

func (q *Queue) Push(f func()) {

	q.push <- f
}

func (q *Queue) run() {

	suspended := false
	closed := false
	finishing := false
	count := 0
	buffer := make([]func(), 0)

	exec := func() {

		if !(suspended || closed) {
			for len(buffer) > 0 && (count < q.ConcurrencyLimit || q.ConcurrencyLimit == 0) {

				f := buffer[0]
				buffer = buffer[1:]
				count++
				go func() {

					f()
					defer func() {
						q.pop <- f
					}()
				}()

			}

		}

	}

	cleanUp := func() {
		q.wg.Add(-len(buffer))
		close(q.push)
		close(q.pop)
		close(q.suspend)
		close(q.close)
	}

	for {

		select {

		case closed = <-q.close:
			finishing = true

		case suspended = <-q.suspend:

		case <-q.pop:
			count--
			q.wg.Done()

		case v := <-q.push:

			if !finishing {
				q.wg.Add(1)
				buffer = append(buffer, v)
			}
		}
		exec()
		if count == 0 && finishing {
			if len(buffer) == 0 {
				closed = true
			}
			if closed {
				cleanUp()
				return
			}
		}

	}

}

func (q *Queue) Wait() {

	q.wg.Wait()
}

func (q *Queue) Suspend() {

	q.suspend <- true
	q.suspended = true
}

func (q *Queue) Resume() {

	q.suspend <- false
	q.suspended = false

}

func (q *Queue) Suspended() bool {

	return q.suspended
}

func (q *Queue) Close(immediate bool) {
	q.close <- immediate

}
<commit_msg>added documentation<commit_after>// Package queue is a utility to queue goroutines
package queue

import "sync"

// Queue is the queue.
type Queue struct {
	push             chan func()
	pop              chan func()
	suspend          chan bool
	close            chan bool
	ConcurrencyLimit int
	wg               sync.WaitGroup
	suspended        bool
}
// Init initializes a new queue.
func Init() (q *Queue) {

	q = &Queue{

		push:    make(chan func()),
		pop:     make(chan func()),
		suspend: make(chan bool),
		close:   make(chan bool),
	}
	go q.run()

	return q

}
// Push addes a new function to the end of the Queue
func (q *Queue) Push(f func()) {

	q.push <- f
}

func (q *Queue) run() {

	suspended := false
	closed := false
	finishing := false
	count := 0
	buffer := make([]func(), 0)

	exec := func() {

		if !(suspended || closed) {
			for len(buffer) > 0 && (count < q.ConcurrencyLimit || q.ConcurrencyLimit == 0) {

				f := buffer[0]
				buffer = buffer[1:]
				count++
				go func() {

					f()
					defer func() {
						q.pop <- f
					}()
				}()

			}

		}

	}

	cleanUp := func() {
		q.wg.Add(-len(buffer))
		close(q.push)
		close(q.pop)
		close(q.suspend)
		close(q.close)
	}

	for {

		select {

		case closed = <-q.close:
			finishing = true

		case suspended = <-q.suspend:

		case <-q.pop:
			count--
			q.wg.Done()

		case v := <-q.push:

			if !finishing {
				q.wg.Add(1)
				buffer = append(buffer, v)
			}
		}
		exec()
		if count == 0 && finishing {
			if len(buffer) == 0 {
				closed = true
			}
			if closed {
				cleanUp()
				return
			}
		}

	}

}

// Wait calls Wait on a WaitGroup, which finishes when the Queue is drained or when closed
func (q *Queue) Wait() {

	q.wg.Wait()
}

// Suspend stops the execution of functions on the queue, new functions can still be queued.
func (q *Queue) Suspend() {

	q.suspend <- true
	q.suspended = true
}

// Resume starts the execution of functions on the queue, after Suspend has been called.
func (q *Queue) Resume() {

	q.suspend <- false
	q.suspended = false

}

// Suspended returns whether or not the queue is in a suspended state.
func (q *Queue) Suspended() bool {

	return q.suspended
}

// Close cleanups the queue by closing the internal channels.
// If true is passed as an arguement, the queue closes immediatley only waiting for the currently executing functions to finish.
// If false is passed the queue finishes executing all the functions in the queue.
// When called the queue does not allow for anymore functions to be enqueued.
func (q *Queue) Close(immediate bool) {
	q.close <- immediate

}
<|endoftext|>


<commit_before>package ali_mns

import (
	"fmt"
	"os"
	"strings"
	"time"
)

var (
	DefaultNumOfMessages int32 = 16
	DefaultQPSLimit      int32 = 2000
)

const (
	PROXY_PREFIX = "MNS_PROXY_"
	GLOBAL_PROXY = "MNS_GLOBAL_PROXY"
)

type AliMNSQueue interface {
	Name() string
	SendMessage(message MessageSendRequest) (resp MessageSendResponse, err error)
	BatchSendMessage(messages ...MessageSendRequest) (resp BatchMessageSendResponse, err error)
	ReceiveMessage(respChan chan MessageReceiveResponse, errChan chan error, waitseconds ...int64)
	BatchReceiveMessage(respChan chan BatchMessageReceiveResponse, errChan chan error, numOfMessages int32, waitseconds ...int64)
	PeekMessage(respChan chan MessageReceiveResponse, errChan chan error, interval ...time.Duration)
	BatchPeekMessage(respChan chan BatchMessageReceiveResponse, errChan chan error, numOfMessages int32, interval ...time.Duration)
	DeleteMessage(receiptHandle string) (err error)
	BatchDeleteMessage(receiptHandles ...string) (err error)
	ChangeMessageVisibility(receiptHandle string, visibilityTimeout int64) (resp MessageVisibilityChangeResponse, err error)
	Stop()
}

type MNSQueue struct {
	name       string
	client     MNSClient
	stopChan   chan bool
	qpsLimit   int32
	qpsMonitor *QPSMonitor
	decoder    MNSDecoder
}

func NewMNSQueue(name string, client MNSClient, qps ...int32) AliMNSQueue {
	if name == "" {
		panic("ali_mns: queue name could not be empty")
	}

	queue := new(MNSQueue)
	queue.client = client
	queue.name = name
	queue.stopChan = make(chan bool)
	queue.qpsLimit = DefaultQPSLimit
	queue.decoder = NewAliMNSDecoder()

	var attr QueueAttribute
	if _, err := send(client, queue.decoder, GET, nil, nil, "queues/"+name, &attr); err != nil {
		panic(err)
	}

	if qps != nil && len(qps) == 1 && qps[0] > 0 {
		queue.qpsLimit = qps[0]
	}

	proxyURL := ""
	queueProxyEnvKey := PROXY_PREFIX + strings.Replace(strings.ToUpper(name), "-", "_", -1)
	if url := os.Getenv(queueProxyEnvKey); url != "" {
		proxyURL = url
	}

	client.SetProxy(proxyURL)

	queue.qpsMonitor = NewQPSMonitor(5)

	return queue
}

func (p *MNSQueue) Name() string {
	return p.name
}

func (p *MNSQueue) SendMessage(message MessageSendRequest) (resp MessageSendResponse, err error) {
	p.checkQPS()
	_, err = send(p.client, p.decoder, POST, nil, message, fmt.Sprintf("queues/%s/%s", p.name, "messages"), &resp)
	return
}

func (p *MNSQueue) BatchSendMessage(messages ...MessageSendRequest) (resp BatchMessageSendResponse, err error) {
	if messages == nil || len(messages) == 0 {
		return
	}

	batchRequest := BatchMessageSendRequest{}
	for _, message := range messages {
		batchRequest.Messages = append(batchRequest.Messages, message)
	}

	p.checkQPS()
	_, err = send(p.client, p.decoder, POST, nil, batchRequest, fmt.Sprintf("queues/%s/%s", p.name, "messages"), &resp)
	return
}

func (p *MNSQueue) Stop() {
	p.stopChan <- true
}

func (p *MNSQueue) ReceiveMessage(respChan chan MessageReceiveResponse, errChan chan error, waitseconds ...int64) {
	resource := fmt.Sprintf("queues/%s/%s", p.name, "messages")
	if waitseconds != nil && len(waitseconds) == 1 && waitseconds[0] >= 0 {
		resource = fmt.Sprintf("queues/%s/%s?waitseconds=%d", p.name, "messages", waitseconds[0])
	}

	for {
		resp := MessageReceiveResponse{}
		_, err := send(p.client, p.decoder, GET, nil, nil, resource, &resp)
		if err != nil {
			errChan <- err
		} else {
			respChan <- resp
		}

		p.checkQPS()

		select {
		case _ = <-p.stopChan:
			{
				return
			}
		default:
		}
	}

	return
}

func (p *MNSQueue) BatchReceiveMessage(respChan chan BatchMessageReceiveResponse, errChan chan error, numOfMessages int32, waitseconds ...int64) {
	if numOfMessages <= 0 {
		numOfMessages = DefaultNumOfMessages
	}

	resource := fmt.Sprintf("queues/%s/%s?numOfMessages=%d", p.name, "messages", numOfMessages)
	if waitseconds != nil && len(waitseconds) == 1 && waitseconds[0] >= 0 {
		resource = fmt.Sprintf("queues/%s/%s?numOfMessages=%d&waitseconds=%d", p.name, "messages", numOfMessages, waitseconds[0])
	}

	for {
		resp := BatchMessageReceiveResponse{}
		_, err := send(p.client, p.decoder, GET, nil, nil, resource, &resp)
		if err != nil {
			errChan <- err
		} else {
			respChan <- resp
		}

		p.checkQPS()

		select {
		case _ = <-p.stopChan:
			{
				return
			}
		default:
		}
	}

	return
}

func (p *MNSQueue) PeekMessage(respChan chan MessageReceiveResponse, errChan chan error, interval ...time.Duration) {
	resource := fmt.Sprintf("queues/%s/%s?peekonly=true", p.name, "messages")

	itv := time.Duration(0)
	if len(interval) == 1 {
		itv = interval[0]
	}

	for {
		resp := MessageReceiveResponse{}
		_, err := send(p.client, p.decoder, GET, nil, nil, resource, &resp)
		if err != nil {
			errChan <- err
		} else {
			respChan <- resp
		}

		if itv > 0 {
			time.Sleep(itv)
		}

		select {
		case _ = <-p.stopChan:
			{
				return
			}
		default:
		}
	}
	return
}

func (p *MNSQueue) BatchPeekMessage(respChan chan BatchMessageReceiveResponse, errChan chan error, numOfMessages int32, interval ...time.Duration) {
	if numOfMessages <= 0 {
		numOfMessages = DefaultNumOfMessages
	}

	itv := time.Duration(0)
	if len(interval) == 1 {
		itv = interval[0]
	}

	for {
		resp := BatchMessageReceiveResponse{}
		_, err := send(p.client, p.decoder, GET, nil, nil, fmt.Sprintf("queues/%s/%s?numOfMessages=%d&peekonly=true", p.name, "messages", numOfMessages), &resp)
		if err != nil {
			errChan <- err
		} else {
			respChan <- resp
		}

		if itv > 0 {
			time.Sleep(itv)
		}

		select {
		case _ = <-p.stopChan:
			{
				return
			}
		default:
		}
	}
	return
}

func (p *MNSQueue) DeleteMessage(receiptHandle string) (err error) {
	p.checkQPS()
	_, err = send(p.client, p.decoder, DELETE, nil, nil, fmt.Sprintf("queues/%s/%s?ReceiptHandle=%s", p.name, "messages", receiptHandle), nil)
	return
}

func (p *MNSQueue) BatchDeleteMessage(receiptHandles ...string) (err error) {
	if receiptHandles == nil || len(receiptHandles) == 0 {
		return
	}

	handlers := ReceiptHandles{}

	for _, handler := range receiptHandles {
		handlers.ReceiptHandles = append(handlers.ReceiptHandles, handler)
	}

	p.checkQPS()
	_, err = send(p.client, p.decoder, DELETE, nil, handlers, fmt.Sprintf("queues/%s/%s", p.name, "messages"), nil)
	return
}

func (p *MNSQueue) ChangeMessageVisibility(receiptHandle string, visibilityTimeout int64) (resp MessageVisibilityChangeResponse, err error) {
	p.checkQPS()
	_, err = send(p.client, p.decoder, PUT, nil, nil, fmt.Sprintf("queues/%s/%s?ReceiptHandle=%s&VisibilityTimeout=%d", p.name, "messages", receiptHandle, visibilityTimeout), &resp)
	return
}

func (p *MNSQueue) checkQPS() {
	p.qpsMonitor.Pulse()
	if p.qpsLimit > 0 {
		for p.qpsMonitor.QPS() > p.qpsLimit {
			p.qpsMonitor.Pulse()
			time.Sleep(time.Millisecond * 10)
		}
	}
}
<commit_msg>Update queue.go<commit_after>package ali_mns

import (
	"fmt"
	"os"
	"strings"
	"time"
)

var (
	DefaultNumOfMessages int32 = 16
	DefaultQPSLimit      int32 = 2000
)

const (
	PROXY_PREFIX = "MNS_PROXY_"
	GLOBAL_PROXY = "MNS_GLOBAL_PROXY"
)

type AliMNSQueue interface {
	Name() string
	SendMessage(message MessageSendRequest) (resp MessageSendResponse, err error)
	BatchSendMessage(messages ...MessageSendRequest) (resp BatchMessageSendResponse, err error)
	ReceiveMessage(respChan chan MessageReceiveResponse, errChan chan error, waitseconds ...int64)
	BatchReceiveMessage(respChan chan BatchMessageReceiveResponse, errChan chan error, numOfMessages int32, waitseconds ...int64)
	PeekMessage(respChan chan MessageReceiveResponse, errChan chan error, interval ...time.Duration)
	BatchPeekMessage(respChan chan BatchMessageReceiveResponse, errChan chan error, numOfMessages int32, interval ...time.Duration)
	DeleteMessage(receiptHandle string) (err error)
	BatchDeleteMessage(receiptHandles ...string) (err error)
	ChangeMessageVisibility(receiptHandle string, visibilityTimeout int64) (resp MessageVisibilityChangeResponse, err error)
	Stop()
}

type MNSQueue struct {
	name       string
	client     MNSClient
	stopChan   chan bool
	qpsLimit   int32
	qpsMonitor *QPSMonitor
	decoder    MNSDecoder
}

func NewMNSQueue(name string, client MNSClient, qps ...int32) AliMNSQueue {
	if name == "" {
		panic("ali_mns: queue name could not be empty")
	}

	queue := new(MNSQueue)
	queue.client = client
	queue.name = name
	queue.stopChan = make(chan bool, 1)
	queue.qpsLimit = DefaultQPSLimit
	queue.decoder = NewAliMNSDecoder()

	var attr QueueAttribute
	if _, err := send(client, queue.decoder, GET, nil, nil, "queues/"+name, &attr); err != nil {
		panic(err)
	}

	if qps != nil && len(qps) == 1 && qps[0] > 0 {
		queue.qpsLimit = qps[0]
	}

	proxyURL := ""
	queueProxyEnvKey := PROXY_PREFIX + strings.Replace(strings.ToUpper(name), "-", "_", -1)
	if url := os.Getenv(queueProxyEnvKey); url != "" {
		proxyURL = url
	}

	client.SetProxy(proxyURL)

	queue.qpsMonitor = NewQPSMonitor(5)

	return queue
}

func (p *MNSQueue) Name() string {
	return p.name
}

func (p *MNSQueue) SendMessage(message MessageSendRequest) (resp MessageSendResponse, err error) {
	p.checkQPS()
	_, err = send(p.client, p.decoder, POST, nil, message, fmt.Sprintf("queues/%s/%s", p.name, "messages"), &resp)
	return
}

func (p *MNSQueue) BatchSendMessage(messages ...MessageSendRequest) (resp BatchMessageSendResponse, err error) {
	if messages == nil || len(messages) == 0 {
		return
	}

	batchRequest := BatchMessageSendRequest{}
	for _, message := range messages {
		batchRequest.Messages = append(batchRequest.Messages, message)
	}

	p.checkQPS()
	_, err = send(p.client, p.decoder, POST, nil, batchRequest, fmt.Sprintf("queues/%s/%s", p.name, "messages"), &resp)
	return
}

func (p *MNSQueue) Stop() {
	p.stopChan <- true
}

func (p *MNSQueue) ReceiveMessage(respChan chan MessageReceiveResponse, errChan chan error, waitseconds ...int64) {
	resource := fmt.Sprintf("queues/%s/%s", p.name, "messages")
	if waitseconds != nil && len(waitseconds) == 1 && waitseconds[0] >= 0 {
		resource = fmt.Sprintf("queues/%s/%s?waitseconds=%d", p.name, "messages", waitseconds[0])
	}

	for {
		resp := MessageReceiveResponse{}
		_, err := send(p.client, p.decoder, GET, nil, nil, resource, &resp)
		if err != nil {
			errChan <- err
		} else {
			respChan <- resp
		}

		p.checkQPS()

		select {
		case _ = <-p.stopChan:
			{
				return
			}
		default:
		}
	}

	return
}

func (p *MNSQueue) BatchReceiveMessage(respChan chan BatchMessageReceiveResponse, errChan chan error, numOfMessages int32, waitseconds ...int64) {
	if numOfMessages <= 0 {
		numOfMessages = DefaultNumOfMessages
	}

	resource := fmt.Sprintf("queues/%s/%s?numOfMessages=%d", p.name, "messages", numOfMessages)
	if waitseconds != nil && len(waitseconds) == 1 && waitseconds[0] >= 0 {
		resource = fmt.Sprintf("queues/%s/%s?numOfMessages=%d&waitseconds=%d", p.name, "messages", numOfMessages, waitseconds[0])
	}

	for {
		resp := BatchMessageReceiveResponse{}
		_, err := send(p.client, p.decoder, GET, nil, nil, resource, &resp)
		if err != nil {
			errChan <- err
		} else {
			respChan <- resp
		}

		p.checkQPS()

		select {
		case _ = <-p.stopChan:
			{
				return
			}
		default:
		}
	}

	return
}

func (p *MNSQueue) PeekMessage(respChan chan MessageReceiveResponse, errChan chan error, interval ...time.Duration) {
	resource := fmt.Sprintf("queues/%s/%s?peekonly=true", p.name, "messages")

	itv := time.Duration(0)
	if len(interval) == 1 {
		itv = interval[0]
	}

	for {
		resp := MessageReceiveResponse{}
		_, err := send(p.client, p.decoder, GET, nil, nil, resource, &resp)
		if err != nil {
			errChan <- err
		} else {
			respChan <- resp
		}

		if itv > 0 {
			time.Sleep(itv)
		}

		select {
		case _ = <-p.stopChan:
			{
				return
			}
		default:
		}
	}
	return
}

func (p *MNSQueue) BatchPeekMessage(respChan chan BatchMessageReceiveResponse, errChan chan error, numOfMessages int32, interval ...time.Duration) {
	if numOfMessages <= 0 {
		numOfMessages = DefaultNumOfMessages
	}

	itv := time.Duration(0)
	if len(interval) == 1 {
		itv = interval[0]
	}

	for {
		resp := BatchMessageReceiveResponse{}
		_, err := send(p.client, p.decoder, GET, nil, nil, fmt.Sprintf("queues/%s/%s?numOfMessages=%d&peekonly=true", p.name, "messages", numOfMessages), &resp)
		if err != nil {
			errChan <- err
		} else {
			respChan <- resp
		}

		if itv > 0 {
			time.Sleep(itv)
		}

		select {
		case _ = <-p.stopChan:
			{
				return
			}
		default:
		}
	}
	return
}

func (p *MNSQueue) DeleteMessage(receiptHandle string) (err error) {
	p.checkQPS()
	_, err = send(p.client, p.decoder, DELETE, nil, nil, fmt.Sprintf("queues/%s/%s?ReceiptHandle=%s", p.name, "messages", receiptHandle), nil)
	return
}

func (p *MNSQueue) BatchDeleteMessage(receiptHandles ...string) (err error) {
	if receiptHandles == nil || len(receiptHandles) == 0 {
		return
	}

	handlers := ReceiptHandles{}

	for _, handler := range receiptHandles {
		handlers.ReceiptHandles = append(handlers.ReceiptHandles, handler)
	}

	p.checkQPS()
	_, err = send(p.client, p.decoder, DELETE, nil, handlers, fmt.Sprintf("queues/%s/%s", p.name, "messages"), nil)
	return
}

func (p *MNSQueue) ChangeMessageVisibility(receiptHandle string, visibilityTimeout int64) (resp MessageVisibilityChangeResponse, err error) {
	p.checkQPS()
	_, err = send(p.client, p.decoder, PUT, nil, nil, fmt.Sprintf("queues/%s/%s?ReceiptHandle=%s&VisibilityTimeout=%d", p.name, "messages", receiptHandle, visibilityTimeout), &resp)
	return
}

func (p *MNSQueue) checkQPS() {
	p.qpsMonitor.Pulse()
	if p.qpsLimit > 0 {
		for p.qpsMonitor.QPS() > p.qpsLimit {
			p.qpsMonitor.Pulse()
			time.Sleep(time.Millisecond * 10)
		}
	}
}
<|endoftext|>


<commit_before>package venom

import (
	"github.com/hashicorp/go-multierror"
)

func allNil(errs []error) bool {
	for _, e := range errs {
		if e != nil {
			return false
		}
	}
	return true
}

//
// Same as multierror.Append but takes extra care to not create
// non-nil multierror.Error object with no errors.
//
func AppendErr(err error, errs ...error) error {
	ret := multierror.Append(err, errs...)
	if len(ret.Errors) == 0 || allNil(ret.Errors) {
		return nil
	}
	return ret.ErrorOrNil()
}
<commit_msg>Do reflect to catch nil wrapped in error<commit_after>package venom

import (
	"reflect"

	"github.com/hashicorp/go-multierror"
)

func allNil(errs []error) bool {
	for _, e := range errs {
		if e != nil {
			return false
		}
		if !reflect.ValueOf(e).IsNil() {
			return false
		}
	}
	return true
}

//
// Same as multierror.Append but takes extra care to not create
// non-nil multierror.Error object with no errors.
//
func AppendErr(err error, errs ...error) error {
	ret := multierror.Append(err, errs...)
	if len(ret.Errors) == 0 || allNil(ret.Errors) {
		return nil
	}
	return ret.ErrorOrNil()
}
<|endoftext|>


<commit_before><commit_msg>pkg/whitelist/gen: recount firefox test urls<commit_after><|endoftext|>


<commit_before>package typeinfer

// Deal with Phi nodes.

import (
	"golang.org/x/tools/go/ssa"
)

// phiSelectEdge selects edge based on predecessor block and returns the edge index.
func phiSelectEdge(instr *ssa.Phi, infer *TypeInfer, f *Function, b *Block, l *Loop) int {
	edge := 0
	for i, pred := range instr.Block().Preds {
		if pred.Index == b.Pred {
			e, ok := f.locals[instr.Edges[i]]
			if !ok {
				c, ok := instr.Edges[i].(*ssa.Const)
				if !ok {
					infer.Logger.Fatalf("phi: create instance Edge[%d]=%#v: %s", i, instr.Edges[i], ErrUnknownValue)
				}
				f.locals[instr], edge = &ConstInstance{c}, pred.Index
				infer.Logger.Printf(f.Sprintf(PhiSymbol+"%s/%s = %s, selected const from block %d", instr.Name(), f.locals[instr], instr.String(), edge))
				if a, ok := f.arrays[e]; ok {
					f.arrays[f.locals[instr]] = a
				}
				if s, ok := f.structs[e]; ok {
					f.structs[f.locals[instr]] = s
				}
				return edge
			}
			f.locals[instr], edge = e, pred.Index
			infer.Logger.Printf(f.Sprintf(PhiSymbol+"%s/%s = %s, selected from block %d", instr.Name(), f.locals[instr], instr.String(), edge))
			if a, ok := f.arrays[e]; ok {
				f.arrays[f.locals[instr]] = a
			}
			if s, ok := f.structs[e]; ok {
				f.structs[f.locals[instr]] = s
			}
			return edge
		}
	}
	infer.Logger.Fatalf("phi: %d->%d: %s", b.Pred, instr.Block().Index, ErrPhiUnknownEdge)
	return edge
}
<commit_msg>Handle functions in Phi<commit_after>package typeinfer

// Deal with Phi nodes.

import (
	"golang.org/x/tools/go/ssa"
)

// phiSelectEdge selects edge based on predecessor block and returns the edge index.
func phiSelectEdge(instr *ssa.Phi, infer *TypeInfer, f *Function, b *Block, l *Loop) (edge int) {
	for i, pred := range instr.Block().Preds {
		if pred.Index == b.Pred {
			e, ok := f.locals[instr.Edges[i]]
			if !ok {
				switch t := instr.Edges[i].(type) {
				case *ssa.Const:
					f.locals[instr.Edges[i]] = &ConstInstance{t}
					e, edge = f.locals[instr.Edges[i]], pred.Index
					infer.Logger.Printf(f.Sprintf(PhiSymbol+"%s/%s = %s, selected const from block %d", instr.Name(), e, instr.String(), edge))
				case *ssa.Function:
					f.locals[instr.Edges[i]] = &Instance{instr.Edges[i], f.InstanceID(), l.Index}
					e, edge = f.locals[instr.Edges[i]], pred.Index
					infer.Logger.Printf(f.Sprintf(PhiSymbol+"%s/%s = %s, selected function from block %d", instr.Name(), e, instr.String(), edge))
				default:
					infer.Logger.Fatalf("phi: create instance Edge[%d]=%#v: %s", i, instr.Edges[i], ErrUnknownValue)
					return
				}
			}
			f.locals[instr], edge = e, pred.Index
			infer.Logger.Printf(f.Sprintf(PhiSymbol+"%s/%s = %s, selected from block %d", instr.Name(), e, instr.String(), edge))
			if a, ok := f.arrays[e]; ok {
				f.arrays[f.locals[instr]] = a
			}
			if s, ok := f.structs[e]; ok {
				f.structs[f.locals[instr]] = s
			}
			return
		}
	}
	infer.Logger.Fatalf("phi: %d->%d: %s", b.Pred, instr.Block().Index, ErrPhiUnknownEdge)
	return
}
<|endoftext|>


<commit_before>package google

import (
	"fmt"
	"io"
	"log"
	"net"
	"net/url"
	"strings"

	"google.golang.org/api/googleapi"
	sqladmin "google.golang.org/api/sqladmin/v1beta4"
)

type RetryErrorPredicateFunc func(error) (bool, string)

/** ADD GLOBAL ERROR RETRY PREDICATES HERE **/
// Retry predicates that shoud apply to all requests should be added here.
var defaultErrorRetryPredicates = []RetryErrorPredicateFunc{
	// Common network errors (usually wrapped by URL error)
	isNetworkTemporaryError,
	isNetworkTimeoutError,
	isIoEOFError,
	isConnectionResetNetworkError,

	// Common GCP error codes
	isCommonRetryableErrorCode,

	//While this might apply only to Cloud SQL, historically,
	// we had this in our global default error retries.
	// Keeping it as a default for now.
	is409OperationInProgressError,
}

/** END GLOBAL ERROR RETRY PREDICATES HERE **/

func isNetworkTemporaryError(err error) (bool, string) {
	if netErr, ok := err.(*net.OpError); ok && netErr.Temporary() {
		return true, "marked as timeout"
	}
	if urlerr, ok := err.(*url.Error); ok && urlerr.Temporary() {
		return true, "marked as timeout"
	}
	return false, ""
}

func isNetworkTimeoutError(err error) (bool, string) {
	if netErr, ok := err.(*net.OpError); ok && netErr.Timeout() {
		return true, "marked as timeout"
	}
	if urlerr, ok := err.(*url.Error); ok && urlerr.Timeout() {
		return true, "marked as timeout"
	}
	return false, ""
}

func isIoEOFError(err error) (bool, string) {
	if err == io.ErrUnexpectedEOF {
		return true, "Got unexpected EOF"
	}

	if urlerr, urlok := err.(*url.Error); urlok {
		wrappedErr := urlerr.Unwrap()
		if wrappedErr == io.ErrUnexpectedEOF {
			return true, "Got unexpected EOF"
		}
	}
	return false, ""
}

const connectionResetByPeerErr = ": connection reset by peer"

func isConnectionResetNetworkError(err error) (bool, string) {
	if strings.HasSuffix(err.Error(), connectionResetByPeerErr) {
		return true, fmt.Sprintf("reset connection error: %v", err)
	}
	return false, ""
}

// Retry 409s because some APIs like Cloud SQL throw a 409 if concurrent calls
// are being made.
//
//The only way right now to determine it is a retryable 409 due to
// concurrent calls is to look at the contents of the error message.
// See https://github.com/hashicorp/terraform-provider-google/issues/3279
func is409OperationInProgressError(err error) (bool, string) {
	gerr, ok := err.(*googleapi.Error)
	if !ok {
		return false, ""
	}

	if gerr.Code == 409 && strings.Contains(gerr.Body, "operationInProgress") {
		log.Printf("[DEBUG] Dismissed an error as retryable based on error code 409 and error reason 'operationInProgress': %s", err)
		return true, "Operation still in progress"
	}
	return false, ""
}

// Retry on comon googleapi error codes for retryable errors.
// TODO(#5609): This may not need to be applied globally - figure out
// what retryable error codes apply to which API.
func isCommonRetryableErrorCode(err error) (bool, string) {
	gerr, ok := err.(*googleapi.Error)
	if !ok {
		return false, ""
	}

	if gerr.Code == 429 || gerr.Code == 500 || gerr.Code == 502 || gerr.Code == 503 {
		log.Printf("[DEBUG] Dismissed an error as retryable based on error code: %s", err)
		return true, fmt.Sprintf("Retryable error code %d", gerr.Code)
	}
	return false, ""
}

// We've encountered a few common fingerprint-related strings; if this is one of
// them, we're confident this is an error due to fingerprints.
var FINGERPRINT_FAIL_ERRORS = []string{"Invalid fingerprint.", "Supplied fingerprint does not match current metadata fingerprint."}

// Retry the operation if it looks like a fingerprint mismatch.
func isFingerprintError(err error) (bool, string) {
	gerr, ok := err.(*googleapi.Error)
	if !ok {
		return false, ""
	}

	if gerr.Code != 412 {
		return false, ""
	}

	for _, msg := range FINGERPRINT_FAIL_ERRORS {
		if strings.Contains(err.Error(), msg) {
			return true, "fingerprint mismatch"
		}
	}

	return false, ""
}

// If a permission necessary to provision a resource is created in the same config
// as the resource itself, the permission may not have propagated by the time terraform
// attempts to create the resource. This allows those errors to be retried until the timeout expires
func iamMemberMissing(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 400 && strings.Contains(gerr.Body, "permission") {
			return true, "Waiting for IAM member permissions to propagate."
		}
	}
	return false, ""
}

// Cloud PubSub returns a 400 error if a topic's parent project was recently created and an
// organization policy has not propagated.
// See https://github.com/hashicorp/terraform-provider-google/issues/4349
func pubsubTopicProjectNotReady(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 400 && strings.Contains(gerr.Body, "retry this operation") {
			log.Printf("[DEBUG] Dismissed error as a retryable operation: %s", err)
			return true, "Waiting for Pubsub topic's project to properly initialize with organiation policy"
		}
	}
	return false, ""
}

// Retry if Cloud SQL operation returns a 429 with a specific message for
// concurrent operations.
func isSqlInternalError(err error) (bool, string) {
	if gerr, ok := err.(*SqlAdminOperationError); ok {
		// SqlAdminOperationError is a non-interface type so we need to cast it through
		// a layer of interface{}.  :)
		var ierr interface{}
		ierr = gerr
		if serr, ok := ierr.(*sqladmin.OperationErrors); ok && serr.Errors[0].Code == "INTERNAL_ERROR" {
			return true, "Received an internal error, which is sometimes retryable for some SQL resources.  Optimistically retrying."
		}

	}
	return false, ""
}

// Retry if Cloud SQL operation returns a 429 with a specific message for
// concurrent operations.
func isSqlOperationInProgressError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 409 {
		if strings.Contains(gerr.Body, "you cannot reuse the name of the deleted instance until one week from the deletion date.") {
			return false, ""
		}

		return true, "Waiting for other concurrent Cloud SQL operations to finish"
	}
	return false, ""
}

// Retry if service usage decides you're activating the same service multiple
// times. This can happen if a service and a dependent service aren't batched
// together- eg container.googleapis.com in one request followed by compute.g.c
// in the next (container relies on compute and implicitly activates it)
func serviceUsageServiceBeingActivated(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 400 {
		if strings.Contains(gerr.Body, "Another activation or deactivation is in progress") {
			return true, "Waiting for same service activation/deactivation to finish"
		}

		return false, ""
	}
	return false, ""
}

// Retry if Bigquery operation returns a 403 with a specific message for
// concurrent operations (which are implemented in terms of 'edit quota').
func isBigqueryIAMQuotaError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 403 && strings.Contains(strings.ToLower(gerr.Body), "exceeded rate limits") {
			return true, "Waiting for Bigquery edit quota to refresh"
		}
	}
	return false, ""
}

// Retry if operation returns a 403 with the message for
// exceeding the quota limit for 'OperationReadGroup'
func isOperationReadQuotaError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 403 && strings.Contains(gerr.Body, "Quota exceeded for quota group 'OperationReadGroup'") {
			return true, "Waiting for quota to refresh"
		}
	}
	return false, ""
}

// Retry if Monitoring operation returns a 409 with a specific message for
// concurrent operations.
func isMonitoringConcurrentEditError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 409 && (strings.Contains(strings.ToLower(gerr.Body), "too many concurrent edits") ||
			strings.Contains(strings.ToLower(gerr.Body), "could not fulfill the request")) {
			return true, "Waiting for other Monitoring changes to finish"
		}
	}
	return false, ""
}

// Retry if filestore operation returns a 429 with a specific message for
// concurrent operations.
func isNotFilestoreQuotaError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 429 {
			return false, ""
		}
	}
	return isCommonRetryableErrorCode(err)
}

// Retry if App Engine operation returns a 409 with a specific message for
// concurrent operations, or a 404 indicating p4sa has not yet propagated.
func isAppEngineRetryableError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 409 && strings.Contains(strings.ToLower(gerr.Body), "operation is already in progress") {
			return true, "Waiting for other concurrent App Engine changes to finish"
		}
		if gerr.Code == 404 && strings.Contains(strings.ToLower(gerr.Body), "unable to retrieve p4sa") {
			return true, "Waiting for P4SA propagation to GAIA"
		}
	}
	return false, ""
}

// Retry if KMS CryptoKeyVersions returns a 400 for PENDING_GENERATION
func isCryptoKeyVersionsPendingGeneration(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 400 {
		if strings.Contains(gerr.Body, "PENDING_GENERATION") {
			return true, "Waiting for pending key generation"
		}
	}
	return false, ""
}

// Retry if getting a resource/operation returns a 404 for specific operations.
// opType should describe the operation for which 404 can be retryable.
func isNotFoundRetryableError(opType string) RetryErrorPredicateFunc {
	return func(err error) (bool, string) {
		if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 404 {
			return true, fmt.Sprintf("Retry 404s for %s", opType)
		}
		return false, ""
	}
}

func isDataflowJobUpdateRetryableError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 404 && strings.Contains(gerr.Body, "in RUNNING OR DRAINING state") {
			return true, "Waiting for job to be in a valid state"
		}
	}
	return false, ""
}

func isPeeringOperationInProgress(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 400 && strings.Contains(gerr.Body, "There is a peering operation in progress") {
			return true, "Waiting peering operation to complete"
		}
	}
	return false, ""
}

func isCloudFunctionsSourceCodeError(err error) (bool, string) {
	if operr, ok := err.(*CommonOpError); ok {
		if operr.Code == 3 && operr.Message == "Failed to retrieve function source code" {
			return true, fmt.Sprintf("Retry on Function failing to pull code from GCS")
		}
	}
	return false, ""
}

func datastoreIndex409Contention(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 409 && strings.Contains(gerr.Body, "too much contention") {
			return true, "too much contention - waiting for less activity"
		}
	}
	return false, ""
}

func iapClient409Operation(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 409 && strings.Contains(strings.ToLower(gerr.Body), "operation was aborted") {
			return true, "operation was aborted possibly due to concurrency issue - retrying"
		}
	}
	return false, ""
}

func healthcareDatasetNotInitialized(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 404 && strings.Contains(strings.ToLower(gerr.Body), "dataset not initialized") {
			return true, "dataset not initialized - retrying"
		}
	}
	return false, ""
}

// Cloud Run APIs may return a 409 on create to indicate that a resource has been deleted in the foreground
// (eg GET and LIST) but not the backing apiserver. When we encounter a 409, we can retry it.
// Note that due to limitations in MMv1's error_retry_predicates this is currently applied to all requests.
// We only expect to receive it on create, though.
func isCloudRunCreationConflict(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 409 {
			return true, "saw a 409 - waiting until background deletion completes"
		}
	}

	return false, ""
}

// If a service account is deleted in the middle of updating an IAM policy
// it can cause the API to return an error. In fine-grained IAM resources we
// read the policy, modify it, then send it back to the API. Retrying is
// useful particularly in high-traffic projects.
// We don't want to retry _every_ time we see this error because the
// user-provided SA could trigger this too. At the callsite, we should check
// if the current etag matches the old etag and short-circuit if they do as
// that indicates the new config is the likely problem.
func iamServiceAccountNotFound(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 400 && strings.Contains(gerr.Body, "Service account") && strings.Contains(gerr.Body, "does not exist") {
			return true, "service account not found in IAM"
		}
	}

	return false, ""
}
<commit_msg>string compare modified to include more cases (#4716)<commit_after>package google

import (
	"fmt"
	"io"
	"log"
	"net"
	"net/url"
	"strings"

	"google.golang.org/api/googleapi"
	sqladmin "google.golang.org/api/sqladmin/v1beta4"
)

type RetryErrorPredicateFunc func(error) (bool, string)

/** ADD GLOBAL ERROR RETRY PREDICATES HERE **/
// Retry predicates that shoud apply to all requests should be added here.
var defaultErrorRetryPredicates = []RetryErrorPredicateFunc{
	// Common network errors (usually wrapped by URL error)
	isNetworkTemporaryError,
	isNetworkTimeoutError,
	isIoEOFError,
	isConnectionResetNetworkError,

	// Common GCP error codes
	isCommonRetryableErrorCode,

	//While this might apply only to Cloud SQL, historically,
	// we had this in our global default error retries.
	// Keeping it as a default for now.
	is409OperationInProgressError,
}

/** END GLOBAL ERROR RETRY PREDICATES HERE **/

func isNetworkTemporaryError(err error) (bool, string) {
	if netErr, ok := err.(*net.OpError); ok && netErr.Temporary() {
		return true, "marked as timeout"
	}
	if urlerr, ok := err.(*url.Error); ok && urlerr.Temporary() {
		return true, "marked as timeout"
	}
	return false, ""
}

func isNetworkTimeoutError(err error) (bool, string) {
	if netErr, ok := err.(*net.OpError); ok && netErr.Timeout() {
		return true, "marked as timeout"
	}
	if urlerr, ok := err.(*url.Error); ok && urlerr.Timeout() {
		return true, "marked as timeout"
	}
	return false, ""
}

func isIoEOFError(err error) (bool, string) {
	if err == io.ErrUnexpectedEOF {
		return true, "Got unexpected EOF"
	}

	if urlerr, urlok := err.(*url.Error); urlok {
		wrappedErr := urlerr.Unwrap()
		if wrappedErr == io.ErrUnexpectedEOF {
			return true, "Got unexpected EOF"
		}
	}
	return false, ""
}

const connectionResetByPeerErr = ": connection reset by peer"

func isConnectionResetNetworkError(err error) (bool, string) {
	if strings.HasSuffix(err.Error(), connectionResetByPeerErr) {
		return true, fmt.Sprintf("reset connection error: %v", err)
	}
	return false, ""
}

// Retry 409s because some APIs like Cloud SQL throw a 409 if concurrent calls
// are being made.
//
//The only way right now to determine it is a retryable 409 due to
// concurrent calls is to look at the contents of the error message.
// See https://github.com/hashicorp/terraform-provider-google/issues/3279
func is409OperationInProgressError(err error) (bool, string) {
	gerr, ok := err.(*googleapi.Error)
	if !ok {
		return false, ""
	}

	if gerr.Code == 409 && strings.Contains(gerr.Body, "operationInProgress") {
		log.Printf("[DEBUG] Dismissed an error as retryable based on error code 409 and error reason 'operationInProgress': %s", err)
		return true, "Operation still in progress"
	}
	return false, ""
}

// Retry on comon googleapi error codes for retryable errors.
// TODO(#5609): This may not need to be applied globally - figure out
// what retryable error codes apply to which API.
func isCommonRetryableErrorCode(err error) (bool, string) {
	gerr, ok := err.(*googleapi.Error)
	if !ok {
		return false, ""
	}

	if gerr.Code == 429 || gerr.Code == 500 || gerr.Code == 502 || gerr.Code == 503 {
		log.Printf("[DEBUG] Dismissed an error as retryable based on error code: %s", err)
		return true, fmt.Sprintf("Retryable error code %d", gerr.Code)
	}
	return false, ""
}

// We've encountered a few common fingerprint-related strings; if this is one of
// them, we're confident this is an error due to fingerprints.
var FINGERPRINT_FAIL_ERRORS = []string{"Invalid fingerprint.", "Supplied fingerprint does not match current metadata fingerprint."}

// Retry the operation if it looks like a fingerprint mismatch.
func isFingerprintError(err error) (bool, string) {
	gerr, ok := err.(*googleapi.Error)
	if !ok {
		return false, ""
	}

	if gerr.Code != 412 {
		return false, ""
	}

	for _, msg := range FINGERPRINT_FAIL_ERRORS {
		if strings.Contains(err.Error(), msg) {
			return true, "fingerprint mismatch"
		}
	}

	return false, ""
}

// If a permission necessary to provision a resource is created in the same config
// as the resource itself, the permission may not have propagated by the time terraform
// attempts to create the resource. This allows those errors to be retried until the timeout expires
func iamMemberMissing(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 400 && strings.Contains(gerr.Body, "permission") {
			return true, "Waiting for IAM member permissions to propagate."
		}
	}
	return false, ""
}

// Cloud PubSub returns a 400 error if a topic's parent project was recently created and an
// organization policy has not propagated.
// See https://github.com/hashicorp/terraform-provider-google/issues/4349
func pubsubTopicProjectNotReady(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 400 && strings.Contains(gerr.Body, "retry this operation") {
			log.Printf("[DEBUG] Dismissed error as a retryable operation: %s", err)
			return true, "Waiting for Pubsub topic's project to properly initialize with organiation policy"
		}
	}
	return false, ""
}

// Retry if Cloud SQL operation returns a 429 with a specific message for
// concurrent operations.
func isSqlInternalError(err error) (bool, string) {
	if gerr, ok := err.(*SqlAdminOperationError); ok {
		// SqlAdminOperationError is a non-interface type so we need to cast it through
		// a layer of interface{}.  :)
		var ierr interface{}
		ierr = gerr
		if serr, ok := ierr.(*sqladmin.OperationErrors); ok && serr.Errors[0].Code == "INTERNAL_ERROR" {
			return true, "Received an internal error, which is sometimes retryable for some SQL resources.  Optimistically retrying."
		}

	}
	return false, ""
}

// Retry if Cloud SQL operation returns a 429 with a specific message for
// concurrent operations.
func isSqlOperationInProgressError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 409 {
		if strings.Contains(gerr.Body, "you cannot reuse the name of the deleted instance until one week from the deletion date.") {
			return false, ""
		}

		return true, "Waiting for other concurrent Cloud SQL operations to finish"
	}
	return false, ""
}

// Retry if service usage decides you're activating the same service multiple
// times. This can happen if a service and a dependent service aren't batched
// together- eg container.googleapis.com in one request followed by compute.g.c
// in the next (container relies on compute and implicitly activates it)
func serviceUsageServiceBeingActivated(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 400 {
		if strings.Contains(gerr.Body, "Another activation or deactivation is in progress") {
			return true, "Waiting for same service activation/deactivation to finish"
		}

		return false, ""
	}
	return false, ""
}

// Retry if Bigquery operation returns a 403 with a specific message for
// concurrent operations (which are implemented in terms of 'edit quota').
func isBigqueryIAMQuotaError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 403 && strings.Contains(strings.ToLower(gerr.Body), "exceeded rate limits") {
			return true, "Waiting for Bigquery edit quota to refresh"
		}
	}
	return false, ""
}

// Retry if operation returns a 403 with the message for
// exceeding the quota limit for 'OperationReadGroup'
func isOperationReadQuotaError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 403 && strings.Contains(gerr.Body, "Quota exceeded for quota group") {
			return true, "Waiting for quota to refresh"
		}
	}
	return false, ""
}

// Retry if Monitoring operation returns a 409 with a specific message for
// concurrent operations.
func isMonitoringConcurrentEditError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 409 && (strings.Contains(strings.ToLower(gerr.Body), "too many concurrent edits") ||
			strings.Contains(strings.ToLower(gerr.Body), "could not fulfill the request")) {
			return true, "Waiting for other Monitoring changes to finish"
		}
	}
	return false, ""
}

// Retry if filestore operation returns a 429 with a specific message for
// concurrent operations.
func isNotFilestoreQuotaError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 429 {
			return false, ""
		}
	}
	return isCommonRetryableErrorCode(err)
}

// Retry if App Engine operation returns a 409 with a specific message for
// concurrent operations, or a 404 indicating p4sa has not yet propagated.
func isAppEngineRetryableError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 409 && strings.Contains(strings.ToLower(gerr.Body), "operation is already in progress") {
			return true, "Waiting for other concurrent App Engine changes to finish"
		}
		if gerr.Code == 404 && strings.Contains(strings.ToLower(gerr.Body), "unable to retrieve p4sa") {
			return true, "Waiting for P4SA propagation to GAIA"
		}
	}
	return false, ""
}

// Retry if KMS CryptoKeyVersions returns a 400 for PENDING_GENERATION
func isCryptoKeyVersionsPendingGeneration(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 400 {
		if strings.Contains(gerr.Body, "PENDING_GENERATION") {
			return true, "Waiting for pending key generation"
		}
	}
	return false, ""
}

// Retry if getting a resource/operation returns a 404 for specific operations.
// opType should describe the operation for which 404 can be retryable.
func isNotFoundRetryableError(opType string) RetryErrorPredicateFunc {
	return func(err error) (bool, string) {
		if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 404 {
			return true, fmt.Sprintf("Retry 404s for %s", opType)
		}
		return false, ""
	}
}

func isDataflowJobUpdateRetryableError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 404 && strings.Contains(gerr.Body, "in RUNNING OR DRAINING state") {
			return true, "Waiting for job to be in a valid state"
		}
	}
	return false, ""
}

func isPeeringOperationInProgress(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 400 && strings.Contains(gerr.Body, "There is a peering operation in progress") {
			return true, "Waiting peering operation to complete"
		}
	}
	return false, ""
}

func isCloudFunctionsSourceCodeError(err error) (bool, string) {
	if operr, ok := err.(*CommonOpError); ok {
		if operr.Code == 3 && operr.Message == "Failed to retrieve function source code" {
			return true, fmt.Sprintf("Retry on Function failing to pull code from GCS")
		}
	}
	return false, ""
}

func datastoreIndex409Contention(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 409 && strings.Contains(gerr.Body, "too much contention") {
			return true, "too much contention - waiting for less activity"
		}
	}
	return false, ""
}

func iapClient409Operation(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 409 && strings.Contains(strings.ToLower(gerr.Body), "operation was aborted") {
			return true, "operation was aborted possibly due to concurrency issue - retrying"
		}
	}
	return false, ""
}

func healthcareDatasetNotInitialized(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 404 && strings.Contains(strings.ToLower(gerr.Body), "dataset not initialized") {
			return true, "dataset not initialized - retrying"
		}
	}
	return false, ""
}

// Cloud Run APIs may return a 409 on create to indicate that a resource has been deleted in the foreground
// (eg GET and LIST) but not the backing apiserver. When we encounter a 409, we can retry it.
// Note that due to limitations in MMv1's error_retry_predicates this is currently applied to all requests.
// We only expect to receive it on create, though.
func isCloudRunCreationConflict(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 409 {
			return true, "saw a 409 - waiting until background deletion completes"
		}
	}

	return false, ""
}

// If a service account is deleted in the middle of updating an IAM policy
// it can cause the API to return an error. In fine-grained IAM resources we
// read the policy, modify it, then send it back to the API. Retrying is
// useful particularly in high-traffic projects.
// We don't want to retry _every_ time we see this error because the
// user-provided SA could trigger this too. At the callsite, we should check
// if the current etag matches the old etag and short-circuit if they do as
// that indicates the new config is the likely problem.
func iamServiceAccountNotFound(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 400 && strings.Contains(gerr.Body, "Service account") && strings.Contains(gerr.Body, "does not exist") {
			return true, "service account not found in IAM"
		}
	}

	return false, ""
}
<|endoftext|>


<commit_before>// Event package is responsible for receiving events from 9volt components and
// dumping them to etcd. The event queue is powered by WORKER_COUNT workers and
// has a BUFFER_LEN buffer.
//
// Unlike other components, the event queue does NOT get shutdown in the event
// of a backend failure - it gets *paused*. In the *pause* state, the event queue
// will simply discard any inbound messages and will NOT attempt to save them to
// the backend.
//
// This is done in order to avoid potential race conditions where lagging/slower
// components continue to write to the event queue even though they've been asked
// to shutdown.
package event

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/9corp/9volt/base"
	"github.com/9corp/9volt/dal"
	"github.com/9corp/9volt/util"
)

const (
	BUFFER_LEN    = 1000
	WORKER_COUNT  = 20
	WORKER_SLEEP  = time.Duration(500) * time.Millisecond
	MAX_EVENT_AGE = 86400 // 24 hours
)

type Queue struct {
	Log       log.FieldLogger
	DalClient dal.IDal
	MemberID  string
	channel   chan *Event
	Running   bool // do not allow more than one Start() to be issued; resets Pause
	Pause     bool // Pause set via Stop(); causes messages to be discarded

	base.Component
}

//go:generate counterfeiter -o ../fakes/eventfakes/fake_client.go event.go IClient

type IClient interface {
	Add(string, string) error
	AddWithErrorLog(string, string, log.FieldLogger, log.Fields) error
}

type Client struct {
	queue *Queue
}

type Event struct {
	Type      string    `json:"type"`
	Message   string    `json:"message"`
	Timestamp time.Time `json:"timestamp"`
	MemberId  string    `json:"memberid"`
}

func NewQueue(memberID string, dalClient dal.IDal) *Queue {
	return &Queue{
		Log:       log.WithFields(log.Fields{"pkg": "event"}),
		DalClient: dalClient,
		MemberID:  memberID,
		channel:   make(chan *Event, BUFFER_LEN),
		Component: base.Component{
			Identifier: "event",
		},
	}
}

func (q *Queue) Start() error {
	q.Pause = false

	if q.Running {
		q.Log.Debug("Already running - nothing to do")
		return nil
	}

	q.Running = true

	// launch worker pool
	q.Log.Debugf("Launching %v queue workers", WORKER_COUNT)

	for i := 1; i <= WORKER_COUNT; i++ {
		go q.runWorker(i)
	}

	return nil
}

func (q *Queue) Stop() error {
	q.Log.Debug("Workers are paused")

	q.Pause = true

	return nil
}

func (q *Queue) runWorker(id int) {
	llog := q.Log.WithFields(log.Fields{"id": id})

	llog.Debug("Event worker started")

	for {
		e := <-q.channel

		if q.Pause {
			llog.Debug("In pause state; dropping inbound message(s)")
			continue
		}

		llog.WithFields(log.Fields{"type": e.Type, "msg": e.Message}).Debug("Worker received new event")

		// marshal the event
		eventBlob, err := json.Marshal(e)
		if err != nil {
			llog.WithFields(log.Fields{"event": e, "err": err}).Errorf("Unable to marshal event")
			continue
		}

		// write it to etcd
		fullKey := fmt.Sprintf("event/%v-%v", e.Type, util.RandomString(6, true))

		if err := q.DalClient.Set(fullKey, string(eventBlob), &dal.SetOptions{Dir: false, TTLSec: MAX_EVENT_AGE, PrevExist: ""}); err != nil {
			llog.WithFields(log.Fields{"event": e, "path": fullKey, "err": err}).Error("Unable to save event blob")
		}

		// Artificially slow down queue workers (and prevent an etcd write flood)
		time.Sleep(WORKER_SLEEP)
	}
}

func (q *Queue) NewClient() *Client {
	return &Client{
		queue: q,
	}
}

// Attempt to send an event to the event channel; discard event if event channel is full
func (c *Client) Add(key, value string) error {
	key = strings.ToLower(key)

	select {
	case c.queue.channel <- &Event{
		Type:      key,
		Message:   value,
		MemberId:  c.queue.MemberID,
		Timestamp: time.Now(),
	}:
		return nil
	default:
		return fmt.Errorf("Event queue is full; discarding event (Type: %v Message: %v)",
			key, value)
	}
}

func (c *Client) AddWithErrorLog(key, value string, logger log.FieldLogger, fields log.Fields) error {
	logger.WithFields(fields).Error(value)

	eventMessage := value + " ["
	for k, v := range fields {
		eventMessage = fmt.Sprintf("%v %v=%v", eventMessage, k, v)
	}
	eventMessage = eventMessage + "]"

	if err := c.Add(key, eventMessage); err != nil {
		return err
	}

	return nil
}
<commit_msg>prettied up the event logger<commit_after>// Event package is responsible for receiving events from 9volt components and
// dumping them to etcd. The event queue is powered by WORKER_COUNT workers and
// has a BUFFER_LEN buffer.
//
// Unlike other components, the event queue does NOT get shutdown in the event
// of a backend failure - it gets *paused*. In the *pause* state, the event queue
// will simply discard any inbound messages and will NOT attempt to save them to
// the backend.
//
// This is done in order to avoid potential race conditions where lagging/slower
// components continue to write to the event queue even though they've been asked
// to shutdown.
package event

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/9corp/9volt/base"
	"github.com/9corp/9volt/dal"
	"github.com/9corp/9volt/util"
)

const (
	BUFFER_LEN    = 1000
	WORKER_COUNT  = 20
	WORKER_SLEEP  = time.Duration(500) * time.Millisecond
	MAX_EVENT_AGE = 86400 // 24 hours
)

type Queue struct {
	Log       log.FieldLogger
	DalClient dal.IDal
	MemberID  string
	channel   chan *Event
	Running   bool // do not allow more than one Start() to be issued; resets Pause
	Pause     bool // Pause set via Stop(); causes messages to be discarded

	base.Component
}

//go:generate counterfeiter -o ../fakes/eventfakes/fake_client.go event.go IClient

type IClient interface {
	Add(string, string) error
	AddWithErrorLog(string, string, log.FieldLogger, log.Fields) error
}

type Client struct {
	queue *Queue
}

type Event struct {
	Type      string    `json:"type"`
	Message   string    `json:"message"`
	Timestamp time.Time `json:"timestamp"`
	MemberId  string    `json:"memberid"`
}

func NewQueue(memberID string, dalClient dal.IDal) *Queue {
	return &Queue{
		Log:       log.WithFields(log.Fields{"pkg": "event"}),
		DalClient: dalClient,
		MemberID:  memberID,
		channel:   make(chan *Event, BUFFER_LEN),
		Component: base.Component{
			Identifier: "event",
		},
	}
}

func (q *Queue) Start() error {
	q.Pause = false

	if q.Running {
		q.Log.Debug("Already running - nothing to do")
		return nil
	}

	q.Running = true

	// launch worker pool
	q.Log.Debugf("Launching %v queue workers", WORKER_COUNT)

	for i := 1; i <= WORKER_COUNT; i++ {
		go q.runWorker(i)
	}

	return nil
}

func (q *Queue) Stop() error {
	q.Log.Debug("Workers are paused")

	q.Pause = true

	return nil
}

func (q *Queue) runWorker(id int) {
	llog := q.Log.WithFields(log.Fields{"id": id})

	llog.Debug("Event worker started")

	for {
		e := <-q.channel

		if q.Pause {
			llog.Debug("In pause state; dropping inbound message(s)")
			continue
		}

		llog.WithFields(log.Fields{"type": e.Type, "msg": e.Message}).Debug("Worker received new event")

		// marshal the event
		eventBlob, err := json.Marshal(e)
		if err != nil {
			llog.WithFields(log.Fields{"event": e, "err": err}).Errorf("Unable to marshal event")
			continue
		}

		// write it to etcd
		fullKey := fmt.Sprintf("event/%v-%v", e.Type, util.RandomString(6, true))

		if err := q.DalClient.Set(fullKey, string(eventBlob), &dal.SetOptions{Dir: false, TTLSec: MAX_EVENT_AGE, PrevExist: ""}); err != nil {
			llog.WithFields(log.Fields{"event": e, "path": fullKey, "err": err}).Error("Unable to save event blob")
		}

		// Artificially slow down queue workers (and prevent an etcd write flood)
		time.Sleep(WORKER_SLEEP)
	}
}

func (q *Queue) NewClient() *Client {
	return &Client{
		queue: q,
	}
}

// Attempt to send an event to the event channel; discard event if event channel is full
func (c *Client) Add(key, value string) error {
	key = strings.ToLower(key)

	select {
	case c.queue.channel <- &Event{
		Type:      key,
		Message:   value,
		MemberId:  c.queue.MemberID,
		Timestamp: time.Now(),
	}:
		return nil
	default:
		return fmt.Errorf("Event queue is full; discarding event (Type: %v Message: %v)",
			key, value)
	}
}

func (c *Client) AddWithErrorLog(key, value string, logger log.FieldLogger, fields log.Fields) error {
	logger.WithFields(fields).Error(value)

	fieldEntries := make([]string, 0)

	for k, v := range fields {
		fieldEntries = append(fieldEntries, fmt.Sprintf("%v=%v", k, v))
	}

	eventMessage := value + " [" + strings.Join(fieldEntries, " ") + "]"

	if err := c.Add(key, eventMessage); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package sse

import (
	"errors"
	"io"
	"net/http"
	"sync"
	"time"
)

type ReadyState byte

const (
	AllowedContentType = "text/event-stream"

	// Connecting while trying to establish connection with the stream.
	Connecting ReadyState = iota - 1
	// Open after connection is established with the server.
	Open
	// Closing after Close is invoked.
	Closing
	// Closed after the connection is closed.
	Closed

	defaultRetry = time.Duration(1000)
)

var (
	ErrContentType = errors.New("eventsource: the content type of the stream is not allowed")

	// Map used by decoders to be able to change the retry time of the eventSource when
	// a retry event is received.
	globalDecoderMap = map[Decoder]*eventSource{}
	retryMux         = sync.RWMutex{}
)

type (
	// EventSource connects and processes events from an SSE stream.
	EventSource interface {
		URL() (url string)
		ReadyState() (state ReadyState)
		LastEventID() (id string)
		Events() (events <-chan Event)
		Close()
	}
	eventSource struct {
		url          string
		d            Decoder
		resp         *http.Response
		out          chan Event
		closeOutOnce chan bool

		// Last recorded event ID
		lastEventID    string
		lastEventIDMux sync.RWMutex

		// Status of the event stream.
		readyState    ReadyState
		readyStateMux sync.RWMutex

		// Reconnection waiting time in milliseconds
		retry time.Duration
	}
)

// NewEventSource constructs returns an EventSource that satisfies the HTML5 EventSource specification.
func NewEventSource(url string) (EventSource, error) {
	es := eventSource{
		d:            nil,
		url:          url,
		out:          make(chan Event),
		closeOutOnce: make(chan bool),
		retry:        defaultRetry,
	}

	// Ensure the output channel is closed only once.
	go es.closeOnce()

	return &es, es.connect()
}

// connect does a connection attempt, if the operation fails, attempt reconnecting
// according to the spec.
func (es *eventSource) connect() (err error) {
	es.setReadyState(Connecting)
	err = es.connectOnce()
	if err != nil {
		err = es.reconnect()
	}
	return
}

// reconnect to the stream several until the operation succeeds or the conditions
// to retry no longer hold true.
func (es *eventSource) reconnect() (err error) {
	es.setReadyState(Connecting)
	for es.mustReconnect(err) {
		time.Sleep(es.retry * time.Millisecond)
		err = es.connectOnce()
	}
	if err != nil {
		es.Close()
	}
	return
}

// Attempts to connect and updates internal status depending on the outcome.
func (es *eventSource) connectOnce() (err error) {
	es.resp, err = es.httpConnect()
	if err != nil {
		return
	}
	delEventSource(es.d)
	es.d = NewDecoder(es.resp.Body)
	setEventSource(es.d, es)
	es.setReadyState(Open)
	go es.consume()
	return
}

func (es *eventSource) httpConnect() (*http.Response, error) {
	// Prepare request
	req, err := http.NewRequest("GET", es.url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", AllowedContentType)

	// Check response
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		es.resp = nil
		return nil, err
	}
	if resp.Header.Get("Content-Type") != AllowedContentType {
		return nil, ErrContentType
	}
	return resp, nil
}

// Method consume() must be called once connect() succeeds.
// It parses the input reader and assigns the event output channel accordingly.
func (es *eventSource) consume() {
	for {
		ev, err := es.d.Decode()
		if err != nil {
			if es.mustReconnect(err) {
				err = es.reconnect()
				return
			}
			es.Close()
			return
		}
		id := ev.ID()
		if id != "" {
			es.setLastEventID(id)
		}
		es.out <- ev
	}
}

// Clients will reconnect if the connection is closed;
// a client can be told to stop reconnecting using the HTTP 204 No Content response code.
func (es *eventSource) mustReconnect(err error) bool {
	switch err {
	case ErrContentType:
		return false
	case io.ErrUnexpectedEOF:
		return true
	}
	if es.resp != nil && es.resp.StatusCode == http.StatusNoContent {
		return false
	}
	return true
}

// Returns the event source URL.
func (es *eventSource) URL() string {
	return es.url
}

// Returns the event source connection state, either connecting, open or closed.
func (es *eventSource) ReadyState() ReadyState {
	es.readyStateMux.RLock()
	defer es.readyStateMux.RUnlock()
	return es.readyState
}

func (es *eventSource) setReadyState(newState ReadyState) {
	es.readyStateMux.Lock()
	defer es.readyStateMux.Unlock()
	es.readyState = newState
}

// Returns the last event source Event id.
func (es *eventSource) LastEventID() string {
	es.lastEventIDMux.RLock()
	defer es.lastEventIDMux.RUnlock()
	return es.lastEventID
}

func (es *eventSource) setLastEventID(id string) {
	es.lastEventIDMux.Lock()
	defer es.lastEventIDMux.Unlock()
	es.lastEventID = id
}

// Returns the channel of events. Events will be queued in the channel as they
// are received.
func (es *eventSource) Events() <-chan Event {
	return es.out
}

// Closes the event source.
// After closing the event source, it cannot be reused again.
func (es *eventSource) Close() {
	state := es.ReadyState()
	if state == Closed || state == Closing {
		return
	}
	es.setReadyState(Closing)
	if es.resp != nil {
		es.resp.Body.Close()
	}
	es.sendClose()
	delEventSource(es.d)
	es.setReadyState(Closed)
}

func (es *eventSource) sendClose() {
	select {
	case es.closeOutOnce <- true:
	default:
	}
}

func (es *eventSource) closeOnce() {
	select {
	case <-es.closeOutOnce:
		close(es.out)
	}
}

func setEventSource(d Decoder, es *eventSource) {
	retryMux.Lock()
	defer retryMux.Unlock()
	globalDecoderMap[d] = es
}

func getEventSource(d Decoder) (es *eventSource, ok bool) {
	retryMux.RLock()
	defer retryMux.RUnlock()
	es, ok = globalDecoderMap[d]
	return
}

func delEventSource(d Decoder) {
	retryMux.Lock()
	defer retryMux.Unlock()
	delete(globalDecoderMap, d)
}
<commit_msg>eventsource: set cache control to 'no-store'. (#21)<commit_after>package sse

import (
	"errors"
	"io"
	"net/http"
	"sync"
	"time"
)

type ReadyState byte

const (
	AllowedContentType = "text/event-stream"

	// Connecting while trying to establish connection with the stream.
	Connecting ReadyState = iota - 1
	// Open after connection is established with the server.
	Open
	// Closing after Close is invoked.
	Closing
	// Closed after the connection is closed.
	Closed

	defaultRetry = time.Duration(1000)
)

var (
	ErrContentType = errors.New("eventsource: the content type of the stream is not allowed")

	// Map used by decoders to be able to change the retry time of the eventSource when
	// a retry event is received.
	globalDecoderMap = map[Decoder]*eventSource{}
	retryMux         = sync.RWMutex{}
)

type (
	// EventSource connects and processes events from an SSE stream.
	EventSource interface {
		URL() (url string)
		ReadyState() (state ReadyState)
		LastEventID() (id string)
		Events() (events <-chan Event)
		Close()
	}
	eventSource struct {
		url          string
		d            Decoder
		resp         *http.Response
		out          chan Event
		closeOutOnce chan bool

		// Last recorded event ID
		lastEventID    string
		lastEventIDMux sync.RWMutex

		// Status of the event stream.
		readyState    ReadyState
		readyStateMux sync.RWMutex

		// Reconnection waiting time in milliseconds
		retry time.Duration
	}
)

// NewEventSource constructs returns an EventSource that satisfies the HTML5 EventSource specification.
func NewEventSource(url string) (EventSource, error) {
	es := eventSource{
		d:            nil,
		url:          url,
		out:          make(chan Event),
		closeOutOnce: make(chan bool),
		retry:        defaultRetry,
	}

	// Ensure the output channel is closed only once.
	go es.closeOnce()

	return &es, es.connect()
}

// connect does a connection attempt, if the operation fails, attempt reconnecting
// according to the spec.
func (es *eventSource) connect() (err error) {
	es.setReadyState(Connecting)
	err = es.connectOnce()
	if err != nil {
		err = es.reconnect()
	}
	return
}

// reconnect to the stream several until the operation succeeds or the conditions
// to retry no longer hold true.
func (es *eventSource) reconnect() (err error) {
	es.setReadyState(Connecting)
	for es.mustReconnect(err) {
		time.Sleep(es.retry * time.Millisecond)
		err = es.connectOnce()
	}
	if err != nil {
		es.Close()
	}
	return
}

// Attempts to connect and updates internal status depending on the outcome.
func (es *eventSource) connectOnce() (err error) {
	es.resp, err = es.httpConnect()
	if err != nil {
		return
	}
	delEventSource(es.d)
	es.d = NewDecoder(es.resp.Body)
	setEventSource(es.d, es)
	es.setReadyState(Open)
	go es.consume()
	return
}

func (es *eventSource) httpConnect() (*http.Response, error) {
	// Prepare request
	req, err := http.NewRequest("GET", es.url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", AllowedContentType)
	req.Header.Set("Cache-Control", "no-store")

	// Check response
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		es.resp = nil
		return nil, err
	}
	if resp.Header.Get("Content-Type") != AllowedContentType {
		return nil, ErrContentType
	}
	return resp, nil
}

// Method consume() must be called once connect() succeeds.
// It parses the input reader and assigns the event output channel accordingly.
func (es *eventSource) consume() {
	for {
		ev, err := es.d.Decode()
		if err != nil {
			if es.mustReconnect(err) {
				err = es.reconnect()
				return
			}
			es.Close()
			return
		}
		id := ev.ID()
		if id != "" {
			es.setLastEventID(id)
		}
		es.out <- ev
	}
}

// Clients will reconnect if the connection is closed;
// a client can be told to stop reconnecting using the HTTP 204 No Content response code.
func (es *eventSource) mustReconnect(err error) bool {
	switch err {
	case ErrContentType:
		return false
	case io.ErrUnexpectedEOF:
		return true
	}
	if es.resp != nil && es.resp.StatusCode == http.StatusNoContent {
		return false
	}
	return true
}

// Returns the event source URL.
func (es *eventSource) URL() string {
	return es.url
}

// Returns the event source connection state, either connecting, open or closed.
func (es *eventSource) ReadyState() ReadyState {
	es.readyStateMux.RLock()
	defer es.readyStateMux.RUnlock()
	return es.readyState
}

func (es *eventSource) setReadyState(newState ReadyState) {
	es.readyStateMux.Lock()
	defer es.readyStateMux.Unlock()
	es.readyState = newState
}

// Returns the last event source Event id.
func (es *eventSource) LastEventID() string {
	es.lastEventIDMux.RLock()
	defer es.lastEventIDMux.RUnlock()
	return es.lastEventID
}

func (es *eventSource) setLastEventID(id string) {
	es.lastEventIDMux.Lock()
	defer es.lastEventIDMux.Unlock()
	es.lastEventID = id
}

// Returns the channel of events. Events will be queued in the channel as they
// are received.
func (es *eventSource) Events() <-chan Event {
	return es.out
}

// Closes the event source.
// After closing the event source, it cannot be reused again.
func (es *eventSource) Close() {
	state := es.ReadyState()
	if state == Closed || state == Closing {
		return
	}
	es.setReadyState(Closing)
	if es.resp != nil {
		es.resp.Body.Close()
	}
	es.sendClose()
	delEventSource(es.d)
	es.setReadyState(Closed)
}

func (es *eventSource) sendClose() {
	select {
	case es.closeOutOnce <- true:
	default:
	}
}

func (es *eventSource) closeOnce() {
	select {
	case <-es.closeOutOnce:
		close(es.out)
	}
}

func setEventSource(d Decoder, es *eventSource) {
	retryMux.Lock()
	defer retryMux.Unlock()
	globalDecoderMap[d] = es
}

func getEventSource(d Decoder) (es *eventSource, ok bool) {
	retryMux.RLock()
	defer retryMux.RUnlock()
	es, ok = globalDecoderMap[d]
	return
}

func delEventSource(d Decoder) {
	retryMux.Lock()
	defer retryMux.Unlock()
	delete(globalDecoderMap, d)
}
<|endoftext|>