<commit_before>package dequeuer

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	log "github.com/inconshreveable/log15"
	"github.com/kevinburke/rickover/dequeuer"
	"github.com/kevinburke/rickover/newmodels"
	"github.com/kevinburke/rickover/test"
	"github.com/kevinburke/rickover/test/factory"
	"github.com/kevinburke/semaphore"
)

func TestAll(t *testing.T) {
	test.SetUp(t)
	t.Cleanup(func() { test.TearDown(t) })
	t.Run("Parallel", func(t *testing.T) {
		t.Run("TestWorkerShutsDown", testWorkerShutsDown)
		t.Run("TestWorkerShutsDownWithContext", testWorkerShutsDownWithContext)
		t.Run("TestWorkerMakesCorrectRequest", testWorkerMakesCorrectRequest)
		t.Run("TestWorkerMakesExactlyOneRequest", testWorkerMakesExactlyOneRequest)
	})
}

func testWorkerShutsDown(t *testing.T) {
	t.Parallel()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	poolname := factory.RandomId("pool")
	pool := dequeuer.NewPool(ctx, poolname.String())
	for i := 0; i < 3; i++ {
		pool.AddDequeuer(ctx, factory.Processor("http://example.com"))
	}
	c1 := make(chan struct{}, 1)
	go func() {
		err := pool.Shutdown(ctx)
		test.AssertNotError(t, err, "")
		c1 <- struct{}{}
	}()
	for {
		select {
		case <-c1:
			return
		case <-time.After(300 * time.Millisecond):
			t.Fatalf("pool did not shut down in 300ms")
		}
	}
}

func testWorkerShutsDownWithContext(t *testing.T) {
	t.Parallel()
	ctx, cancel := context.WithCancel(context.Background())
	poolname := factory.RandomId("pool")
	pool := dequeuer.NewPool(ctx, poolname.String())
	for i := 0; i < 3; i++ {
		pool.AddDequeuer(ctx, factory.Processor("http://example.com"))
	}
	c1 := make(chan struct{}, 1)
	go func() {
		for {
			if pool.Len() == 0 {
				c1 <- struct{}{}
				break
			}
			time.Sleep(2 * time.Millisecond)
		}
	}()
	cancel()
	for {
		select {
		case <-c1:
			return
		case <-time.After(300 * time.Millisecond):
			t.Fatalf("pool did not shut down in 300ms")
		}
	}
}

// 1. Create a job type
// 2. Enqueue a job
// 3. Create a test server that replies with a 202
// 4. Ensure that the correct request is made to the server
func testWorkerMakesCorrectRequest(t *testing.T) {
	qj := factory.CreateQJ(t)

	c1 := make(chan bool, 1)
	var path, method, user string
	var ok bool
	var workRequest struct {
		Data     *factory.RandomData `json:"data"`
		Attempts int16               `json:"attempts"`
	}
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		path = r.URL.Path
		method = r.Method
		user, _, ok = r.BasicAuth()
		err := json.NewDecoder(r.Body).Decode(&workRequest)
		test.AssertNotError(t, err, "decoding request body")
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		w.WriteHeader(http.StatusAccepted)
		w.Write([]byte("{}"))
		c1 <- true
		close(c1)
	}))
	defer s.Close()
	jp := factory.Processor(s.URL)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	pool := dequeuer.NewPool(ctx, qj.Name)
	pool.AddDequeuer(ctx, jp)
	defer pool.Shutdown(context.Background())
	select {
	case <-c1:
		test.AssertEquals(t, path, fmt.Sprintf("/v1/jobs/%s/%s", qj.Name, qj.ID.String()))
		test.AssertEquals(t, method, "POST")
		test.AssertEquals(t, ok, true)
		test.AssertEquals(t, user, "jobs")
		test.AssertDeepEquals(t, workRequest.Data, factory.RD)
		test.AssertEquals(t, workRequest.Attempts, qj.Attempts)
		return
	case <-time.After(200 * time.Millisecond):
		t.Fatalf("Server did not receive a request in 200ms, quitting")
	}
}

// 1. Create a job type
// 2. Enqueue a job
// 2a. Create twenty worker nodes
// 3. Create a test server that replies with a 202
// 4. Ensure that only one request is made to the server
func testWorkerMakesExactlyOneRequest(t *testing.T) {
	qj := factory.CreateQJ(t)

	c1 := int32(0)
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		w.WriteHeader(http.StatusAccepted)
		w.Write([]byte("{}"))
		atomic.AddInt32(&c1, 1)
	}))
	defer s.Close()
	ctx, cancel := context.WithCancel(context.Background())
	pool := dequeuer.NewPool(ctx, qj.Name)
	for i := 0; i < 20; i++ {
		jp := factory.Processor(s.URL)
		pool.AddDequeuer(ctx, jp)
	}
	defer cancel()
	<-time.After(100 * time.Millisecond)
	test.Assert(t, atomic.LoadInt32(&c1) == int32(1), "request called once")
}

func TestCreatePools(t *testing.T) {
	qj := factory.CreateQJ(t)
	factory.CreateQJ(t)
	proc := factory.Processor("http://example.com")
	ctx := context.Background()
	pools, err := dequeuer.CreatePools(ctx, proc, 0)
	test.AssertNotError(t, err, "CreatePools")
	test.AssertEquals(t, len(pools), 2)
	foundPool := false
	for _, pool := range pools {
		if pool.Name == qj.Name {
			foundPool = true
			test.AssertEquals(t, len(pool.Dequeuers), 3)
			for i, dq := range pool.Dequeuers {
				test.AssertEquals(t, dq.ID, i+1)
				test.AssertEquals(t, dq.W, proc)
			}
		}
	}
	test.Assert(t, foundPool, "Didn't create a pool for the job type")
}

func runDQBench(b *testing.B, populate bool, concurrency int16) {
	test.SetUp(b)
	// defer test.TearDown(b)
	data, _ := json.Marshal(factory.RD)
	if populate {
		if _, err := newmodels.DB.DeleteAllQueuedJobs(context.Background()); err != nil {
			b.Fatal(err)
		}
		job := factory.CreateJob(b, newmodels.CreateJobParams{
			Name:             factory.RandomId("").String()[:8],
			Concurrency:      concurrency,
			DeliveryStrategy: newmodels.DeliveryStrategyAtLeastOnce,
			Attempts:         1,
		})
		var wg sync.WaitGroup
		sem := semaphore.New(16)
		// Idea here is basically to insert enough jobs that the dequeuer always has
		// work to do, no matter how long the benchmark takes to run.
		for j := 0; j < 50000; j++ {
			sem.Acquire()
			wg.Add(1)
			go func() {
				defer sem.Release()
				defer wg.Done()
				factory.CreateQueuedJobOnly(b, job.Name, data)
			}()
		}
		wg.Wait()
	}
	logger := log.New()
	logger.SetHandler(log.DiscardHandler())
	w := &channelProcessor{
		Logger: logger,
		Ch:     make(chan struct{}, 1000),
	}
	b.ResetTimer()
	b.ReportAllocs()
	b.SetBytes(int64(len(data)))
	ctx := context.Background()
	pools, err := dequeuer.CreatePools(ctx, w, 0)
	test.AssertNotError(b, err, "CreatePools")
	defer func() {
		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		for _, pool := range pools {
			pool.Shutdown(ctx)
		}
		cancel()
	}()
	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			<-w.Ch
		}
	})
	count, err := newmodels.DB.CountReadyAndAll(context.Background())
	if err != nil {
		b.Fatal(err)
	}
	if count.All == 0 {
		b.Fatal("benchmark removed all rows from the queued_jobs table")
	}
}

func BenchmarkDequeue(b *testing.B) {
	b.Run("Dequeue1", func(b1 *testing.B) { runDQBench(b1, false, 1) })
	b.Run("Dequeue4", func(b4 *testing.B) { runDQBench(b4, false, 4) })
	b.Run("Dequeue8", func(b8 *testing.B) { runDQBench(b8, false, 8) })
	b.Run("Dequeue16", func(b16 *testing.B) { runDQBench(b16, false, 16) })
	b.Run("Dequeue64", func(b64 *testing.B) { runDQBench(b64, false, 64) })
	b.Run("Dequeue128", func(b128 *testing.B) { runDQBench(b128, false, 128) })
}
<commit_msg>test/dequeuer: add better error message<commit_after>package dequeuer

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	log "github.com/inconshreveable/log15"
	"github.com/kevinburke/rickover/dequeuer"
	"github.com/kevinburke/rickover/newmodels"
	"github.com/kevinburke/rickover/test"
	"github.com/kevinburke/rickover/test/factory"
	"github.com/kevinburke/semaphore"
)

func TestAll(t *testing.T) {
	test.SetUp(t)
	t.Cleanup(func() { test.TearDown(t) })
	t.Run("Parallel", func(t *testing.T) {
		t.Run("TestWorkerShutsDown", testWorkerShutsDown)
		t.Run("TestWorkerShutsDownWithContext", testWorkerShutsDownWithContext)
		t.Run("TestWorkerMakesCorrectRequest", testWorkerMakesCorrectRequest)
		t.Run("TestWorkerMakesExactlyOneRequest", testWorkerMakesExactlyOneRequest)
	})
}

func testWorkerShutsDown(t *testing.T) {
	t.Parallel()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	poolname := factory.RandomId("pool")
	pool := dequeuer.NewPool(ctx, poolname.String())
	for i := 0; i < 3; i++ {
		pool.AddDequeuer(ctx, factory.Processor("http://example.com"))
	}
	c1 := make(chan struct{}, 1)
	go func() {
		err := pool.Shutdown(ctx)
		test.AssertNotError(t, err, "")
		c1 <- struct{}{}
	}()
	for {
		select {
		case <-c1:
			return
		case <-time.After(300 * time.Millisecond):
			t.Fatalf("pool did not shut down in 300ms")
		}
	}
}

func testWorkerShutsDownWithContext(t *testing.T) {
	t.Parallel()
	ctx, cancel := context.WithCancel(context.Background())
	poolname := factory.RandomId("pool")
	pool := dequeuer.NewPool(ctx, poolname.String())
	for i := 0; i < 3; i++ {
		pool.AddDequeuer(ctx, factory.Processor("http://example.com"))
	}
	c1 := make(chan struct{}, 1)
	go func() {
		for {
			if pool.Len() == 0 {
				c1 <- struct{}{}
				break
			}
			time.Sleep(2 * time.Millisecond)
		}
	}()
	cancel()
	for {
		select {
		case <-c1:
			return
		case <-time.After(300 * time.Millisecond):
			t.Fatalf("pool did not shut down in 300ms")
		}
	}
}

// 1. Create a job type
// 2. Enqueue a job
// 3. Create a test server that replies with a 202
// 4. Ensure that the correct request is made to the server
func testWorkerMakesCorrectRequest(t *testing.T) {
	qj := factory.CreateQJ(t)

	c1 := make(chan bool, 1)
	var path, method, user string
	var ok bool
	var workRequest struct {
		Data     *factory.RandomData `json:"data"`
		Attempts int16               `json:"attempts"`
	}
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		path = r.URL.Path
		method = r.Method
		user, _, ok = r.BasicAuth()
		err := json.NewDecoder(r.Body).Decode(&workRequest)
		test.AssertNotError(t, err, "decoding request body")
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		w.WriteHeader(http.StatusAccepted)
		w.Write([]byte("{}"))
		c1 <- true
		close(c1)
	}))
	defer s.Close()
	jp := factory.Processor(s.URL)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	pool := dequeuer.NewPool(ctx, qj.Name)
	pool.AddDequeuer(ctx, jp)
	defer pool.Shutdown(context.Background())
	select {
	case <-c1:
		test.AssertEquals(t, path, fmt.Sprintf("/v1/jobs/%s/%s", qj.Name, qj.ID.String()))
		test.AssertEquals(t, method, "POST")
		test.AssertEquals(t, ok, true)
		test.AssertEquals(t, user, "jobs")
		test.AssertDeepEquals(t, workRequest.Data, factory.RD)
		test.AssertEquals(t, workRequest.Attempts, qj.Attempts)
		return
	case <-time.After(200 * time.Millisecond):
		t.Fatalf("Server did not receive a request in 200ms, quitting")
	}
}

// 1. Create a job type
// 2. Enqueue a job
// 2a. Create twenty worker nodes
// 3. Create a test server that replies with a 202
// 4. Ensure that only one request is made to the server
func testWorkerMakesExactlyOneRequest(t *testing.T) {
	qj := factory.CreateQJ(t)

	c1 := int32(0)
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		w.WriteHeader(http.StatusAccepted)
		w.Write([]byte("{}"))
		atomic.AddInt32(&c1, 1)
	}))
	defer s.Close()
	ctx, cancel := context.WithCancel(context.Background())
	pool := dequeuer.NewPool(ctx, qj.Name)
	for i := 0; i < 20; i++ {
		jp := factory.Processor(s.URL)
		pool.AddDequeuer(ctx, jp)
	}
	defer cancel()
	<-time.After(100 * time.Millisecond)
	val := atomic.LoadInt32(&c1)
	test.Assert(t, val == int32(1), fmt.Sprintf("request should have been called once, was called %d times", val))
}

func TestCreatePools(t *testing.T) {
	qj := factory.CreateQJ(t)
	factory.CreateQJ(t)
	proc := factory.Processor("http://example.com")
	ctx := context.Background()
	pools, err := dequeuer.CreatePools(ctx, proc, 0)
	test.AssertNotError(t, err, "CreatePools")
	test.AssertEquals(t, len(pools), 2)
	foundPool := false
	for _, pool := range pools {
		if pool.Name == qj.Name {
			foundPool = true
			test.AssertEquals(t, len(pool.Dequeuers), 3)
			for i, dq := range pool.Dequeuers {
				test.AssertEquals(t, dq.ID, i+1)
				test.AssertEquals(t, dq.W, proc)
			}
		}
	}
	test.Assert(t, foundPool, "Didn't create a pool for the job type")
}

func runDQBench(b *testing.B, populate bool, concurrency int16) {
	test.SetUp(b)
	// defer test.TearDown(b)
	data, _ := json.Marshal(factory.RD)
	if populate {
		if _, err := newmodels.DB.DeleteAllQueuedJobs(context.Background()); err != nil {
			b.Fatal(err)
		}
		job := factory.CreateJob(b, newmodels.CreateJobParams{
			Name:             factory.RandomId("").String()[:8],
			Concurrency:      concurrency,
			DeliveryStrategy: newmodels.DeliveryStrategyAtLeastOnce,
			Attempts:         1,
		})
		var wg sync.WaitGroup
		sem := semaphore.New(16)
		// Idea here is basically to insert enough jobs that the dequeuer always has
		// work to do, no matter how long the benchmark takes to run.
		for j := 0; j < 50000; j++ {
			sem.Acquire()
			wg.Add(1)
			go func() {
				defer sem.Release()
				defer wg.Done()
				factory.CreateQueuedJobOnly(b, job.Name, data)
			}()
		}
		wg.Wait()
	}
	logger := log.New()
	logger.SetHandler(log.DiscardHandler())
	w := &channelProcessor{
		Logger: logger,
		Ch:     make(chan struct{}, 1000),
	}
	b.ResetTimer()
	b.ReportAllocs()
	b.SetBytes(int64(len(data)))
	ctx := context.Background()
	pools, err := dequeuer.CreatePools(ctx, w, 0)
	test.AssertNotError(b, err, "CreatePools")
	defer func() {
		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		for _, pool := range pools {
			pool.Shutdown(ctx)
		}
		cancel()
	}()
	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			<-w.Ch
		}
	})
	count, err := newmodels.DB.CountReadyAndAll(context.Background())
	if err != nil {
		b.Fatal(err)
	}
	if count.All == 0 {
		b.Fatal("benchmark removed all rows from the queued_jobs table")
	}
}

func BenchmarkDequeue(b *testing.B) {
	b.Run("Dequeue1", func(b1 *testing.B) { runDQBench(b1, false, 1) })
	b.Run("Dequeue4", func(b4 *testing.B) { runDQBench(b4, false, 4) })
	b.Run("Dequeue8", func(b8 *testing.B) { runDQBench(b8, false, 8) })
	b.Run("Dequeue16", func(b16 *testing.B) { runDQBench(b16, false, 16) })
	b.Run("Dequeue64", func(b64 *testing.B) { runDQBench(b64, false, 64) })
	b.Run("Dequeue128", func(b128 *testing.B) { runDQBench(b128, false, 128) })
}
<|endoftext|>


<commit_before>package parser

import (
	"fmt"
	"plaid/lexer"
	"strconv"
	"strings"
)

// Node is the ancestor of all AST nodes
type Node interface {
	Start() lexer.Loc
	String() string
	isNode()
}

// Program describes all top-level statements within a script
type Program struct {
	Stmts []Stmt
}

// Start returns a location that this node can be considered to start at
func (p Program) Start() lexer.Loc {
	if len(p.Stmts) > 0 {
		return p.Stmts[0].Start()
	}

	return lexer.Loc{Line: 1, Col: 1}
}

func (p Program) String() string {
	out := ""
	for i, stmt := range p.Stmts {
		if i > 0 {
			out += "\n"
		}

		out += stmt.String()
	}
	return out
}

func (p Program) isNode() {}

// Stmt describes all constructs that return no value
type Stmt interface {
	Start() lexer.Loc
	String() string
	isNode()
	isStmt()
}

// StmtBlock describes any series of statements bounded by curly braces
type StmtBlock struct {
	left  lexer.Token
	stmts []Stmt
	right lexer.Token
}

// Start returns a location that this node can be considered to start at
func (sb StmtBlock) Start() lexer.Loc { return sb.left.Loc }

func (sb StmtBlock) String() string {
	out := "{"
	for _, stmt := range sb.stmts {
		out += "\n" + indentBlock("  ", stmt.String())
	}
	return out + "}"
}

func (sb StmtBlock) isNode() {}

// DeclarationStmt describes the declaration and assignment of a variable
type DeclarationStmt struct {
	tok  lexer.Token
	name IdentExpr
	expr Expr
}

// Start returns a location that this node can be considered to start at
func (ds DeclarationStmt) Start() lexer.Loc { return ds.tok.Loc }
func (ds DeclarationStmt) String() string   { return fmt.Sprintf("(let %s %s)", ds.name, ds.expr) }
func (ds DeclarationStmt) isNode()          {}
func (ds DeclarationStmt) isStmt()          {}

// ReturnStmt describes a return keyword and an optional returned expression.
type ReturnStmt struct {
	tok  lexer.Token
	expr Expr
}

// Start returns a location that this node can be considered to start at
func (rs ReturnStmt) Start() lexer.Loc { return rs.tok.Loc }

func (rs ReturnStmt) String() string {
	if rs.expr != nil {
		return fmt.Sprintf("(return %s)", rs.expr)
	}

	return "(return)"
}

func (rs ReturnStmt) isNode() {}
func (rs ReturnStmt) isStmt() {}

// ExprStmt describes certain expressions that can be used in the place of statements
type ExprStmt struct {
	expr Expr
}

// Start returns a location that this node can be considered to start at
func (es ExprStmt) Start() lexer.Loc { return es.expr.Start() }
func (es ExprStmt) String() string   { return es.expr.String() }
func (es ExprStmt) isNode()          {}
func (es ExprStmt) isStmt()          {}

// TypeSig describes a syntax type annotation
type TypeSig interface {
	Start() lexer.Loc
	String() string
	isNode()
	isType()
}

// TypeTuple describes a set of 0 or more types wrapped in parentheses
type TypeTuple struct {
	tok   lexer.Token
	elems []TypeSig
}

// Start returns a location that this node can be considered to start at
func (tt TypeTuple) Start() lexer.Loc { return tt.tok.Loc }

func (tt TypeTuple) String() string {
	out := "("
	for i, elem := range tt.elems {
		if i > 0 {
			out += " "
		}
		out += elem.String()
	}
	out += ")"
	return out
}

func (tt TypeTuple) isNode() {}
func (tt TypeTuple) isType() {}

// TypeFunction describes a function type annotation
type TypeFunction struct {
	params TypeTuple
	ret    TypeSig
}

// Start returns a location that this node can be considered to start at
func (tf TypeFunction) Start() lexer.Loc { return tf.params.Start() }

func (tf TypeFunction) String() string {
	out := tf.params.String()
	out += " => "
	out += tf.ret.String()
	return out
}

func (tf TypeFunction) isNode() {}
func (tf TypeFunction) isType() {}

// TypeIdent describes a named reference to a type
type TypeIdent struct {
	tok  lexer.Token
	name string
}

// Start returns a location that this node can be considered to start at
func (ti TypeIdent) Start() lexer.Loc { return ti.tok.Loc }
func (ti TypeIdent) String() string   { return ti.name }
func (ti TypeIdent) isNode()          {}
func (ti TypeIdent) isType()          {}

// TypeList describes a list type
type TypeList struct {
	tok   lexer.Token
	child TypeSig
}

// Start returns a location that this node can be considered to start at
func (tl TypeList) Start() lexer.Loc { return tl.tok.Loc }
func (tl TypeList) String() string   { return fmt.Sprintf("[%s]", tl.child) }
func (tl TypeList) isNode()          {}
func (tl TypeList) isType()          {}

// TypeOptional describes a list type
type TypeOptional struct {
	tok   lexer.Token
	child TypeSig
}

// Start returns a location that this node can be considered to start at
func (to TypeOptional) Start() lexer.Loc { return to.child.Start() }
func (to TypeOptional) String() string   { return fmt.Sprintf("%s?", to.child) }
func (to TypeOptional) isNode()          {}
func (to TypeOptional) isType()          {}

// Expr describes all constructs that resolve to a value
type Expr interface {
	Start() lexer.Loc
	String() string
	isNode()
	isExpr()
}

// FunctionExpr describes a function's entire type signature and body
type FunctionExpr struct {
	tok    lexer.Token
	params []FunctionParam
	ret    TypeSig
	block  StmtBlock
}

// Start returns a location that this node can be considered to start at
func (fe FunctionExpr) Start() lexer.Loc { return fe.tok.Loc }

func (fe FunctionExpr) String() string {
	out := "(fn ("
	for i, param := range fe.params {
		if i > 0 {
			out += " "
		}
		out += param.String()
	}
	out += ")"
	if fe.ret != nil {
		out += fmt.Sprintf(":%s", fe.ret)
	}
	out += fmt.Sprintf(" %s)", fe.block)
	return out
}

func (fe FunctionExpr) isExpr() {}
func (fe FunctionExpr) isNode() {}

// FunctionParam describes a single function argument's name and type signature
type FunctionParam struct {
	name IdentExpr
	sig  TypeSig
}

// Start returns a location that this node can be considered to start at
func (fp FunctionParam) Start() lexer.Loc { return fp.name.Start() }

func (fp FunctionParam) String() string {
	if fp.sig != nil {
		return fmt.Sprintf("%s:%s", fp.name, fp.sig)
	}

	return fp.name.String()
}

func (fp FunctionParam) isNode() {}

// DispatchExpr describes a function call including the callee and any arguments
type DispatchExpr struct {
	callee Expr
	args   []Expr
}

// Start returns a location that this node can be considered to start at
func (de DispatchExpr) Start() lexer.Loc { return de.callee.Start() }

func (de DispatchExpr) String() string {
	out := "("
	out += de.callee.String()
	out += " ("
	for i, arg := range de.args {
		if i > 0 {
			out += " "
		}
		out += arg.String()
	}
	out += "))"
	return out
}

func (de DispatchExpr) isNode() {}
func (de DispatchExpr) isExpr() {}

// AssignExpr describes the binding of a value to an assignable expression
type AssignExpr struct {
	tok   lexer.Token
	left  Expr
	right Expr
}

// Start returns a location that this node can be considered to start at
func (ae AssignExpr) Start() lexer.Loc { return ae.left.Start() }
func (ae AssignExpr) String() string   { return fmt.Sprintf("(= %s %s)", ae.left, ae.right) }
func (ae AssignExpr) isNode()          {}
func (ae AssignExpr) isExpr()          {}

// BinaryExpr describes any two expressions associated by an operator
type BinaryExpr struct {
	oper  string
	tok   lexer.Token
	left  Expr
	right Expr
}

// Start returns a location that this node can be considered to start at
func (be BinaryExpr) Start() lexer.Loc { return be.left.Start() }
func (be BinaryExpr) String() string   { return fmt.Sprintf("(%s %s %s)", be.oper, be.left, be.right) }
func (be BinaryExpr) isNode()          {}
func (be BinaryExpr) isExpr()          {}

// UnaryExpr describes any single expression associated to an operator
type UnaryExpr struct {
	oper string
	tok  lexer.Token
	expr Expr
}

// Start returns a location that this node can be considered to start at
func (ue UnaryExpr) Start() lexer.Loc { return lexer.SmallerLoc(ue.tok.Loc, ue.expr.Start()) }
func (ue UnaryExpr) String() string   { return fmt.Sprintf("(%s %s)", ue.oper, ue.expr) }
func (ue UnaryExpr) isNode()          {}
func (ue UnaryExpr) isExpr()          {}

// IdentExpr describes an identifier
type IdentExpr struct {
	tok  lexer.Token
	name string
}

// Start returns a location that this node can be considered to start at
func (ie IdentExpr) Start() lexer.Loc { return ie.tok.Loc }
func (ie IdentExpr) String() string   { return ie.name }
func (ie IdentExpr) isNode()          {}
func (ie IdentExpr) isExpr()          {}

// StringExpr describes a string literal
type StringExpr struct {
	tok lexer.Token
	val string
}

// Start returns a location that this node can be considered to start at
func (se StringExpr) Start() lexer.Loc { return se.tok.Loc }
func (se StringExpr) String() string   { return fmt.Sprintf("\"%s\"", se.val) }
func (se StringExpr) isNode()          {}
func (se StringExpr) isExpr()          {}

// NumberExpr describes a string literal
type NumberExpr struct {
	tok lexer.Token
	val int
}

// Start returns a location that this node can be considered to start at
func (ne NumberExpr) Start() lexer.Loc { return ne.tok.Loc }
func (ne NumberExpr) String() string   { return strconv.Itoa(ne.val) }
func (ne NumberExpr) isNode()          {}
func (ne NumberExpr) isExpr()          {}

func indentBlock(indent string, source string) string {
	lines := strings.Split(source, "\n")
	for i, line := range lines {
		lines[i] = indent + line
	}

	return strings.Join(lines, "\n")
}
<commit_msg>export AST node properties<commit_after>package parser

import (
	"fmt"
	"plaid/lexer"
	"strconv"
	"strings"
)

// Node is the ancestor of all AST nodes
type Node interface {
	Start() lexer.Loc
	String() string
	isNode()
}

// Program describes all top-level statements within a script
type Program struct {
	Stmts []Stmt
}

// Start returns a location that this node can be considered to start at
func (p Program) Start() lexer.Loc {
	if len(p.Stmts) > 0 {
		return p.Stmts[0].Start()
	}

	return lexer.Loc{Line: 1, Col: 1}
}

func (p Program) String() string {
	out := ""
	for i, stmt := range p.Stmts {
		if i > 0 {
			out += "\n"
		}

		out += stmt.String()
	}
	return out
}

func (p Program) isNode() {}

// Stmt describes all constructs that return no value
type Stmt interface {
	Start() lexer.Loc
	String() string
	isNode()
	isStmt()
}

// StmtBlock describes any series of statements bounded by curly braces
type StmtBlock struct {
	Left  lexer.Token
	Stmts []Stmt
	Right lexer.Token
}

// Start returns a location that this node can be considered to start at
func (sb StmtBlock) Start() lexer.Loc { return sb.Left.Loc }

func (sb StmtBlock) String() string {
	out := "{"
	for _, stmt := range sb.Stmts {
		out += "\n" + indentBlock("  ", stmt.String())
	}
	return out + "}"
}

func (sb StmtBlock) isNode() {}

// DeclarationStmt describes the declaration and assignment of a variable
type DeclarationStmt struct {
	Tok  lexer.Token
	Name IdentExpr
	Expr Expr
}

// Start returns a location that this node can be considered to start at
func (ds DeclarationStmt) Start() lexer.Loc { return ds.Tok.Loc }
func (ds DeclarationStmt) String() string   { return fmt.Sprintf("(let %s %s)", ds.Name, ds.Expr) }
func (ds DeclarationStmt) isNode()          {}
func (ds DeclarationStmt) isStmt()          {}

// ReturnStmt describes a return keyword and an optional returned expression.
type ReturnStmt struct {
	Tok  lexer.Token
	Expr Expr
}

// Start returns a location that this node can be considered to start at
func (rs ReturnStmt) Start() lexer.Loc { return rs.Tok.Loc }

func (rs ReturnStmt) String() string {
	if rs.Expr != nil {
		return fmt.Sprintf("(return %s)", rs.Expr)
	}

	return "(return)"
}

func (rs ReturnStmt) isNode() {}
func (rs ReturnStmt) isStmt() {}

// ExprStmt describes certain expressions that can be used in the place of statements
type ExprStmt struct {
	Expr Expr
}

// Start returns a location that this node can be considered to start at
func (es ExprStmt) Start() lexer.Loc { return es.Expr.Start() }
func (es ExprStmt) String() string   { return es.Expr.String() }
func (es ExprStmt) isNode()          {}
func (es ExprStmt) isStmt()          {}

// TypeSig describes a syntax type annotation
type TypeSig interface {
	Start() lexer.Loc
	String() string
	isNode()
	isType()
}

// TypeTuple describes a set of 0 or more types wrapped in parentheses
type TypeTuple struct {
	Tok   lexer.Token
	Elems []TypeSig
}

// Start returns a location that this node can be considered to start at
func (tt TypeTuple) Start() lexer.Loc { return tt.Tok.Loc }

func (tt TypeTuple) String() string {
	out := "("
	for i, elem := range tt.Elems {
		if i > 0 {
			out += " "
		}
		out += elem.String()
	}
	out += ")"
	return out
}

func (tt TypeTuple) isNode() {}
func (tt TypeTuple) isType() {}

// TypeFunction describes a function type annotation
type TypeFunction struct {
	Params TypeTuple
	Ret    TypeSig
}

// Start returns a location that this node can be considered to start at
func (tf TypeFunction) Start() lexer.Loc { return tf.Params.Start() }

func (tf TypeFunction) String() string {
	out := tf.Params.String()
	out += " => "
	out += tf.Ret.String()
	return out
}

func (tf TypeFunction) isNode() {}
func (tf TypeFunction) isType() {}

// TypeIdent describes a named reference to a type
type TypeIdent struct {
	Tok  lexer.Token
	Name string
}

// Start returns a location that this node can be considered to start at
func (ti TypeIdent) Start() lexer.Loc { return ti.Tok.Loc }
func (ti TypeIdent) String() string   { return ti.Name }
func (ti TypeIdent) isNode()          {}
func (ti TypeIdent) isType()          {}

// TypeList describes a list type
type TypeList struct {
	Tok   lexer.Token
	Child TypeSig
}

// Start returns a location that this node can be considered to start at
func (tl TypeList) Start() lexer.Loc { return tl.Tok.Loc }
func (tl TypeList) String() string   { return fmt.Sprintf("[%s]", tl.Child) }
func (tl TypeList) isNode()          {}
func (tl TypeList) isType()          {}

// TypeOptional describes a list type
type TypeOptional struct {
	Tok   lexer.Token
	Child TypeSig
}

// Start returns a location that this node can be considered to start at
func (to TypeOptional) Start() lexer.Loc { return to.Child.Start() }
func (to TypeOptional) String() string   { return fmt.Sprintf("%s?", to.Child) }
func (to TypeOptional) isNode()          {}
func (to TypeOptional) isType()          {}

// Expr describes all constructs that resolve to a value
type Expr interface {
	Start() lexer.Loc
	String() string
	isNode()
	isExpr()
}

// FunctionExpr describes a function's entire type signature and body
type FunctionExpr struct {
	Tok    lexer.Token
	Params []FunctionParam
	Ret    TypeSig
	Block  StmtBlock
}

// Start returns a location that this node can be considered to start at
func (fe FunctionExpr) Start() lexer.Loc { return fe.Tok.Loc }

func (fe FunctionExpr) String() string {
	out := "(fn ("
	for i, param := range fe.Params {
		if i > 0 {
			out += " "
		}
		out += param.String()
	}
	out += ")"
	if fe.Ret != nil {
		out += fmt.Sprintf(":%s", fe.Ret)
	}
	out += fmt.Sprintf(" %s)", fe.Block)
	return out
}

func (fe FunctionExpr) isExpr() {}
func (fe FunctionExpr) isNode() {}

// FunctionParam describes a single function argument's name and type signature
type FunctionParam struct {
	Name IdentExpr
	Sig  TypeSig
}

// Start returns a location that this node can be considered to start at
func (fp FunctionParam) Start() lexer.Loc { return fp.Name.Start() }

func (fp FunctionParam) String() string {
	if fp.Sig != nil {
		return fmt.Sprintf("%s:%s", fp.Name, fp.Sig)
	}

	return fp.Name.String()
}

func (fp FunctionParam) isNode() {}

// DispatchExpr describes a function call including the callee and any arguments
type DispatchExpr struct {
	Callee Expr
	Args   []Expr
}

// Start returns a location that this node can be considered to start at
func (de DispatchExpr) Start() lexer.Loc { return de.Callee.Start() }

func (de DispatchExpr) String() string {
	out := "("
	out += de.Callee.String()
	out += " ("
	for i, arg := range de.Args {
		if i > 0 {
			out += " "
		}
		out += arg.String()
	}
	out += "))"
	return out
}

func (de DispatchExpr) isNode() {}
func (de DispatchExpr) isExpr() {}

// AssignExpr describes the binding of a value to an assignable expression
type AssignExpr struct {
	Tok   lexer.Token
	Left  Expr
	Right Expr
}

// Start returns a location that this node can be considered to start at
func (ae AssignExpr) Start() lexer.Loc { return ae.Left.Start() }
func (ae AssignExpr) String() string   { return fmt.Sprintf("(= %s %s)", ae.Left, ae.Right) }
func (ae AssignExpr) isNode()          {}
func (ae AssignExpr) isExpr()          {}

// BinaryExpr describes any two expressions associated by an operator
type BinaryExpr struct {
	Oper  string
	Tok   lexer.Token
	Left  Expr
	Right Expr
}

// Start returns a location that this node can be considered to start at
func (be BinaryExpr) Start() lexer.Loc { return be.Left.Start() }
func (be BinaryExpr) String() string   { return fmt.Sprintf("(%s %s %s)", be.Oper, be.Left, be.Right) }
func (be BinaryExpr) isNode()          {}
func (be BinaryExpr) isExpr()          {}

// UnaryExpr describes any single expression associated to an operator
type UnaryExpr struct {
	Oper string
	Tok  lexer.Token
	Expr Expr
}

// Start returns a location that this node can be considered to start at
func (ue UnaryExpr) Start() lexer.Loc { return lexer.SmallerLoc(ue.Tok.Loc, ue.Expr.Start()) }
func (ue UnaryExpr) String() string   { return fmt.Sprintf("(%s %s)", ue.Oper, ue.Expr) }
func (ue UnaryExpr) isNode()          {}
func (ue UnaryExpr) isExpr()          {}

// IdentExpr describes an identifier
type IdentExpr struct {
	Tok  lexer.Token
	Name string
}

// Start returns a location that this node can be considered to start at
func (ie IdentExpr) Start() lexer.Loc { return ie.Tok.Loc }
func (ie IdentExpr) String() string   { return ie.Name }
func (ie IdentExpr) isNode()          {}
func (ie IdentExpr) isExpr()          {}

// StringExpr describes a string literal
type StringExpr struct {
	Tok lexer.Token
	Val string
}

// Start returns a location that this node can be considered to start at
func (se StringExpr) Start() lexer.Loc { return se.Tok.Loc }
func (se StringExpr) String() string   { return fmt.Sprintf("\"%s\"", se.Val) }
func (se StringExpr) isNode()          {}
func (se StringExpr) isExpr()          {}

// NumberExpr describes a string literal
type NumberExpr struct {
	Tok lexer.Token
	Val int
}

// Start returns a location that this node can be considered to start at
func (ne NumberExpr) Start() lexer.Loc { return ne.Tok.Loc }
func (ne NumberExpr) String() string   { return strconv.Itoa(ne.Val) }
func (ne NumberExpr) isNode()          {}
func (ne NumberExpr) isExpr()          {}

func indentBlock(indent string, source string) string {
	lines := strings.Split(source, "\n")
	for i, line := range lines {
		lines[i] = indent + line
	}

	return strings.Join(lines, "\n")
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package auth

import (
	"fmt"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"

	auditregv1alpha1 "k8s.io/api/auditregistration/v1alpha1"
	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/wait"
	auditinternal "k8s.io/apiserver/pkg/apis/audit"
	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
	clientset "k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
	"k8s.io/kubernetes/test/e2e/framework"
	"k8s.io/kubernetes/test/utils"
	imageutils "k8s.io/kubernetes/test/utils/image"
)

var _ = SIGDescribe("[Feature:DynamicAudit]", func() {
	f := framework.NewDefaultFramework("audit")
	BeforeEach(func() {
		framework.SkipUnlessProviderIs("gce")
	})

	// TODO: Get rid of [DisabledForLargeClusters] when feature request #53455 is ready.
	It("should dynamically audit API calls [DisabledForLargeClusters]", func() {
		namespace := f.Namespace.Name

		By("Creating a kubernetes client that impersonates an unauthorized anonymous user")
		config, err := framework.LoadConfig()
		framework.ExpectNoError(err, "failed to fetch config")

		config.Impersonate = restclient.ImpersonationConfig{
			UserName: "system:anonymous",
			Groups:   []string{"system:unauthenticated"},
		}
		anonymousClient, err := clientset.NewForConfig(config)
		framework.ExpectNoError(err, "failed to create the anonymous client")

		_, err = f.ClientSet.CoreV1().Namespaces().Create(&apiv1.Namespace{
			ObjectMeta: metav1.ObjectMeta{
				Name: "audit",
			},
		})
		framework.ExpectNoError(err, "failed to create namespace")

		_, err = f.ClientSet.CoreV1().Pods(namespace).Create(&apiv1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "audit-proxy",
				Labels: map[string]string{
					"app": "audit",
				},
			},
			Spec: apiv1.PodSpec{
				Containers: []apiv1.Container{
					{
						Name:  "proxy",
						Image: imageutils.GetE2EImage(imageutils.AuditProxy),
						Ports: []apiv1.ContainerPort{
							{
								ContainerPort: 8080,
							},
						},
					},
				},
			},
		})
		framework.ExpectNoError(err, "failed to create proxy pod")

		_, err = f.ClientSet.CoreV1().Services(namespace).Create(&apiv1.Service{
			ObjectMeta: metav1.ObjectMeta{
				Name: "audit",
			},
			Spec: apiv1.ServiceSpec{
				Ports: []apiv1.ServicePort{
					{
						Port:       80,
						TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080},
					},
				},
				Selector: map[string]string{
					"app": "audit",
				},
			},
		})
		framework.ExpectNoError(err, "failed to create proxy service")

		config, err = framework.LoadConfig()
		framework.ExpectNoError(err, "failed to load config")

		var podIP string
		// get pod ip
		err = wait.Poll(100*time.Millisecond, 10*time.Second, func() (done bool, err error) {
			p, err := f.ClientSet.CoreV1().Pods(namespace).Get("audit-proxy", metav1.GetOptions{})
			if errors.IsNotFound(err) {
				return false, nil
			} else if err != nil {
				return false, err
			}
			podIP = p.Status.PodIP
			if podIP == "" {
				return false, nil
			}
			return true, nil
		})
		framework.ExpectNoError(err, "could not look up pod ip")

		podURL := fmt.Sprintf("http://%s:8080", podIP)
		// create audit sink
		sink := auditregv1alpha1.AuditSink{
			ObjectMeta: metav1.ObjectMeta{
				Name: "test",
			},
			Spec: auditregv1alpha1.AuditSinkSpec{
				Policy: auditregv1alpha1.Policy{
					Level: auditregv1alpha1.LevelRequestResponse,
					Stages: []auditregv1alpha1.Stage{
						auditregv1alpha1.StageRequestReceived,
						auditregv1alpha1.StageResponseStarted,
						auditregv1alpha1.StageResponseComplete,
						auditregv1alpha1.StagePanic,
					},
				},
				Webhook: auditregv1alpha1.Webhook{
					ClientConfig: auditregv1alpha1.WebhookClientConfig{
						URL: &podURL,
					},
				},
			},
		}

		_, err = f.ClientSet.AuditregistrationV1alpha1().AuditSinks().Create(&sink)
		framework.ExpectNoError(err, "failed to create audit sink")
		framework.Logf("created audit sink")

		// check that we are receiving logs in the proxy
		err = wait.Poll(100*time.Millisecond, 10*time.Second, func() (done bool, err error) {
			logs, err := framework.GetPodLogs(f.ClientSet, namespace, "audit-proxy", "proxy")
			if err != nil {
				return false, nil
			}
			if logs == "" {
				return false, nil
			}
			return true, nil
		})
		framework.ExpectNoError(err, "failed to get logs from sink")

		auditTestUser = "kubernetes-admin"
		testCases := []struct {
			action func()
			events []utils.AuditEvent
		}{
			// Create, get, update, patch, delete, list, watch pods.
			// TODO: dedup this with the main audit test once policy functionality is available
			// https://github.com/kubernetes/kubernetes/issues/70818
			{
				func() {
					pod := &apiv1.Pod{
						ObjectMeta: metav1.ObjectMeta{
							Name: "audit-pod",
						},
						Spec: apiv1.PodSpec{
							Containers: []apiv1.Container{{
								Name:  "pause",
								Image: imageutils.GetPauseImageName(),
							}},
						},
					}
					updatePod := func(pod *apiv1.Pod) {}

					f.PodClient().CreateSync(pod)

					_, err := f.PodClient().Get(pod.Name, metav1.GetOptions{})
					framework.ExpectNoError(err, "failed to get audit-pod")

					podChan, err := f.PodClient().Watch(watchOptions)
					framework.ExpectNoError(err, "failed to create watch for pods")
					for range podChan.ResultChan() {
					}

					f.PodClient().Update(pod.Name, updatePod)

					_, err = f.PodClient().List(metav1.ListOptions{})
					framework.ExpectNoError(err, "failed to list pods")

					_, err = f.PodClient().Patch(pod.Name, types.JSONPatchType, patch)
					framework.ExpectNoError(err, "failed to patch pod")

					f.PodClient().DeleteSync(pod.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
				},
				[]utils.AuditEvent{
					{
						Level:             auditinternal.LevelRequestResponse,
						Stage:             auditinternal.StageResponseComplete,
						RequestURI:        fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace),
						Verb:              "create",
						Code:              201,
						User:              auditTestUser,
						Resource:          "pods",
						Namespace:         namespace,
						RequestObject:     true,
						ResponseObject:    true,
						AuthorizeDecision: "allow",
					}, {
						Level:             auditinternal.LevelRequestResponse,
						Stage:             auditinternal.StageResponseComplete,
						RequestURI:        fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace),
						Verb:              "get",
						Code:              200,
						User:              auditTestUser,
						Resource:          "pods",
						Namespace:         namespace,
						RequestObject:     false,
						ResponseObject:    true,
						AuthorizeDecision: "allow",
					}, {
						Level:             auditinternal.LevelRequestResponse,
						Stage:             auditinternal.StageResponseComplete,
						RequestURI:        fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace),
						Verb:              "list",
						Code:              200,
						User:              auditTestUser,
						Resource:          "pods",
						Namespace:         namespace,
						RequestObject:     false,
						ResponseObject:    true,
						AuthorizeDecision: "allow",
					}, {
						Level:             auditinternal.LevelRequestResponse,
						Stage:             auditinternal.StageResponseStarted,
						RequestURI:        fmt.Sprintf("/api/v1/namespaces/%s/pods?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout),
						Verb:              "watch",
						Code:              200,
						User:              auditTestUser,
						Resource:          "pods",
						Namespace:         namespace,
						RequestObject:     false,
						ResponseObject:    false,
						AuthorizeDecision: "allow",
					}, {
						Level:             auditinternal.LevelRequestResponse,
						Stage:             auditinternal.StageResponseComplete,
						RequestURI:        fmt.Sprintf("/api/v1/namespaces/%s/pods?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout),
						Verb:              "watch",
						Code:              200,
						User:              auditTestUser,
						Resource:          "pods",
						Namespace:         namespace,
						RequestObject:     false,
						ResponseObject:    false,
						AuthorizeDecision: "allow",
					}, {
						Level:             auditinternal.LevelRequestResponse,
						Stage:             auditinternal.StageResponseComplete,
						RequestURI:        fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace),
						Verb:              "update",
						Code:              200,
						User:              auditTestUser,
						Resource:          "pods",
						Namespace:         namespace,
						RequestObject:     true,
						ResponseObject:    true,
						AuthorizeDecision: "allow",
					}, {
						Level:             auditinternal.LevelRequestResponse,
						Stage:             auditinternal.StageResponseComplete,
						RequestURI:        fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace),
						Verb:              "patch",
						Code:              200,
						User:              auditTestUser,
						Resource:          "pods",
						Namespace:         namespace,
						RequestObject:     true,
						ResponseObject:    true,
						AuthorizeDecision: "allow",
					}, {
						Level:             auditinternal.LevelRequestResponse,
						Stage:             auditinternal.StageResponseComplete,
						RequestURI:        fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace),
						Verb:              "delete",
						Code:              200,
						User:              auditTestUser,
						Resource:          "pods",
						Namespace:         namespace,
						RequestObject:     true,
						ResponseObject:    true,
						AuthorizeDecision: "allow",
					},
				},
			},
		}

		// test authorizer annotations, RBAC is required.
		annotationTestCases := []struct {
			action func()
			events []utils.AuditEvent
		}{

			// get a pod with unauthorized user
			{
				func() {
					_, err := anonymousClient.CoreV1().Pods(namespace).Get("another-audit-pod", metav1.GetOptions{})
					expectForbidden(err)
				},
				[]utils.AuditEvent{
					{
						Level:              auditinternal.LevelRequestResponse,
						Stage:              auditinternal.StageResponseComplete,
						RequestURI:         fmt.Sprintf("/api/v1/namespaces/%s/pods/another-audit-pod", namespace),
						Verb:               "get",
						Code:               403,
						User:               auditTestUser,
						ImpersonatedUser:   "system:anonymous",
						ImpersonatedGroups: "system:unauthenticated",
						Resource:           "pods",
						Namespace:          namespace,
						RequestObject:      false,
						ResponseObject:     true,
						AuthorizeDecision:  "forbid",
					},
				},
			},
		}

		if framework.IsRBACEnabled(f) {
			testCases = append(testCases, annotationTestCases...)
		}
		expectedEvents := []utils.AuditEvent{}
		for _, t := range testCases {
			t.action()
			expectedEvents = append(expectedEvents, t.events...)
		}

		// The default flush timeout is 30 seconds, therefore it should be enough to retry once
		// to find all expected events. However, we're waiting for 5 minutes to avoid flakes.
		pollingInterval := 30 * time.Second
		pollingTimeout := 5 * time.Minute
		err = wait.Poll(pollingInterval, pollingTimeout, func() (bool, error) {
			// Fetch the logs
			logs, err := framework.GetPodLogs(f.ClientSet, namespace, "audit-proxy", "proxy")
			if err != nil {
				return false, err
			}
			reader := strings.NewReader(logs)
			missing, err := utils.CheckAuditLines(reader, expectedEvents, auditv1.SchemeGroupVersion)
			if err != nil {
				framework.Logf("Failed to observe audit events: %v", err)
			} else if len(missing) > 0 {
				framework.Logf("Events %#v not found!", missing)
			}
			return len(missing) == 0, nil
		})
		framework.ExpectNoError(err, "after %v failed to observe audit events", pollingTimeout)
		err = f.ClientSet.AuditregistrationV1alpha1().AuditSinks().Delete("test", &metav1.DeleteOptions{})
		framework.ExpectNoError(err, "could not delete audit configuration")
	})
})
<commit_msg>adds better logging around retry loops in dynamic audit e2e test<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package auth

import (
	"fmt"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"

	auditregv1alpha1 "k8s.io/api/auditregistration/v1alpha1"
	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/wait"
	auditinternal "k8s.io/apiserver/pkg/apis/audit"
	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
	clientset "k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
	"k8s.io/kubernetes/test/e2e/framework"
	"k8s.io/kubernetes/test/utils"
	imageutils "k8s.io/kubernetes/test/utils/image"
)

var _ = SIGDescribe("[Feature:DynamicAudit]", func() {
	f := framework.NewDefaultFramework("audit")
	BeforeEach(func() {
		framework.SkipUnlessProviderIs("gce")
	})

	// TODO: Get rid of [DisabledForLargeClusters] when feature request #53455 is ready.
	It("should dynamically audit API calls [DisabledForLargeClusters]", func() {
		namespace := f.Namespace.Name

		By("Creating a kubernetes client that impersonates an unauthorized anonymous user")
		config, err := framework.LoadConfig()
		framework.ExpectNoError(err, "failed to fetch config")

		config.Impersonate = restclient.ImpersonationConfig{
			UserName: "system:anonymous",
			Groups:   []string{"system:unauthenticated"},
		}
		anonymousClient, err := clientset.NewForConfig(config)
		framework.ExpectNoError(err, "failed to create the anonymous client")

		_, err = f.ClientSet.CoreV1().Namespaces().Create(&apiv1.Namespace{
			ObjectMeta: metav1.ObjectMeta{
				Name: "audit",
			},
		})
		framework.ExpectNoError(err, "failed to create namespace")

		_, err = f.ClientSet.CoreV1().Pods(namespace).Create(&apiv1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "audit-proxy",
				Labels: map[string]string{
					"app": "audit",
				},
			},
			Spec: apiv1.PodSpec{
				Containers: []apiv1.Container{
					{
						Name:  "proxy",
						Image: imageutils.GetE2EImage(imageutils.AuditProxy),
						Ports: []apiv1.ContainerPort{
							{
								ContainerPort: 8080,
							},
						},
					},
				},
			},
		})
		framework.ExpectNoError(err, "failed to create proxy pod")

		_, err = f.ClientSet.CoreV1().Services(namespace).Create(&apiv1.Service{
			ObjectMeta: metav1.ObjectMeta{
				Name: "audit",
			},
			Spec: apiv1.ServiceSpec{
				Ports: []apiv1.ServicePort{
					{
						Port:       80,
						TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080},
					},
				},
				Selector: map[string]string{
					"app": "audit",
				},
			},
		})
		framework.ExpectNoError(err, "failed to create proxy service")

		config, err = framework.LoadConfig()
		framework.ExpectNoError(err, "failed to load config")

		var podIP string
		// get pod ip
		err = wait.Poll(100*time.Millisecond, 10*time.Second, func() (done bool, err error) {
			p, err := f.ClientSet.CoreV1().Pods(namespace).Get("audit-proxy", metav1.GetOptions{})
			if errors.IsNotFound(err) {
				framework.Logf("waiting for audit-proxy pod to be present")
				return false, nil
			} else if err != nil {
				return false, err
			}
			podIP = p.Status.PodIP
			if podIP == "" {
				framework.Logf("waiting for audit-proxy pod IP to be ready")
				return false, nil
			}
			return true, nil
		})
		framework.ExpectNoError(err, "timed out waiting for audit-proxy pod to be ready")

		podURL := fmt.Sprintf("http://%s:8080", podIP)
		// create audit sink
		sink := auditregv1alpha1.AuditSink{
			ObjectMeta: metav1.ObjectMeta{
				Name: "test",
			},
			Spec: auditregv1alpha1.AuditSinkSpec{
				Policy: auditregv1alpha1.Policy{
					Level: auditregv1alpha1.LevelRequestResponse,
					Stages: []auditregv1alpha1.Stage{
						auditregv1alpha1.StageRequestReceived,
						auditregv1alpha1.StageResponseStarted,
						auditregv1alpha1.StageResponseComplete,
						auditregv1alpha1.StagePanic,
					},
				},
				Webhook: auditregv1alpha1.Webhook{
					ClientConfig: auditregv1alpha1.WebhookClientConfig{
						URL: &podURL,
					},
				},
			},
		}

		_, err = f.ClientSet.AuditregistrationV1alpha1().AuditSinks().Create(&sink)
		framework.ExpectNoError(err, "failed to create audit sink")
		framework.Logf("created audit sink")

		// check that we are receiving logs in the proxy
		err = wait.Poll(100*time.Millisecond, 10*time.Second, func() (done bool, err error) {
			logs, err := framework.GetPodLogs(f.ClientSet, namespace, "audit-proxy", "proxy")
			if err != nil {
				framework.Logf("waiting for audit-proxy pod logs to be available")
				return false, nil
			}
			if logs == "" {
				framework.Logf("waiting for audit-proxy pod logs to be non-empty")
				return false, nil
			}
			return true, nil
		})
		framework.ExpectNoError(err, "failed to get logs from audit-proxy pod")

		auditTestUser = "kubernetes-admin"
		testCases := []struct {
			action func()
			events []utils.AuditEvent
		}{
			// Create, get, update, patch, delete, list, watch pods.
			// TODO: dedup this with the main audit test once policy functionality is available
			// https://github.com/kubernetes/kubernetes/issues/70818
			{
				func() {
					pod := &apiv1.Pod{
						ObjectMeta: metav1.ObjectMeta{
							Name: "audit-pod",
						},
						Spec: apiv1.PodSpec{
							Containers: []apiv1.Container{{
								Name:  "pause",
								Image: imageutils.GetPauseImageName(),
							}},
						},
					}
					updatePod := func(pod *apiv1.Pod) {}

					f.PodClient().CreateSync(pod)

					_, err := f.PodClient().Get(pod.Name, metav1.GetOptions{})
					framework.ExpectNoError(err, "failed to get audit-pod")

					podChan, err := f.PodClient().Watch(watchOptions)
					framework.ExpectNoError(err, "failed to create watch for pods")
					for range podChan.ResultChan() {
					}

					f.PodClient().Update(pod.Name, updatePod)

					_, err = f.PodClient().List(metav1.ListOptions{})
					framework.ExpectNoError(err, "failed to list pods")

					_, err = f.PodClient().Patch(pod.Name, types.JSONPatchType, patch)
					framework.ExpectNoError(err, "failed to patch pod")

					f.PodClient().DeleteSync(pod.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
				},
				[]utils.AuditEvent{
					{
						Level:             auditinternal.LevelRequestResponse,
						Stage:             auditinternal.StageResponseComplete,
						RequestURI:        fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace),
						Verb:              "create",
						Code:              201,
						User:              auditTestUser,
						Resource:          "pods",
						Namespace:         namespace,
						RequestObject:     true,
						ResponseObject:    true,
						AuthorizeDecision: "allow",
					}, {
						Level:             auditinternal.LevelRequestResponse,
						Stage:             auditinternal.StageResponseComplete,
						RequestURI:        fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace),
						Verb:              "get",
						Code:              200,
						User:              auditTestUser,
						Resource:          "pods",
						Namespace:         namespace,
						RequestObject:     false,
						ResponseObject:    true,
						AuthorizeDecision: "allow",
					}, {
						Level:             auditinternal.LevelRequestResponse,
						Stage:             auditinternal.StageResponseComplete,
						RequestURI:        fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace),
						Verb:              "list",
						Code:              200,
						User:              auditTestUser,
						Resource:          "pods",
						Namespace:         namespace,
						RequestObject:     false,
						ResponseObject:    true,
						AuthorizeDecision: "allow",
					}, {
						Level:             auditinternal.LevelRequestResponse,
						Stage:             auditinternal.StageResponseStarted,
						RequestURI:        fmt.Sprintf("/api/v1/namespaces/%s/pods?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout),
						Verb:              "watch",
						Code:              200,
						User:              auditTestUser,
						Resource:          "pods",
						Namespace:         namespace,
						RequestObject:     false,
						ResponseObject:    false,
						AuthorizeDecision: "allow",
					}, {
						Level:             auditinternal.LevelRequestResponse,
						Stage:             auditinternal.StageResponseComplete,
						RequestURI:        fmt.Sprintf("/api/v1/namespaces/%s/pods?timeout=%ds&timeoutSeconds=%d&watch=true", namespace, watchTestTimeout, watchTestTimeout),
						Verb:              "watch",
						Code:              200,
						User:              auditTestUser,
						Resource:          "pods",
						Namespace:         namespace,
						RequestObject:     false,
						ResponseObject:    false,
						AuthorizeDecision: "allow",
					}, {
						Level:             auditinternal.LevelRequestResponse,
						Stage:             auditinternal.StageResponseComplete,
						RequestURI:        fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace),
						Verb:              "update",
						Code:              200,
						User:              auditTestUser,
						Resource:          "pods",
						Namespace:         namespace,
						RequestObject:     true,
						ResponseObject:    true,
						AuthorizeDecision: "allow",
					}, {
						Level:             auditinternal.LevelRequestResponse,
						Stage:             auditinternal.StageResponseComplete,
						RequestURI:        fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace),
						Verb:              "patch",
						Code:              200,
						User:              auditTestUser,
						Resource:          "pods",
						Namespace:         namespace,
						RequestObject:     true,
						ResponseObject:    true,
						AuthorizeDecision: "allow",
					}, {
						Level:             auditinternal.LevelRequestResponse,
						Stage:             auditinternal.StageResponseComplete,
						RequestURI:        fmt.Sprintf("/api/v1/namespaces/%s/pods/audit-pod", namespace),
						Verb:              "delete",
						Code:              200,
						User:              auditTestUser,
						Resource:          "pods",
						Namespace:         namespace,
						RequestObject:     true,
						ResponseObject:    true,
						AuthorizeDecision: "allow",
					},
				},
			},
		}

		// test authorizer annotations, RBAC is required.
		annotationTestCases := []struct {
			action func()
			events []utils.AuditEvent
		}{

			// get a pod with unauthorized user
			{
				func() {
					_, err := anonymousClient.CoreV1().Pods(namespace).Get("another-audit-pod", metav1.GetOptions{})
					expectForbidden(err)
				},
				[]utils.AuditEvent{
					{
						Level:              auditinternal.LevelRequestResponse,
						Stage:              auditinternal.StageResponseComplete,
						RequestURI:         fmt.Sprintf("/api/v1/namespaces/%s/pods/another-audit-pod", namespace),
						Verb:               "get",
						Code:               403,
						User:               auditTestUser,
						ImpersonatedUser:   "system:anonymous",
						ImpersonatedGroups: "system:unauthenticated",
						Resource:           "pods",
						Namespace:          namespace,
						RequestObject:      false,
						ResponseObject:     true,
						AuthorizeDecision:  "forbid",
					},
				},
			},
		}

		if framework.IsRBACEnabled(f) {
			testCases = append(testCases, annotationTestCases...)
		}
		expectedEvents := []utils.AuditEvent{}
		for _, t := range testCases {
			t.action()
			expectedEvents = append(expectedEvents, t.events...)
		}

		// The default flush timeout is 30 seconds, therefore it should be enough to retry once
		// to find all expected events. However, we're waiting for 5 minutes to avoid flakes.
		pollingInterval := 30 * time.Second
		pollingTimeout := 5 * time.Minute
		err = wait.Poll(pollingInterval, pollingTimeout, func() (bool, error) {
			// Fetch the logs
			logs, err := framework.GetPodLogs(f.ClientSet, namespace, "audit-proxy", "proxy")
			if err != nil {
				return false, err
			}
			reader := strings.NewReader(logs)
			missing, err := utils.CheckAuditLines(reader, expectedEvents, auditv1.SchemeGroupVersion)
			if err != nil {
				framework.Logf("Failed to observe audit events: %v", err)
			} else if len(missing) > 0 {
				framework.Logf("Events %#v not found!", missing)
			}
			return len(missing) == 0, nil
		})
		framework.ExpectNoError(err, "after %v failed to observe audit events", pollingTimeout)
		err = f.ClientSet.AuditregistrationV1alpha1().AuditSinks().Delete("test", &metav1.DeleteOptions{})
		framework.ExpectNoError(err, "could not delete audit configuration")
	})
})
<|endoftext|>


<commit_before>/*
   Copyright 2014 Outbrain Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package logic

import (
	"os"
	"os/signal"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/github/orchestrator/go/agent"
	"github.com/github/orchestrator/go/collection"
	"github.com/github/orchestrator/go/config"
	"github.com/github/orchestrator/go/discovery"
	"github.com/github/orchestrator/go/inst"
	ometrics "github.com/github/orchestrator/go/metrics"
	"github.com/github/orchestrator/go/process"
	"github.com/github/orchestrator/go/raft"
	"github.com/openark/golib/log"
	"github.com/patrickmn/go-cache"
	"github.com/rcrowley/go-metrics"
	"github.com/sjmudd/stopwatch"
)

const discoveryMetricsName = "DISCOVERY_METRICS"

// discoveryQueue is a channel of deduplicated instanceKey-s
// that were requested for discovery.  It can be continuously updated
// as discovery process progresses.
var discoveryQueue *discovery.Queue

var discoveriesCounter = metrics.NewCounter()
var failedDiscoveriesCounter = metrics.NewCounter()
var instancePollSecondsExceededCounter = metrics.NewCounter()
var discoveryQueueLengthGauge = metrics.NewGauge()
var discoveryRecentCountGauge = metrics.NewGauge()
var isElectedGauge = metrics.NewGauge()
var discoveryMetrics = collection.CreateOrReturnCollection(discoveryMetricsName)

var isElectedNode int64 = 0

var recentDiscoveryOperationKeys *cache.Cache

func init() {
	metrics.Register("discoveries.attempt", discoveriesCounter)
	metrics.Register("discoveries.fail", failedDiscoveriesCounter)
	metrics.Register("discoveries.instance_poll_seconds_exceeded", instancePollSecondsExceededCounter)
	metrics.Register("discoveries.queue_length", discoveryQueueLengthGauge)
	metrics.Register("discoveries.recent_count", discoveryRecentCountGauge)
	metrics.Register("elect.is_elected", isElectedGauge)

	ometrics.OnGraphiteTick(func() { discoveryQueueLengthGauge.Update(int64(discoveryQueue.QueueLen())) })
	ometrics.OnGraphiteTick(func() {
		if recentDiscoveryOperationKeys == nil {
			return
		}
		discoveryRecentCountGauge.Update(int64(recentDiscoveryOperationKeys.ItemCount()))
	})
	ometrics.OnGraphiteTick(func() { isElectedGauge.Update(int64(atomic.LoadInt64(&isElectedNode))) })
}

func IsLeader() bool {
	if orcraft.IsRaftEnabled() {
		return orcraft.IsLeader()
	}
	return atomic.LoadInt64(&isElectedNode) == 1
}

// used in several places
func instancePollSecondsDuration() time.Duration {
	return time.Duration(config.Config.InstancePollSeconds) * time.Second
}

// acceptSignals registers for OS signals
func acceptSignals() {
	c := make(chan os.Signal, 1)

	signal.Notify(c, syscall.SIGHUP)
	signal.Notify(c, syscall.SIGTERM)
	go func() {
		for sig := range c {
			switch sig {
			case syscall.SIGHUP:
				log.Infof("Received SIGHUP. Reloading configuration")
				inst.AuditOperation("reload-configuration", nil, "Triggered via SIGHUP")
				config.Reload()
				discoveryMetrics.SetExpirePeriod(time.Duration(config.Config.DiscoveryCollectionRetentionSeconds) * time.Second)
			case syscall.SIGTERM:
				log.Infof("Received SIGTERM. Shutting down orchestrator")
				discoveryMetrics.StopAutoExpiration()
				// probably should poke other go routines to stop cleanly here ...
				inst.AuditOperation("shutdown", nil, "Triggered via SIGTERM")
				os.Exit(0)
			}
		}
	}()
}

// handleDiscoveryRequests iterates the discoveryQueue channel and calls upon
// instance discovery per entry.
func handleDiscoveryRequests() {
	discoveryQueue = discovery.CreateOrReturnQueue("DEFAULT")

	// create a pool of discovery workers
	for i := uint(0); i < config.Config.DiscoveryMaxConcurrency; i++ {
		go func() {
			for {
				instanceKey := discoveryQueue.Consume()
				// Possibly this used to be the elected node, but has
				// been demoted, while still the queue is full.
				if !IsLeader() {
					log.Debugf("Node apparently demoted. Skipping discovery of %+v. "+
						"Remaining queue size: %+v", instanceKey, discoveryQueue.QueueLen())

					discoveryQueue.Release(instanceKey)

					continue
				}

				discoverInstance(instanceKey)

				discoveryQueue.Release(instanceKey)
			}
		}()
	}
}

// discoverInstance will attempt to discover (poll) an instance (unless
// it is already up to date) and will also ensure that its master and
// replicas (if any) are also checked.
func discoverInstance(instanceKey inst.InstanceKey) {
	// create stopwatch entries
	latency := stopwatch.NewNamedStopwatch()
	latency.AddMany([]string{
		"backend",
		"instance",
		"total"})
	latency.Start("total") // start the total stopwatch (not changed anywhere else)

	defer func() {
		latency.Stop("total")
		discoveryTime := latency.Elapsed("total")
		if discoveryTime > instancePollSecondsDuration() {
			instancePollSecondsExceededCounter.Inc(1)
			log.Warningf("discoverInstance exceeded InstancePollSeconds for %+v, took %.4fs", instanceKey, discoveryTime.Seconds())
		}
	}()

	instanceKey.Formalize()
	if !instanceKey.IsValid() {
		return
	}

	// Calculate the expiry period each time as InstancePollSeconds
	// _may_ change during the run of the process (via SIGHUP) and
	// it is not possible to change the cache's default expiry..
	if existsInCacheError := recentDiscoveryOperationKeys.Add(instanceKey.DisplayString(), true, instancePollSecondsDuration()); existsInCacheError != nil {
		// Just recently attempted
		return
	}

	latency.Start("backend")
	instance, found, err := inst.ReadInstance(&instanceKey)
	latency.Stop("backend")
	if found && instance.IsUpToDate && instance.IsLastCheckValid {
		// we've already discovered this one. Skip!
		return
	}

	discoveriesCounter.Inc(1)

	// First we've ever heard of this instance. Continue investigation:
	instance, err = inst.ReadTopologyInstanceBufferable(&instanceKey, config.Config.BufferInstanceWrites, latency)
	// panic can occur (IO stuff). Therefore it may happen
	// that instance is nil. Check it, but first get the timing metrics.
	totalLatency := latency.Elapsed("total")
	backendLatency := latency.Elapsed("backend")
	instanceLatency := latency.Elapsed("instance")

	if instance == nil {
		failedDiscoveriesCounter.Inc(1)
		discoveryMetrics.Append(&discovery.Metric{
			Timestamp:       time.Now(),
			InstanceKey:     instanceKey,
			TotalLatency:    totalLatency,
			BackendLatency:  backendLatency,
			InstanceLatency: instanceLatency,
			Err:             err,
		})
		log.Warningf("discoverInstance(%+v) instance is nil in %.3fs (Backend: %.3fs, Instance: %.3fs), error=%+v",
			instanceKey,
			totalLatency.Seconds(),
			backendLatency.Seconds(),
			instanceLatency.Seconds(),
			err)
		return
	}

	discoveryMetrics.Append(&discovery.Metric{
		Timestamp:       time.Now(),
		InstanceKey:     instanceKey,
		TotalLatency:    totalLatency,
		BackendLatency:  backendLatency,
		InstanceLatency: instanceLatency,
		Err:             nil,
	})
	log.Debugf("Discovered host: %+v, master: %+v, version: %+v in %.3fs (Backend: %.3fs, Instance: %.3fs)",
		instance.Key,
		instance.MasterKey,
		instance.Version,
		totalLatency.Seconds(),
		backendLatency.Seconds(),
		instanceLatency.Seconds())

	if !IsLeader() {
		// Maybe this node was elected before, but isn't elected anymore.
		// If not elected, stop drilling up/down the topology
		return
	}

	// Investigate replicas:
	for _, replicaKey := range instance.SlaveHosts.GetInstanceKeys() {
		replicaKey := replicaKey // not needed? no concurrency here?

		// Avoid noticing some hosts we would otherwise discover
		if inst.RegexpMatchPatterns(replicaKey.Hostname, config.Config.DiscoveryIgnoreReplicaHostnameFilters) {
			continue
		}

		if replicaKey.IsValid() {
			discoveryQueue.Push(replicaKey)
		}
	}
	// Investigate master:
	if instance.MasterKey.IsValid() {
		discoveryQueue.Push(instance.MasterKey)
	}
}

// onDiscoveryTick handles the actions to take to discover/poll instances
func onDiscoveryTick() {
	wasAlreadyElected := IsLeader()
	myIsElectedNode, err := process.AttemptElection()
	if err != nil {
		log.Errore(err)
	}
	if myIsElectedNode {
		atomic.StoreInt64(&isElectedNode, 1)
	} else {
		atomic.StoreInt64(&isElectedNode, 0)
	}

	if !myIsElectedNode {
		if electedNode, _, err := process.ElectedNode(); err == nil {
			log.Infof("Not elected as active node; active node: %v; polling", electedNode.Hostname)
		} else {
			log.Infof("Not elected as active node; active node: Unable to determine: %v; polling", err)
		}
		return
	}

	// I'm elected!
	instanceKeys, err := inst.ReadOutdatedInstanceKeys()
	if err != nil {
		log.Errore(err)
	}

	if !wasAlreadyElected {
		// Just turned to be leader!
		go process.RegisterNode("", "", false)
	}

	// avoid any logging unless there's something to be done
	if len(instanceKeys) > 0 {
		if len(instanceKeys) > config.Config.MaxOutdatedKeysToShow {
			log.Debugf("polling %d outdated keys", len(instanceKeys))
		} else {
			log.Debugf("outdated keys: %+v", instanceKeys)
		}
		for _, instanceKey := range instanceKeys {
			instanceKey := instanceKey

			if instanceKey.IsValid() {
				discoveryQueue.Push(instanceKey)
			}
		}
	}
}

// ContinuousDiscovery starts an asynchronuous infinite discovery process where instances are
// periodically investigated and their status captured, and long since unseen instances are
// purged and forgotten.
func ContinuousDiscovery() {
	log.Infof("Starting continuous discovery")
	recentDiscoveryOperationKeys = cache.New(instancePollSecondsDuration(), time.Second)

	inst.LoadHostnameResolveCache()
	go handleDiscoveryRequests()

	discoveryTick := time.Tick(config.DiscoveryPollSeconds * time.Second)
	instancePollTick := time.Tick(instancePollSecondsDuration())
	caretakingTick := time.Tick(time.Minute)
	recoveryTick := time.Tick(time.Duration(config.Config.RecoveryPollSeconds) * time.Second)
	var snapshotTopologiesTick <-chan time.Time
	if config.Config.SnapshotTopologiesIntervalHours > 0 {
		snapshotTopologiesTick = time.Tick(time.Duration(config.Config.SnapshotTopologiesIntervalHours) * time.Hour)
	}

	go ometrics.InitGraphiteMetrics()
	go acceptSignals()

	if *config.RuntimeCLIFlags.GrabElection {
		process.GrabElection()
	}
	for {
		select {
		case <-discoveryTick:
			go func() {
				onDiscoveryTick()
			}()
		case <-instancePollTick:
			go func() {
				// This tick does NOT do instance poll (these are handled by the oversampling discoveryTick)
				// But rather should invoke such routinely operations that need to be as (or roughly as) frequent
				// as instance poll
				if IsLeader() {
					go inst.RecordInstanceCoordinatesHistory()
					go inst.UpdateClusterAliases()
					go inst.ExpireDowntime()
				}
			}()
		case <-caretakingTick:
			// Various periodic internal maintenance tasks
			go func() {
				if IsLeader() {
					go inst.RecordInstanceBinlogFileHistory()
					go inst.ForgetLongUnseenInstances()
					go inst.ForgetUnseenInstancesDifferentlyResolved()
					go inst.ForgetExpiredHostnameResolves()
					go inst.DeleteInvalidHostnameResolves()
					go inst.ReviewUnseenInstances()
					go inst.InjectUnseenMasters()
					go inst.ResolveUnknownMasterHostnameResolves()
					go inst.ExpireMaintenance()
					go inst.ExpireCandidateInstances()
					go inst.ExpireHostnameUnresolve()
					go inst.ExpireClusterDomainName()
					go inst.ExpireAudit()
					go inst.ExpireMasterPositionEquivalence()
					go inst.ExpirePoolInstances()
					go inst.FlushNontrivialResolveCacheToDatabase()
					go process.ExpireNodesHistory()
					go process.ExpireAccessTokens()
					go process.ExpireAvailableNodes()
				} else {
					// Take this opportunity to refresh yourself
					go inst.LoadHostnameResolveCache()
				}
			}()
		case <-recoveryTick:
			go func() {
				if IsLeader() {
					go ClearActiveFailureDetections()
					go ClearActiveRecoveries()
					go ExpireBlockedRecoveries()
					go AcknowledgeCrashedRecoveries()
					go inst.ExpireInstanceAnalysisChangelog()
					go CheckAndRecover(nil, nil, false)
				}
			}()
		case <-snapshotTopologiesTick:
			go func() {
				go inst.SnapshotTopologies()
			}()
		}
	}
}

func pollAgent(hostname string) error {
	polledAgent, err := agent.GetAgent(hostname)
	agent.UpdateAgentLastChecked(hostname)

	if err != nil {
		return log.Errore(err)
	}

	err = agent.UpdateAgentInfo(hostname, polledAgent)
	if err != nil {
		return log.Errore(err)
	}

	return nil
}

// ContinuousAgentsPoll starts an asynchronuous infinite process where agents are
// periodically investigated and their status captured, and long since unseen agents are
// purged and forgotten.
func ContinuousAgentsPoll() {
	log.Infof("Starting continuous agents poll")

	go discoverSeededAgents()

	tick := time.Tick(config.DiscoveryPollSeconds * time.Second)
	caretakingTick := time.Tick(time.Hour)
	for range tick {
		agentsHosts, _ := agent.ReadOutdatedAgentsHosts()
		log.Debugf("outdated agents hosts: %+v", agentsHosts)
		for _, hostname := range agentsHosts {
			go pollAgent(hostname)
		}
		// See if we should also forget agents (lower frequency)
		select {
		case <-caretakingTick:
			agent.ForgetLongUnseenAgents()
			agent.FailStaleSeeds()
		default:
		}
	}
}

func discoverSeededAgents() {
	for seededAgent := range agent.SeededAgents {
		instanceKey := &inst.InstanceKey{Hostname: seededAgent.Hostname, Port: int(seededAgent.MySQLPort)}
		go inst.ReadTopologyInstance(instanceKey)
	}
}
<commit_msg>raft nodes get to run cleanups and everything; just not recoveries<commit_after>/*
   Copyright 2014 Outbrain Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package logic

import (
	"os"
	"os/signal"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/github/orchestrator/go/agent"
	"github.com/github/orchestrator/go/collection"
	"github.com/github/orchestrator/go/config"
	"github.com/github/orchestrator/go/discovery"
	"github.com/github/orchestrator/go/inst"
	ometrics "github.com/github/orchestrator/go/metrics"
	"github.com/github/orchestrator/go/process"
	"github.com/github/orchestrator/go/raft"
	"github.com/openark/golib/log"
	"github.com/patrickmn/go-cache"
	"github.com/rcrowley/go-metrics"
	"github.com/sjmudd/stopwatch"
)

const discoveryMetricsName = "DISCOVERY_METRICS"

// discoveryQueue is a channel of deduplicated instanceKey-s
// that were requested for discovery.  It can be continuously updated
// as discovery process progresses.
var discoveryQueue *discovery.Queue

var discoveriesCounter = metrics.NewCounter()
var failedDiscoveriesCounter = metrics.NewCounter()
var instancePollSecondsExceededCounter = metrics.NewCounter()
var discoveryQueueLengthGauge = metrics.NewGauge()
var discoveryRecentCountGauge = metrics.NewGauge()
var isElectedGauge = metrics.NewGauge()
var discoveryMetrics = collection.CreateOrReturnCollection(discoveryMetricsName)

var isElectedNode int64 = 0

var recentDiscoveryOperationKeys *cache.Cache

func init() {
	metrics.Register("discoveries.attempt", discoveriesCounter)
	metrics.Register("discoveries.fail", failedDiscoveriesCounter)
	metrics.Register("discoveries.instance_poll_seconds_exceeded", instancePollSecondsExceededCounter)
	metrics.Register("discoveries.queue_length", discoveryQueueLengthGauge)
	metrics.Register("discoveries.recent_count", discoveryRecentCountGauge)
	metrics.Register("elect.is_elected", isElectedGauge)

	ometrics.OnGraphiteTick(func() { discoveryQueueLengthGauge.Update(int64(discoveryQueue.QueueLen())) })
	ometrics.OnGraphiteTick(func() {
		if recentDiscoveryOperationKeys == nil {
			return
		}
		discoveryRecentCountGauge.Update(int64(recentDiscoveryOperationKeys.ItemCount()))
	})
	ometrics.OnGraphiteTick(func() { isElectedGauge.Update(int64(atomic.LoadInt64(&isElectedNode))) })
}

func IsLeader() bool {
	if orcraft.IsRaftEnabled() {
		return orcraft.IsLeader()
	}
	return atomic.LoadInt64(&isElectedNode) == 1
}

func IsLeaderOrActive() bool {
	if orcraft.IsRaftEnabled() {
		return true
	}
	return atomic.LoadInt64(&isElectedNode) == 1
}

// used in several places
func instancePollSecondsDuration() time.Duration {
	return time.Duration(config.Config.InstancePollSeconds) * time.Second
}

// acceptSignals registers for OS signals
func acceptSignals() {
	c := make(chan os.Signal, 1)

	signal.Notify(c, syscall.SIGHUP)
	signal.Notify(c, syscall.SIGTERM)
	go func() {
		for sig := range c {
			switch sig {
			case syscall.SIGHUP:
				log.Infof("Received SIGHUP. Reloading configuration")
				inst.AuditOperation("reload-configuration", nil, "Triggered via SIGHUP")
				config.Reload()
				discoveryMetrics.SetExpirePeriod(time.Duration(config.Config.DiscoveryCollectionRetentionSeconds) * time.Second)
			case syscall.SIGTERM:
				log.Infof("Received SIGTERM. Shutting down orchestrator")
				discoveryMetrics.StopAutoExpiration()
				// probably should poke other go routines to stop cleanly here ...
				inst.AuditOperation("shutdown", nil, "Triggered via SIGTERM")
				os.Exit(0)
			}
		}
	}()
}

// handleDiscoveryRequests iterates the discoveryQueue channel and calls upon
// instance discovery per entry.
func handleDiscoveryRequests() {
	discoveryQueue = discovery.CreateOrReturnQueue("DEFAULT")

	// create a pool of discovery workers
	for i := uint(0); i < config.Config.DiscoveryMaxConcurrency; i++ {
		go func() {
			for {
				instanceKey := discoveryQueue.Consume()
				// Possibly this used to be the elected node, but has
				// been demoted, while still the queue is full.
				if !IsLeaderOrActive() {
					log.Debugf("Node apparently demoted. Skipping discovery of %+v. "+
						"Remaining queue size: %+v", instanceKey, discoveryQueue.QueueLen())

					discoveryQueue.Release(instanceKey)

					continue
				}

				discoverInstance(instanceKey)

				discoveryQueue.Release(instanceKey)
			}
		}()
	}
}

// discoverInstance will attempt to discover (poll) an instance (unless
// it is already up to date) and will also ensure that its master and
// replicas (if any) are also checked.
func discoverInstance(instanceKey inst.InstanceKey) {
	// create stopwatch entries
	latency := stopwatch.NewNamedStopwatch()
	latency.AddMany([]string{
		"backend",
		"instance",
		"total"})
	latency.Start("total") // start the total stopwatch (not changed anywhere else)

	defer func() {
		latency.Stop("total")
		discoveryTime := latency.Elapsed("total")
		if discoveryTime > instancePollSecondsDuration() {
			instancePollSecondsExceededCounter.Inc(1)
			log.Warningf("discoverInstance exceeded InstancePollSeconds for %+v, took %.4fs", instanceKey, discoveryTime.Seconds())
		}
	}()

	instanceKey.Formalize()
	if !instanceKey.IsValid() {
		return
	}

	// Calculate the expiry period each time as InstancePollSeconds
	// _may_ change during the run of the process (via SIGHUP) and
	// it is not possible to change the cache's default expiry..
	if existsInCacheError := recentDiscoveryOperationKeys.Add(instanceKey.DisplayString(), true, instancePollSecondsDuration()); existsInCacheError != nil {
		// Just recently attempted
		return
	}

	latency.Start("backend")
	instance, found, err := inst.ReadInstance(&instanceKey)
	latency.Stop("backend")
	if found && instance.IsUpToDate && instance.IsLastCheckValid {
		// we've already discovered this one. Skip!
		return
	}

	discoveriesCounter.Inc(1)

	// First we've ever heard of this instance. Continue investigation:
	instance, err = inst.ReadTopologyInstanceBufferable(&instanceKey, config.Config.BufferInstanceWrites, latency)
	// panic can occur (IO stuff). Therefore it may happen
	// that instance is nil. Check it, but first get the timing metrics.
	totalLatency := latency.Elapsed("total")
	backendLatency := latency.Elapsed("backend")
	instanceLatency := latency.Elapsed("instance")

	if instance == nil {
		failedDiscoveriesCounter.Inc(1)
		discoveryMetrics.Append(&discovery.Metric{
			Timestamp:       time.Now(),
			InstanceKey:     instanceKey,
			TotalLatency:    totalLatency,
			BackendLatency:  backendLatency,
			InstanceLatency: instanceLatency,
			Err:             err,
		})
		log.Warningf("discoverInstance(%+v) instance is nil in %.3fs (Backend: %.3fs, Instance: %.3fs), error=%+v",
			instanceKey,
			totalLatency.Seconds(),
			backendLatency.Seconds(),
			instanceLatency.Seconds(),
			err)
		return
	}

	discoveryMetrics.Append(&discovery.Metric{
		Timestamp:       time.Now(),
		InstanceKey:     instanceKey,
		TotalLatency:    totalLatency,
		BackendLatency:  backendLatency,
		InstanceLatency: instanceLatency,
		Err:             nil,
	})
	log.Debugf("Discovered host: %+v, master: %+v, version: %+v in %.3fs (Backend: %.3fs, Instance: %.3fs)",
		instance.Key,
		instance.MasterKey,
		instance.Version,
		totalLatency.Seconds(),
		backendLatency.Seconds(),
		instanceLatency.Seconds())

	if !IsLeaderOrActive() {
		// Maybe this node was elected before, but isn't elected anymore.
		// If not elected, stop drilling up/down the topology
		return
	}

	// Investigate replicas:
	for _, replicaKey := range instance.SlaveHosts.GetInstanceKeys() {
		replicaKey := replicaKey // not needed? no concurrency here?

		// Avoid noticing some hosts we would otherwise discover
		if inst.RegexpMatchPatterns(replicaKey.Hostname, config.Config.DiscoveryIgnoreReplicaHostnameFilters) {
			continue
		}

		if replicaKey.IsValid() {
			discoveryQueue.Push(replicaKey)
		}
	}
	// Investigate master:
	if instance.MasterKey.IsValid() {
		discoveryQueue.Push(instance.MasterKey)
	}
}

// onDiscoveryTick handles the actions to take to discover/poll instances
func onDiscoveryTick() {
	wasAlreadyElected := IsLeader()
	myIsElectedNode, err := process.AttemptElection()
	if err != nil {
		log.Errore(err)
	}
	if myIsElectedNode {
		atomic.StoreInt64(&isElectedNode, 1)
	} else {
		atomic.StoreInt64(&isElectedNode, 0)
	}

	if !myIsElectedNode {
		if electedNode, _, err := process.ElectedNode(); err == nil {
			log.Infof("Not elected as active node; active node: %v; polling", electedNode.Hostname)
		} else {
			log.Infof("Not elected as active node; active node: Unable to determine: %v; polling", err)
		}
		return
	}

	// I'm elected!
	instanceKeys, err := inst.ReadOutdatedInstanceKeys()
	if err != nil {
		log.Errore(err)
	}

	if !wasAlreadyElected {
		// Just turned to be leader!
		go process.RegisterNode("", "", false)
	}

	// avoid any logging unless there's something to be done
	if len(instanceKeys) > 0 {
		if len(instanceKeys) > config.Config.MaxOutdatedKeysToShow {
			log.Debugf("polling %d outdated keys", len(instanceKeys))
		} else {
			log.Debugf("outdated keys: %+v", instanceKeys)
		}
		for _, instanceKey := range instanceKeys {
			instanceKey := instanceKey

			if instanceKey.IsValid() {
				discoveryQueue.Push(instanceKey)
			}
		}
	}
}

// ContinuousDiscovery starts an asynchronuous infinite discovery process where instances are
// periodically investigated and their status captured, and long since unseen instances are
// purged and forgotten.
func ContinuousDiscovery() {
	log.Infof("Starting continuous discovery")
	recentDiscoveryOperationKeys = cache.New(instancePollSecondsDuration(), time.Second)

	inst.LoadHostnameResolveCache()
	go handleDiscoveryRequests()

	discoveryTick := time.Tick(config.DiscoveryPollSeconds * time.Second)
	instancePollTick := time.Tick(instancePollSecondsDuration())
	caretakingTick := time.Tick(time.Minute)
	recoveryTick := time.Tick(time.Duration(config.Config.RecoveryPollSeconds) * time.Second)
	var snapshotTopologiesTick <-chan time.Time
	if config.Config.SnapshotTopologiesIntervalHours > 0 {
		snapshotTopologiesTick = time.Tick(time.Duration(config.Config.SnapshotTopologiesIntervalHours) * time.Hour)
	}

	go ometrics.InitGraphiteMetrics()
	go acceptSignals()

	if *config.RuntimeCLIFlags.GrabElection {
		process.GrabElection()
	}
	for {
		select {
		case <-discoveryTick:
			go func() {
				onDiscoveryTick()
			}()
		case <-instancePollTick:
			go func() {
				// This tick does NOT do instance poll (these are handled by the oversampling discoveryTick)
				// But rather should invoke such routinely operations that need to be as (or roughly as) frequent
				// as instance poll
				if IsLeaderOrActive() {
					go inst.RecordInstanceCoordinatesHistory()
					go inst.UpdateClusterAliases()
					go inst.ExpireDowntime()
				}
			}()
		case <-caretakingTick:
			// Various periodic internal maintenance tasks
			go func() {
				if IsLeaderOrActive() {
					go inst.RecordInstanceBinlogFileHistory()
					go inst.ForgetLongUnseenInstances()
					go inst.ForgetUnseenInstancesDifferentlyResolved()
					go inst.ForgetExpiredHostnameResolves()
					go inst.DeleteInvalidHostnameResolves()
					go inst.ReviewUnseenInstances()
					go inst.InjectUnseenMasters()
					go inst.ResolveUnknownMasterHostnameResolves()
					go inst.ExpireMaintenance()
					go inst.ExpireCandidateInstances()
					go inst.ExpireHostnameUnresolve()
					go inst.ExpireClusterDomainName()
					go inst.ExpireAudit()
					go inst.ExpireMasterPositionEquivalence()
					go inst.ExpirePoolInstances()
					go inst.FlushNontrivialResolveCacheToDatabase()
					go process.ExpireNodesHistory()
					go process.ExpireAccessTokens()
					go process.ExpireAvailableNodes()
				} else {
					// Take this opportunity to refresh yourself
					go inst.LoadHostnameResolveCache()
				}
			}()
		case <-recoveryTick:
			go func() {
				if IsLeaderOrActive() {
					go ClearActiveFailureDetections()
					go ClearActiveRecoveries()
					go ExpireBlockedRecoveries()
					go AcknowledgeCrashedRecoveries()
					go inst.ExpireInstanceAnalysisChangelog()
				}
				if IsLeader() {
					go CheckAndRecover(nil, nil, false)
				}
			}()
		case <-snapshotTopologiesTick:
			go func() {
				go inst.SnapshotTopologies()
			}()
		}
	}
}

func pollAgent(hostname string) error {
	polledAgent, err := agent.GetAgent(hostname)
	agent.UpdateAgentLastChecked(hostname)

	if err != nil {
		return log.Errore(err)
	}

	err = agent.UpdateAgentInfo(hostname, polledAgent)
	if err != nil {
		return log.Errore(err)
	}

	return nil
}

// ContinuousAgentsPoll starts an asynchronuous infinite process where agents are
// periodically investigated and their status captured, and long since unseen agents are
// purged and forgotten.
func ContinuousAgentsPoll() {
	log.Infof("Starting continuous agents poll")

	go discoverSeededAgents()

	tick := time.Tick(config.DiscoveryPollSeconds * time.Second)
	caretakingTick := time.Tick(time.Hour)
	for range tick {
		agentsHosts, _ := agent.ReadOutdatedAgentsHosts()
		log.Debugf("outdated agents hosts: %+v", agentsHosts)
		for _, hostname := range agentsHosts {
			go pollAgent(hostname)
		}
		// See if we should also forget agents (lower frequency)
		select {
		case <-caretakingTick:
			agent.ForgetLongUnseenAgents()
			agent.FailStaleSeeds()
		default:
		}
	}
}

func discoverSeededAgents() {
	for seededAgent := range agent.SeededAgents {
		instanceKey := &inst.InstanceKey{Hostname: seededAgent.Hostname, Port: int(seededAgent.MySQLPort)}
		go inst.ReadTopologyInstance(instanceKey)
	}
}
<|endoftext|>


<commit_before>package paths

import (
	"strings"
)

// Path represents an endpoint added to a bogus server and how it should respond
type Path struct {
	Payload []byte
	Hits    int
	Status  int
	methods []string
}

// New returns a newly instantiated path object with everything initialized as
// needed.
func New() *Path {
	return &Path{}
}

// SetPayload sets the response payload for the path and returns the path for
// additional configuration
func (p *Path) SetPayload(payload []byte) *Path {
	p.Payload = payload
	return p
}

// SetStatus sets the http status for the path and returns the path for
// additional configuration
func (p *Path) SetStatus(status int) *Path {
	p.Status = status
	return p
}

// SetMethods accepts a list of methods the path should respond to
func (p *Path) SetMethods(methods ...string) *Path {
	for i, m := range methods {
		methods[i] = strings.ToUpper(m)
	}

	p.methods = methods
	return p
}

func (p *Path) HasMethod(method string) bool {
	method = strings.ToUpper(method)

	// Check for configured methods first
	if len(p.methods) != 0 {
		for _, m := range p.methods {
			if m == method {
				return true
			}
		}

		return false
	}

	// If no methods configured, handle GET by default for convenience
	if method == "GET" {
		return true
	}

	return false
}
<commit_msg>adding missing doc<commit_after>package paths

import (
	"strings"
)

// Path represents an endpoint added to a bogus server and how it should respond
type Path struct {
	Payload []byte
	Hits    int
	Status  int
	methods []string
}

// New returns a newly instantiated path object with everything initialized as
// needed.
func New() *Path {
	return &Path{}
}

// SetPayload sets the response payload for the path and returns the path for
// additional configuration
func (p *Path) SetPayload(payload []byte) *Path {
	p.Payload = payload
	return p
}

// SetStatus sets the http status for the path and returns the path for
// additional configuration
func (p *Path) SetStatus(status int) *Path {
	p.Status = status
	return p
}

// SetMethods accepts a list of methods the path should respond to
func (p *Path) SetMethods(methods ...string) *Path {
	for i, m := range methods {
		methods[i] = strings.ToUpper(m)
	}

	p.methods = methods
	return p
}

// HasMethod returns true or false based on whether a path resonds to a given
// method
func (p *Path) HasMethod(method string) bool {
	method = strings.ToUpper(method)

	// Check for configured methods first
	if len(p.methods) != 0 {
		for _, m := range p.methods {
			if m == method {
				return true
			}
		}

		return false
	}

	// If no methods configured, handle GET by default for convenience
	if method == "GET" {
		return true
	}

	return false
}
<|endoftext|>