<commit_before>// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"math/rand"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/ethereum/go-ethereum/rpc"
)

// Tests that a node embedded within a console can be started up properly and
// then terminated by closing the input stream.
func TestConsoleWelcome(t *testing.T) {
	coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182"

	// Start a geth console, make sure it's cleaned up and terminate the console
	geth := runGeth(t,
		"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none",
		"--etherbase", coinbase, "--shh",
		"console")

	// Gather all the infos the welcome message needs to contain
	geth.setTemplateFunc("goos", func() string { return runtime.GOOS })
	geth.setTemplateFunc("gover", runtime.Version)
	geth.setTemplateFunc("gethver", func() string { return verString })
	geth.setTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
	geth.setTemplateFunc("apis", func() []string {
		apis := append(strings.Split(rpc.DefaultIPCApis, ","), rpc.MetadataApi)
		sort.Strings(apis)
		return apis
	})

	// Verify the actual welcome message to the required template
	geth.expect(`
Welcome to the Geth JavaScript console!

instance: Geth/v{{gethver}}/{{goos}}/{{gover}}
coinbase: {{.Etherbase}}
at block: 0 ({{niltime}})
 datadir: {{.Datadir}}
 modules:{{range apis}} {{.}}:1.0{{end}}

> {{.InputLine "exit"}}
`)
	geth.expectExit()
}

// Tests that a console can be attached to a running node via various means.
func TestIPCAttachWelcome(t *testing.T) {
	// Configure the instance for IPC attachement
	coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182"
	var ipc string
	if runtime.GOOS == "windows" {
		ipc = `\\.\pipe\geth` + strconv.Itoa(rand.Int())
	} else {
		ws := tmpdir(t)
		defer os.RemoveAll(ws)
		ipc = filepath.Join(ws, "geth.ipc")
	}
	// Note: we need --shh because testAttachWelcome checks for default
	// list of ipc modules and shh is included there.
	geth := runGeth(t,
		"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none",
		"--etherbase", coinbase, "--shh", "--ipcpath", ipc)

	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
	testAttachWelcome(t, geth, "ipc:"+ipc)

	geth.interrupt()
	geth.expectExit()
}

func TestHTTPAttachWelcome(t *testing.T) {
	coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182"
	port := strconv.Itoa(rand.Intn(65535-1024) + 1024) // Yeah, sometimes this will fail, sorry :P
	geth := runGeth(t,
		"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none",
		"--etherbase", coinbase, "--rpc", "--rpcport", port)

	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
	testAttachWelcome(t, geth, "http://localhost:"+port)

	geth.interrupt()
	geth.expectExit()
}

func TestWSAttachWelcome(t *testing.T) {
	coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182"
	port := strconv.Itoa(rand.Intn(65535-1024) + 1024) // Yeah, sometimes this will fail, sorry :P

	geth := runGeth(t,
		"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none",
		"--etherbase", coinbase, "--ws", "--wsport", port)

	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
	testAttachWelcome(t, geth, "ws://localhost:"+port)

	geth.interrupt()
	geth.expectExit()
}

func testAttachWelcome(t *testing.T, geth *testgeth, endpoint string) {
	// Attach to a running geth note and terminate immediately
	attach := runGeth(t, "attach", endpoint)
	defer attach.expectExit()
	attach.stdin.Close()

	// Gather all the infos the welcome message needs to contain
	attach.setTemplateFunc("goos", func() string { return runtime.GOOS })
	attach.setTemplateFunc("gover", runtime.Version)
	attach.setTemplateFunc("gethver", func() string { return verString })
	attach.setTemplateFunc("etherbase", func() string { return geth.Etherbase })
	attach.setTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
	attach.setTemplateFunc("ipc", func() bool { return strings.HasPrefix(endpoint, "ipc") })
	attach.setTemplateFunc("datadir", func() string { return geth.Datadir })
	attach.setTemplateFunc("apis", func() []string {
		var apis []string
		if strings.HasPrefix(endpoint, "ipc") {
			apis = append(strings.Split(rpc.DefaultIPCApis, ","), rpc.MetadataApi)
		} else {
			apis = append(strings.Split(rpc.DefaultHTTPApis, ","), rpc.MetadataApi)
		}
		sort.Strings(apis)
		return apis
	})

	// Verify the actual welcome message to the required template
	attach.expect(`
Welcome to the Geth JavaScript console!

instance: Geth/v{{gethver}}/{{goos}}/{{gover}}
coinbase: {{etherbase}}
at block: 0 ({{niltime}}){{if ipc}}
 datadir: {{datadir}}{{end}}
 modules:{{range apis}} {{.}}:1.0{{end}}

> {{.InputLine "exit" }}
`)
	attach.expectExit()
}
<commit_msg>cmd/geth: truly randomize console test RPC endpoints<commit_after>// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"crypto/rand"
	"math/big"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/ethereum/go-ethereum/rpc"
)

// Tests that a node embedded within a console can be started up properly and
// then terminated by closing the input stream.
func TestConsoleWelcome(t *testing.T) {
	coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182"

	// Start a geth console, make sure it's cleaned up and terminate the console
	geth := runGeth(t,
		"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none",
		"--etherbase", coinbase, "--shh",
		"console")

	// Gather all the infos the welcome message needs to contain
	geth.setTemplateFunc("goos", func() string { return runtime.GOOS })
	geth.setTemplateFunc("gover", runtime.Version)
	geth.setTemplateFunc("gethver", func() string { return verString })
	geth.setTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
	geth.setTemplateFunc("apis", func() []string {
		apis := append(strings.Split(rpc.DefaultIPCApis, ","), rpc.MetadataApi)
		sort.Strings(apis)
		return apis
	})

	// Verify the actual welcome message to the required template
	geth.expect(`
Welcome to the Geth JavaScript console!

instance: Geth/v{{gethver}}/{{goos}}/{{gover}}
coinbase: {{.Etherbase}}
at block: 0 ({{niltime}})
 datadir: {{.Datadir}}
 modules:{{range apis}} {{.}}:1.0{{end}}

> {{.InputLine "exit"}}
`)
	geth.expectExit()
}

// Tests that a console can be attached to a running node via various means.
func TestIPCAttachWelcome(t *testing.T) {
	// Configure the instance for IPC attachement
	coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182"
	var ipc string
	if runtime.GOOS == "windows" {
		ipc = `\\.\pipe\geth` + strconv.Itoa(trulyRandInt(100000, 999999))
	} else {
		ws := tmpdir(t)
		defer os.RemoveAll(ws)
		ipc = filepath.Join(ws, "geth.ipc")
	}
	// Note: we need --shh because testAttachWelcome checks for default
	// list of ipc modules and shh is included there.
	geth := runGeth(t,
		"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none",
		"--etherbase", coinbase, "--shh", "--ipcpath", ipc)

	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
	testAttachWelcome(t, geth, "ipc:"+ipc)

	geth.interrupt()
	geth.expectExit()
}

func TestHTTPAttachWelcome(t *testing.T) {
	coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182"
	port := strconv.Itoa(trulyRandInt(1024, 65536)) // Yeah, sometimes this will fail, sorry :P
	geth := runGeth(t,
		"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none",
		"--etherbase", coinbase, "--rpc", "--rpcport", port)

	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
	testAttachWelcome(t, geth, "http://localhost:"+port)

	geth.interrupt()
	geth.expectExit()
}

func TestWSAttachWelcome(t *testing.T) {
	coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182"
	port := strconv.Itoa(trulyRandInt(1024, 65536)) // Yeah, sometimes this will fail, sorry :P

	geth := runGeth(t,
		"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none",
		"--etherbase", coinbase, "--ws", "--wsport", port)

	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
	testAttachWelcome(t, geth, "ws://localhost:"+port)

	geth.interrupt()
	geth.expectExit()
}

func testAttachWelcome(t *testing.T, geth *testgeth, endpoint string) {
	// Attach to a running geth note and terminate immediately
	attach := runGeth(t, "attach", endpoint)
	defer attach.expectExit()
	attach.stdin.Close()

	// Gather all the infos the welcome message needs to contain
	attach.setTemplateFunc("goos", func() string { return runtime.GOOS })
	attach.setTemplateFunc("gover", runtime.Version)
	attach.setTemplateFunc("gethver", func() string { return verString })
	attach.setTemplateFunc("etherbase", func() string { return geth.Etherbase })
	attach.setTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
	attach.setTemplateFunc("ipc", func() bool { return strings.HasPrefix(endpoint, "ipc") })
	attach.setTemplateFunc("datadir", func() string { return geth.Datadir })
	attach.setTemplateFunc("apis", func() []string {
		var apis []string
		if strings.HasPrefix(endpoint, "ipc") {
			apis = append(strings.Split(rpc.DefaultIPCApis, ","), rpc.MetadataApi)
		} else {
			apis = append(strings.Split(rpc.DefaultHTTPApis, ","), rpc.MetadataApi)
		}
		sort.Strings(apis)
		return apis
	})

	// Verify the actual welcome message to the required template
	attach.expect(`
Welcome to the Geth JavaScript console!

instance: Geth/v{{gethver}}/{{goos}}/{{gover}}
coinbase: {{etherbase}}
at block: 0 ({{niltime}}){{if ipc}}
 datadir: {{datadir}}{{end}}
 modules:{{range apis}} {{.}}:1.0{{end}}

> {{.InputLine "exit" }}
`)
	attach.expectExit()
}

// trulyRandInt generates a crypto random integer used by the console tests to
// not clash network ports with other tests running cocurrently.
func trulyRandInt(lo, hi int) int {
	num, _ := rand.Int(rand.Reader, big.NewInt(int64(hi-lo)))
	return int(num.Int64()) + lo
}
<|endoftext|>


<commit_before>package command

import (
	"flag"
	"fmt"
	"net"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
)

type TopBroker struct {
	Ui  cli.Ui
	Cmd string

	mu sync.Mutex

	zone, cluster, topic string
	drawMode             bool
	interval             time.Duration
	shortIp              bool

	offsets     map[string]int64 // host => offset sum
	lastOffsets map[string]int64
}

func (this *TopBroker) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("topbroker", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.StringVar(&this.topic, "t", "", "")
	cmdFlags.BoolVar(&this.drawMode, "d", false, "")
	cmdFlags.BoolVar(&this.shortIp, "shortip", false, "")
	cmdFlags.DurationVar(&this.interval, "i", time.Second*5, "refresh interval")

	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	this.offsets = make(map[string]int64)
	this.lastOffsets = make(map[string]int64)

	if this.interval.Seconds() < 1 {
		this.interval = time.Second
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		if !patternMatched(zkcluster.Name(), this.cluster) {
			return
		}

		go this.clusterTopProducers(zkcluster)
	})

	if this.drawMode {
		this.drawDashboard()
		return
	}

	ticker := time.NewTicker(this.interval)
	defer ticker.Stop()

	for {
		select {
		case <-ticker.C:
			refreshScreen()
			this.showAndResetCounters(true)

		}
	}

	return
}

func (this *TopBroker) drawDashboard() {
	ticker := time.NewTicker(this.interval)
	defer ticker.Stop()

	var maxQps float64
	width := 140 // TODO
	for {
		select {
		case <-ticker.C:
			refreshScreen()
			datas := this.showAndResetCounters(false)
			maxQps = .0
			for _, data := range datas {
				if data.qps > maxQps {
					maxQps = data.qps
				}
			}

			maxWidth := width - 22
			for _, data := range datas {
				if data.qps < 0 {
					data.qps = -data.qps // FIXME
				}
				w := int(data.qps*100/maxQps) * maxWidth / 100
				this.Ui.Output(fmt.Sprintf("%20s %s", data.host,
					strings.Repeat(color.Green("|"), w)))
			}

		}
	}

}

type brokerQps struct {
	host string
	qps  float64
}

func (this *TopBroker) showAndResetCounters(show bool) []brokerQps {
	this.mu.Lock()
	defer this.mu.Unlock()

	d := this.interval.Seconds()
	sortedHost := make([]string, 0, len(this.offsets))
	for host, _ := range this.offsets {
		sortedHost = append(sortedHost, host)
	}
	sort.Strings(sortedHost)

	if show {
		this.Ui.Output(fmt.Sprintf("%20s %8s", "host", "mps"))
	}

	totalQps := .0
	r := make([]brokerQps, 0, len(sortedHost))
	for _, host := range sortedHost {
		offset := this.offsets[host]
		qps := float64(0)
		if lastOffset, present := this.lastOffsets[host]; present {
			qps = float64(offset-lastOffset) / d
		}

		r = append(r, brokerQps{host, qps})
		totalQps += qps

		if show {
			this.Ui.Output(fmt.Sprintf("%20s %10.1f", host, qps))
		}
	}

	if show {
		this.Ui.Output(fmt.Sprintf("%20s %10.1f", "-TOTAL-", totalQps))
	}

	for host, offset := range this.offsets {
		this.lastOffsets[host] = offset
	}
	this.offsets = make(map[string]int64)

	return r
}

func (this *TopBroker) clusterTopProducers(zkcluster *zk.ZkCluster) {
	kfk, err := sarama.NewClient(zkcluster.BrokerList(), sarama.NewConfig())
	if err != nil {
		return
	}
	defer kfk.Close()

	for {
		topics, err := kfk.Topics()
		swallow(err)

		for _, topic := range topics {
			if !patternMatched(topic, this.topic) {
				continue
			}

			partions, err := kfk.WritablePartitions(topic)
			swallow(err)
			for _, partitionID := range partions {
				leader, err := kfk.Leader(topic, partitionID)
				swallow(err)

				latestOffset, err := kfk.GetOffset(topic, partitionID,
					sarama.OffsetNewest)
				if err != nil {
					panic(err)
				}

				host, _, err := net.SplitHostPort(leader.Addr())
				swallow(err)

				if this.shortIp {
					host = shortIp(host)
				}

				this.mu.Lock()
				if _, present := this.offsets[host]; !present {
					this.offsets[host] = 0
				}
				this.offsets[host] += latestOffset
				this.mu.Unlock()
			}
		}

		time.Sleep(time.Second)
		kfk.RefreshMetadata(topics...)
	}
}

func (*TopBroker) Synopsis() string {
	return "Unix “top” like utility for kafka brokers"
}

func (this *TopBroker) Help() string {
	help := fmt.Sprintf(`
Usage: %s topbroker [options]

    Unix “top” like utility for kafka brokers

Options:

    -z zone
      Default %s

    -c cluster pattern

    -t topic pattern

    -i interval
      Refresh interval in seconds.
      e,g. 5s

    -d
      Draw dashboard in graph.

    -shortip

`, this.Cmd, ctx.ZkDefaultZone())
	return strings.TrimSpace(help)
}
<commit_msg>tweak of the output fmt<commit_after>package command

import (
	"flag"
	"fmt"
	"net"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
)

type TopBroker struct {
	Ui  cli.Ui
	Cmd string

	mu sync.Mutex

	zone, cluster, topic string
	drawMode             bool
	interval             time.Duration
	shortIp              bool

	offsets     map[string]int64 // host => offset sum
	lastOffsets map[string]int64
}

func (this *TopBroker) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("topbroker", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.StringVar(&this.topic, "t", "", "")
	cmdFlags.BoolVar(&this.drawMode, "d", false, "")
	cmdFlags.BoolVar(&this.shortIp, "shortip", false, "")
	cmdFlags.DurationVar(&this.interval, "i", time.Second*5, "refresh interval")

	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	this.offsets = make(map[string]int64)
	this.lastOffsets = make(map[string]int64)

	if this.interval.Seconds() < 1 {
		this.interval = time.Second
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		if !patternMatched(zkcluster.Name(), this.cluster) {
			return
		}

		go this.clusterTopProducers(zkcluster)
	})

	if this.drawMode {
		this.drawDashboard()
		return
	}

	ticker := time.NewTicker(this.interval)
	defer ticker.Stop()

	for {
		select {
		case <-ticker.C:
			refreshScreen()
			this.showAndResetCounters(true)

		}
	}

	return
}

func (this *TopBroker) drawDashboard() {
	ticker := time.NewTicker(this.interval)
	defer ticker.Stop()

	var maxQps float64
	width := 140 // TODO
	maxWidth := width - 23
	for {
		select {
		case <-ticker.C:
			refreshScreen()
			datas := this.showAndResetCounters(false)
			maxQps = .0
			for _, data := range datas {
				if data.qps > maxQps {
					maxQps = data.qps
				}
			}

			if maxQps < 1 {
				continue
			}

			for _, data := range datas {
				if data.qps < 0 {
					data.qps = -data.qps // FIXME
				}
				w := int(data.qps*100/maxQps) * maxWidth / 100
				this.Ui.Output(fmt.Sprintf("%20s [%-118s]",
					data.host,
					strings.Repeat("|", w)))
			}

		}
	}

}

type brokerQps struct {
	host string
	qps  float64
}

func (this *TopBroker) showAndResetCounters(show bool) []brokerQps {
	this.mu.Lock()
	defer this.mu.Unlock()

	d := this.interval.Seconds()
	sortedHost := make([]string, 0, len(this.offsets))
	for host, _ := range this.offsets {
		sortedHost = append(sortedHost, host)
	}
	sort.Strings(sortedHost)

	if show {
		this.Ui.Output(fmt.Sprintf("%20s %8s", "host", "mps"))
	}

	totalQps := .0
	r := make([]brokerQps, 0, len(sortedHost))
	for _, host := range sortedHost {
		offset := this.offsets[host]
		qps := float64(0)
		if lastOffset, present := this.lastOffsets[host]; present {
			qps = float64(offset-lastOffset) / d
		}

		r = append(r, brokerQps{host, qps})
		totalQps += qps

		if show {
			this.Ui.Output(fmt.Sprintf("%20s %10.1f", host, qps))
		}
	}

	if show {
		this.Ui.Output(fmt.Sprintf("%20s %10.1f", "-TOTAL-", totalQps))
	}

	for host, offset := range this.offsets {
		this.lastOffsets[host] = offset
	}
	this.offsets = make(map[string]int64)

	return r
}

func (this *TopBroker) clusterTopProducers(zkcluster *zk.ZkCluster) {
	kfk, err := sarama.NewClient(zkcluster.BrokerList(), sarama.NewConfig())
	if err != nil {
		return
	}
	defer kfk.Close()

	for {
		topics, err := kfk.Topics()
		swallow(err)

		for _, topic := range topics {
			if !patternMatched(topic, this.topic) {
				continue
			}

			partions, err := kfk.WritablePartitions(topic)
			swallow(err)
			for _, partitionID := range partions {
				leader, err := kfk.Leader(topic, partitionID)
				swallow(err)

				latestOffset, err := kfk.GetOffset(topic, partitionID,
					sarama.OffsetNewest)
				if err != nil {
					panic(err)
				}

				host, _, err := net.SplitHostPort(leader.Addr())
				swallow(err)

				if this.shortIp {
					host = shortIp(host)
				}

				this.mu.Lock()
				if _, present := this.offsets[host]; !present {
					this.offsets[host] = 0
				}
				this.offsets[host] += latestOffset
				this.mu.Unlock()
			}
		}

		time.Sleep(time.Second)
		kfk.RefreshMetadata(topics...)
	}
}

func (*TopBroker) Synopsis() string {
	return "Unix “top” like utility for kafka brokers"
}

func (this *TopBroker) Help() string {
	help := fmt.Sprintf(`
Usage: %s topbroker [options]

    Unix “top” like utility for kafka brokers

Options:

    -z zone
      Default %s

    -c cluster pattern

    -t topic pattern

    -i interval
      Refresh interval in seconds.
      e,g. 5s

    -d
      Draw dashboard in graph.

    -shortip

`, this.Cmd, ctx.ZkDefaultZone())
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package shell

import (
	"bufio"
	"fmt"
	"github.com/peterh/liner"
	"io"
	"os"
	"path"
	"pfi/sensorbee/sensorbee/client"
	"strings"
)

// App is the application server of SensorBee.
type App struct {
	historyFn  string
	requester  *client.Requester
	commandMap map[string]Command
}

var tempDir = os.TempDir()

// SetUpCommands set up application. Commands are initialized with it.
func SetUpCommands(commands []Command) App {
	app := App{
		historyFn:  path.Join(tempDir, ".sensorbee_liner_history"),
		commandMap: map[string]Command{},
	}
	if len(commands) == 0 {
		return app
	}

	for _, cmd := range commands {
		cmd.Init()
		for _, v := range cmd.Name() {
			app.commandMap[v] = cmd
		}
	}
	return app
}

func (a *App) prompt(line *liner.State) {
	// create a function to read from the terminal
	// and output an appropriate prompt
	// (if `continued` is true, then the prompt
	// for a continued statement will be shown)
	getNextLine := func(continued bool) (string, error) {
		// create prompt
		promptStart := "(no topology)" + promptLineStart
		if currentTopology.name != "" {
			promptStart = fmt.Sprintf("(%s)%s", currentTopology.name,
				promptLineStart)
		}
		if continued {
			promptStart = promptLineContinue
		}

		// get line from terminal
		input, err := line.Prompt(promptStart)
		if err == nil && input != "" {
			line.AppendHistory(input)
		}
		return input, err
	}

	// continue as long as there is input
	for a.readStartOfNextCommand(getNextLine) {
	}
}

func (a *App) readStartOfNextCommand(getNextLine func(bool) (string, error)) bool {
	input, err := getNextLine(false)
	// if there is no next line, stop
	if err != nil {
		if err != io.EOF {
			fmt.Fprintf(os.Stderr, "error reading line: %v", err)
		}
		// there was an EOF control character, e.g., the
		// user pressed Ctrl+D. in order not to mess up the
		// terminal, write an additional newline character
		fmt.Println("")
		return false
	}

	// if there is input, find the type of command that was input
	if input != "" {
		if strings.ToLower(input) == "exit" {
			fmt.Fprintln(os.Stdout, "SensorBee shell tool is closed")
			return false
		}

		if strings.HasPrefix(input, "--") { // BQL comment
			return true
		}

		// detect which type of command we have
		in := strings.ToLower(strings.Split(input, " ")[0])
		if cmd, ok := a.commandMap[in]; ok {
			// continue to read until the end of the statement
			a.readCompleteCommand(cmd, input, getNextLine)
		} else {
			fmt.Fprintf(os.Stderr, "undefined command: %v\n", in)
		}
	}
	return true
}

func (a *App) readCompleteCommand(cmd Command, input string, getNextLine func(bool) (string, error)) {
	for {
		if input != "" {
			// feed string to the command we detected and check
			// if the command accepts this as valid continuation
			status, err := cmd.Input(input)
			switch status {
			case invalidCMD:
				fmt.Fprintf(os.Stderr, "input command is invalid: %v\n", err)
				return
			case preparedCMD:
				// if the statement is complete, evaluate it
				if fileCmd, ok := cmd.(*fileLoadCmd); ok {
					r := strings.NewReader(fileCmd.queries)
					b := bufio.NewReader(r)
					getNextLineInFile := func(continued bool) (string, error) {
						s, err := b.ReadString('\n')
						return strings.TrimSuffix(s, "\n"), err
					}
					// continue as long as there is input
					for a.readStartOfNextCommand(getNextLineInFile) {
					}
					return
				}
				cmd.Eval(a.requester)
				return
			}
		}

		var err error
		input, err = getNextLine(true)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			return
		}
	}
}

// Run begins SensorBee command line tool to management SensorBee
// and execute BQL/UDF/UDTF statements.
func (a *App) Run(requester *client.Requester) {
	line := liner.NewLiner()
	defer line.Close()

	line.SetCompleter(func(line string) (c []string) {
		// set auto complete command, if need
		return
	})

	if f, err := os.Open(a.historyFn); err == nil {
		line.ReadHistory(f)
		f.Close()
	}

	fmt.Fprintln(os.Stdout, appRunMsg)
	a.requester = requester
	a.prompt(line)

	if f, err := os.Create(a.historyFn); err != nil {
		fmt.Fprintf(os.Stderr, "error writing history file: %v", err)
	} else {
		line.WriteHistory(f)
		f.Close()
	}
}

const (
	appRunMsg          = "SensorBee shell tool is started!"
	promptLineStart    = ">>> "
	promptLineContinue = "... "
)
<commit_msg>deal with EOF and 'exit' commands different if they are in BQL files<commit_after>package shell

import (
	"bufio"
	"fmt"
	"github.com/peterh/liner"
	"io"
	"os"
	"path"
	"pfi/sensorbee/sensorbee/client"
	"strings"
)

// App is the application server of SensorBee.
type App struct {
	historyFn  string
	requester  *client.Requester
	commandMap map[string]Command
}

var tempDir = os.TempDir()

// SetUpCommands set up application. Commands are initialized with it.
func SetUpCommands(commands []Command) App {
	app := App{
		historyFn:  path.Join(tempDir, ".sensorbee_liner_history"),
		commandMap: map[string]Command{},
	}
	if len(commands) == 0 {
		return app
	}

	for _, cmd := range commands {
		cmd.Init()
		for _, v := range cmd.Name() {
			app.commandMap[v] = cmd
		}
	}
	return app
}

func (a *App) prompt(line *liner.State) {
	// create a function to read from the terminal
	// and output an appropriate prompt
	// (if `continued` is true, then the prompt
	// for a continued statement will be shown)
	getNextLine := func(continued bool) (string, error) {
		// create prompt
		promptStart := "(no topology)" + promptLineStart
		if currentTopology.name != "" {
			promptStart = fmt.Sprintf("(%s)%s", currentTopology.name,
				promptLineStart)
		}
		if continued {
			promptStart = promptLineContinue
		}

		// get line from terminal
		input, err := line.Prompt(promptStart)
		if err == nil && input != "" {
			line.AppendHistory(input)
		}
		return input, err
	}

	// continue as long as there is input
	for a.readStartOfNextCommand(getNextLine, true) {
	}
}

func (a *App) readStartOfNextCommand(getNextLine func(bool) (string, error), topLevel bool) bool {
	input, err := getNextLine(false)
	// if there is no next line, stop
	if err != nil {
		if err != io.EOF {
			fmt.Fprintf(os.Stderr, "error reading line: %v", err)
		}
		if topLevel {
			// there was an EOF control character, e.g., the
			// user pressed Ctrl+D. in order not to mess up the
			// terminal, write an additional newline character
			fmt.Println("")
		}
		return false
	}

	// if there is input, find the type of command that was input
	if input != "" {
		if strings.ToLower(input) == "exit" {
			if topLevel {
				fmt.Fprintln(os.Stdout, "SensorBee shell tool is closed")
			} else {
				fmt.Fprintln(os.Stdout, "exit from file processing")
			}
			return false
		}

		if strings.HasPrefix(input, "--") { // BQL comment
			return true
		}

		// detect which type of command we have
		in := strings.ToLower(strings.Split(input, " ")[0])
		if cmd, ok := a.commandMap[in]; ok {
			// continue to read until the end of the statement
			a.readCompleteCommand(cmd, input, getNextLine)
		} else {
			fmt.Fprintf(os.Stderr, "undefined command: %v\n", in)
		}
	}
	return true
}

func (a *App) readCompleteCommand(cmd Command, input string, getNextLine func(bool) (string, error)) {
	for {
		if input != "" {
			// feed string to the command we detected and check
			// if the command accepts this as valid continuation
			status, err := cmd.Input(input)
			switch status {
			case invalidCMD:
				fmt.Fprintf(os.Stderr, "input command is invalid: %v\n", err)
				return
			case preparedCMD:
				// if the statement is complete, evaluate it
				if fileCmd, ok := cmd.(*fileLoadCmd); ok {
					r := strings.NewReader(fileCmd.queries)
					b := bufio.NewReader(r)
					getNextLineInFile := func(continued bool) (string, error) {
						s, err := b.ReadString('\n')
						return strings.TrimSuffix(s, "\n"), err
					}
					// continue as long as there is input
					for a.readStartOfNextCommand(getNextLineInFile, false) {
					}
					return
				}
				cmd.Eval(a.requester)
				return
			}
		}

		var err error
		input, err = getNextLine(true)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			return
		}
	}
}

// Run begins SensorBee command line tool to management SensorBee
// and execute BQL/UDF/UDTF statements.
func (a *App) Run(requester *client.Requester) {
	line := liner.NewLiner()
	defer line.Close()

	line.SetCompleter(func(line string) (c []string) {
		// set auto complete command, if need
		return
	})

	if f, err := os.Open(a.historyFn); err == nil {
		line.ReadHistory(f)
		f.Close()
	}

	fmt.Fprintln(os.Stdout, appRunMsg)
	a.requester = requester
	a.prompt(line)

	if f, err := os.Create(a.historyFn); err != nil {
		fmt.Fprintf(os.Stderr, "error writing history file: %v", err)
	} else {
		line.WriteHistory(f)
		f.Close()
	}
}

const (
	appRunMsg          = "SensorBee shell tool is started!"
	promptLineStart    = ">>> "
	promptLineContinue = "... "
)
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/tobstarr/tobstarr.github.io/cmd/ts-gen-tobstarr/Godeps/_workspace/src/github.com/shurcooL/github_flavored_markdown"
)

var sources = map[string][]Source{
	"docker.html":         Layout(MarkdownSource("src/articles/docker/index.md")),
	"dotfiles.html":       Layout(FileSource("src/dotfiles/index.tpl")),
	"dotfiles/vimrc.conf": FileSources("dotfiles/vimrc"),
	"gnupg.html":          Layout(MarkdownSource("src/articles/gnupg/index.md")),
	"id_rsa.pub":          FileSources("src/id_rsa.pub"),
	"index.html":          Layout(FileSource("src/index.tpl")),
	"kb.html":             Layout(FileSource("src/kb.tpl")),
	"tobstarr.gpg":        FileSources("src/tobstarr.gpg"),
	"versions.html":       Layout(MarkdownSource("src/versions.md")),
}

func main() {
	l := log.New(os.Stderr, "", 0)
	if err := run(l); err != nil {
		log.Fatal(err)
	}
}

func FileSources(paths ...string) (list []Source) {
	for _, p := range paths {
		list = append(list, FileSource(p))
	}
	return list

}

func loadHTMLFiles() (list map[string]struct{}, err error) {
	list = map[string]struct{}{}
	return list, filepath.Walk(".", func(p string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		} else if strings.HasSuffix(p, ".html") {
			list[p] = struct{}{}
		}
		return nil
	})
}

func Layout(s Source) []Source {
	return []Source{FileSource("src/header.tpl"), s, FileSource("src/footer.tpl")}
}

func MarkdownSource(path string) Source {
	return func(w io.Writer) error {
		b, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		b = github_flavored_markdown.Markdown(b)
		_, err = w.Write(b)
		return err
	}
}

func run(l *log.Logger) error {
	start := time.Now()
	m, err := loadHTMLFiles()
	if err != nil {
		return err
	}
	for path, sources := range sources {
		if err := writeFile(l, path, sources...); err != nil {
			return err
		}
		delete(m, path)
	}
	for k := range m {
		l.Printf("deleting file %s", k)
		os.RemoveAll(k)
	}
	l.Printf("finished in %.06f", time.Since(start).Seconds())
	return nil
}

func FileSource(path string) Source {
	return func(w io.Writer) error {
		f, err := os.Open(path)
		if err != nil {
			return err
		}
		defer f.Close()
		_, err = io.Copy(w, f)
		return err
	}
}

type Source func(w io.Writer) error

type Logger interface {
	Printf(string, ...interface{})
}

func writeFile(l Logger, out string, sources ...Source) error {
	start := time.Now()
	buf := &bytes.Buffer{}
	for _, src := range sources {
		if err := src(buf); err != nil {
			return err
		}
	}
	tmp := out + ".tmp." + strconv.FormatInt(time.Now().UTC().UnixNano(), 36)
	f, err := os.Create(tmp)
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmp)
	_, err = io.Copy(f, buf)
	if err != nil {
		return err
	}
	err = os.Rename(tmp, out)
	if err != nil {
		return err
	}
	l.Printf("write file %s in %.6f", out, time.Since(start).Seconds())
	return nil
}
<commit_msg>skip some directories when deleting .html<commit_after>package main

import (
	"bytes"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/tobstarr/tobstarr.github.io/cmd/ts-gen-tobstarr/Godeps/_workspace/src/github.com/shurcooL/github_flavored_markdown"
)

var sources = map[string][]Source{
	"docker.html":         Layout(MarkdownSource("src/articles/docker/index.md")),
	"dotfiles.html":       Layout(FileSource("src/dotfiles/index.tpl")),
	"dotfiles/vimrc.conf": FileSources("dotfiles/vimrc"),
	"gnupg.html":          Layout(MarkdownSource("src/articles/gnupg/index.md")),
	"id_rsa.pub":          FileSources("src/id_rsa.pub"),
	"index.html":          Layout(FileSource("src/index.tpl")),
	"kb.html":             Layout(FileSource("src/kb.tpl")),
	"tobstarr.gpg":        FileSources("src/tobstarr.gpg"),
	"versions.html":       Layout(MarkdownSource("src/versions.md")),
}

func main() {
	l := log.New(os.Stderr, "", 0)
	if err := run(l); err != nil {
		log.Fatal(err)
	}
}

func FileSources(paths ...string) (list []Source) {
	for _, p := range paths {
		list = append(list, FileSource(p))
	}
	return list

}

func loadHTMLFiles() (list map[string]struct{}, err error) {
	list = map[string]struct{}{}
	return list, filepath.Walk(".", func(p string, info os.FileInfo, err error) error {
		switch {
		case info.IsDir():
			switch p {
			case ".git", "cmd":
				return filepath.SkipDir
			}
			log.Printf("checking %s", p)
			return nil
		case strings.HasSuffix(p, ".html"):
			list[p] = struct{}{}
		}
		return nil
	})
}

func Layout(s Source) []Source {
	return []Source{FileSource("src/header.tpl"), s, FileSource("src/footer.tpl")}
}

func MarkdownSource(path string) Source {
	return func(w io.Writer) error {
		b, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		b = github_flavored_markdown.Markdown(b)
		_, err = w.Write(b)
		return err
	}
}

func run(l *log.Logger) error {
	start := time.Now()
	m, err := loadHTMLFiles()
	if err != nil {
		return err
	}
	for path, sources := range sources {
		if err := writeFile(l, path, sources...); err != nil {
			return err
		}
		delete(m, path)
	}
	for k := range m {
		l.Printf("deleting file %s", k)
		os.RemoveAll(k)
	}
	l.Printf("finished in %.06f", time.Since(start).Seconds())
	return nil
}

func FileSource(path string) Source {
	return func(w io.Writer) error {
		f, err := os.Open(path)
		if err != nil {
			return err
		}
		defer f.Close()
		_, err = io.Copy(w, f)
		return err
	}
}

type Source func(w io.Writer) error

type Logger interface {
	Printf(string, ...interface{})
}

func writeFile(l Logger, out string, sources ...Source) error {
	start := time.Now()
	buf := &bytes.Buffer{}
	for _, src := range sources {
		if err := src(buf); err != nil {
			return err
		}
	}
	tmp := out + ".tmp." + strconv.FormatInt(time.Now().UTC().UnixNano(), 36)
	f, err := os.Create(tmp)
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmp)
	_, err = io.Copy(f, buf)
	if err != nil {
		return err
	}
	err = os.Rename(tmp, out)
	if err != nil {
		return err
	}
	l.Printf("write file %s in %.6f", out, time.Since(start).Seconds())
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduling

import (
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"k8s.io/api/core/v1"
	extensions "k8s.io/api/extensions/v1beta1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/uuid"
	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/test/e2e/framework"
	imageutils "k8s.io/kubernetes/test/utils/image"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	testPodNamePrefix = "nvidia-gpu-"
	cosOSImage        = "Container-Optimized OS from Google"
	// Nvidia driver installation can take upwards of 5 minutes.
	driverInstallTimeout = 10 * time.Minute
)

type podCreationFuncType func() *v1.Pod

var (
	gpuResourceName v1.ResourceName
	dsYamlUrl       string
	podCreationFunc podCreationFuncType
)

func makeCudaAdditionTestPod() *v1.Pod {
	podName := testPodNamePrefix + string(uuid.NewUUID())
	testPod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: podName,
		},
		Spec: v1.PodSpec{
			RestartPolicy: v1.RestartPolicyNever,
			Containers: []v1.Container{
				{
					Name:  "vector-addition",
					Image: imageutils.GetE2EImage(imageutils.CudaVectorAdd),
					Resources: v1.ResourceRequirements{
						Limits: v1.ResourceList{
							gpuResourceName: *resource.NewQuantity(1, resource.DecimalSI),
						},
					},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      "nvidia-libraries",
							MountPath: "/usr/local/nvidia/lib64",
						},
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: "nvidia-libraries",
					VolumeSource: v1.VolumeSource{
						HostPath: &v1.HostPathVolumeSource{
							Path: "/home/kubernetes/bin/nvidia/lib",
						},
					},
				},
			},
		},
	}
	return testPod
}

func makeCudaAdditionDevicePluginTestPod() *v1.Pod {
	podName := testPodNamePrefix + string(uuid.NewUUID())
	testPod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: podName,
		},
		Spec: v1.PodSpec{
			RestartPolicy: v1.RestartPolicyNever,
			Containers: []v1.Container{
				{
					Name:  "vector-addition",
					Image: imageutils.GetE2EImage(imageutils.CudaVectorAdd),
					Resources: v1.ResourceRequirements{
						Limits: v1.ResourceList{
							gpuResourceName: *resource.NewQuantity(1, resource.DecimalSI),
						},
					},
				},
			},
		},
	}
	return testPod
}

func isClusterRunningCOS(f *framework.Framework) bool {
	nodeList, err := f.ClientSet.CoreV1().Nodes().List(metav1.ListOptions{})
	framework.ExpectNoError(err, "getting node list")
	for _, node := range nodeList.Items {
		if !strings.Contains(node.Status.NodeInfo.OSImage, cosOSImage) {
			return false
		}
	}
	return true
}

func areGPUsAvailableOnAllSchedulableNodes(f *framework.Framework) bool {
	framework.Logf("Getting list of Nodes from API server")
	nodeList, err := f.ClientSet.CoreV1().Nodes().List(metav1.ListOptions{})
	framework.ExpectNoError(err, "getting node list")
	for _, node := range nodeList.Items {
		if node.Spec.Unschedulable {
			continue
		}
		framework.Logf("gpuResourceName %s", gpuResourceName)
		if val, ok := node.Status.Capacity[gpuResourceName]; !ok || val.Value() == 0 {
			framework.Logf("Nvidia GPUs not available on Node: %q", node.Name)
			return false
		}
	}
	framework.Logf("Nvidia GPUs exist on all schedulable nodes")
	return true
}

func getGPUsAvailable(f *framework.Framework) int64 {
	nodeList, err := f.ClientSet.CoreV1().Nodes().List(metav1.ListOptions{})
	framework.ExpectNoError(err, "getting node list")
	var gpusAvailable int64
	for _, node := range nodeList.Items {
		if val, ok := node.Status.Capacity[gpuResourceName]; ok {
			gpusAvailable += (&val).Value()
		}
	}
	return gpusAvailable
}

func testNvidiaGPUsOnCOS(f *framework.Framework) {
	// Skip the test if the base image is not COS.
	// TODO: Add support for other base images.
	// CUDA apps require host mounts which is not portable across base images (yet).
	framework.Logf("Checking base image")
	if !isClusterRunningCOS(f) {
		Skip("Nvidia GPU tests are supproted only on Container Optimized OS image currently")
	}
	framework.Logf("Cluster is running on COS. Proceeding with test")

	if f.BaseName == "device-plugin-gpus" {
		dsYamlUrl = "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/device-plugin-daemonset.yaml"
		gpuResourceName = "nvidia.com/gpu"
		podCreationFunc = makeCudaAdditionDevicePluginTestPod
	} else {
		dsYamlUrl = "https://raw.githubusercontent.com/ContainerEngine/accelerators/master/cos-nvidia-gpu-installer/daemonset.yaml"
		gpuResourceName = v1.ResourceNvidiaGPU
		podCreationFunc = makeCudaAdditionTestPod
	}

	// GPU drivers might have already been installed.
	if !areGPUsAvailableOnAllSchedulableNodes(f) {
		// Install Nvidia Drivers.
		ds := dsFromManifest(dsYamlUrl)
		ds.Namespace = f.Namespace.Name
		_, err := f.ClientSet.Extensions().DaemonSets(f.Namespace.Name).Create(ds)
		framework.ExpectNoError(err, "failed to create daemonset")
		framework.Logf("Successfully created daemonset to install Nvidia drivers. Waiting for drivers to be installed and GPUs to be available in Node Capacity...")
		// Wait for Nvidia GPUs to be available on nodes
		Eventually(func() bool {
			return areGPUsAvailableOnAllSchedulableNodes(f)
		}, driverInstallTimeout, time.Second).Should(BeTrue())
	}
	framework.Logf("Creating as many pods as there are Nvidia GPUs and have the pods run a CUDA app")
	podList := []*v1.Pod{}
	for i := int64(0); i < getGPUsAvailable(f); i++ {
		podList = append(podList, f.PodClient().Create(podCreationFunc()))
	}
	framework.Logf("Wait for all test pods to succeed")
	// Wait for all pods to succeed
	for _, po := range podList {
		f.PodClient().WaitForSuccess(po.Name, 5*time.Minute)
	}
}

// dsFromManifest reads a .json/yaml file and returns the daemonset in it.
func dsFromManifest(url string) *extensions.DaemonSet {
	var controller extensions.DaemonSet
	framework.Logf("Parsing ds from %v", url)

	var response *http.Response
	var err error
	for i := 1; i <= 5; i++ {
		response, err = http.Get(url)
		if err == nil && response.StatusCode == 200 {
			break
		}
		time.Sleep(time.Duration(i) * time.Second)
	}
	Expect(err).NotTo(HaveOccurred())
	Expect(response.StatusCode).To(Equal(200))
	defer response.Body.Close()

	data, err := ioutil.ReadAll(response.Body)
	Expect(err).NotTo(HaveOccurred())

	json, err := utilyaml.ToJSON(data)
	Expect(err).NotTo(HaveOccurred())

	Expect(runtime.DecodeInto(api.Codecs.UniversalDecoder(), json, &controller)).NotTo(HaveOccurred())
	return &controller
}

var _ = SIGDescribe("[Feature:GPU]", func() {
	f := framework.NewDefaultFramework("gpus")
	It("run Nvidia GPU tests on Container Optimized OS only", func() {
		testNvidiaGPUsOnCOS(f)
	})
})

var _ = SIGDescribe("[Feature:GPUDevicePlugin]", func() {
	f := framework.NewDefaultFramework("device-plugin-gpus")
	It("run Nvidia GPU Device Plugin tests on Container Optimized OS only", func() {
		testNvidiaGPUsOnCOS(f)
	})
})
<commit_msg>Extends GPUDevicePlugin e2e test to exercise device plugin restarts.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduling

import (
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"k8s.io/api/core/v1"
	extensions "k8s.io/api/extensions/v1beta1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/uuid"
	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/test/e2e/framework"
	imageutils "k8s.io/kubernetes/test/utils/image"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	testPodNamePrefix = "nvidia-gpu-"
	cosOSImage        = "Container-Optimized OS from Google"
	// Nvidia driver installation can take upwards of 5 minutes.
	driverInstallTimeout = 10 * time.Minute
)

type podCreationFuncType func() *v1.Pod

var (
	gpuResourceName v1.ResourceName
	dsYamlUrl       string
	podCreationFunc podCreationFuncType
)

func makeCudaAdditionTestPod() *v1.Pod {
	podName := testPodNamePrefix + string(uuid.NewUUID())
	testPod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: podName,
		},
		Spec: v1.PodSpec{
			RestartPolicy: v1.RestartPolicyNever,
			Containers: []v1.Container{
				{
					Name:  "vector-addition",
					Image: imageutils.GetE2EImage(imageutils.CudaVectorAdd),
					Resources: v1.ResourceRequirements{
						Limits: v1.ResourceList{
							gpuResourceName: *resource.NewQuantity(1, resource.DecimalSI),
						},
					},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      "nvidia-libraries",
							MountPath: "/usr/local/nvidia/lib64",
						},
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: "nvidia-libraries",
					VolumeSource: v1.VolumeSource{
						HostPath: &v1.HostPathVolumeSource{
							Path: "/home/kubernetes/bin/nvidia/lib",
						},
					},
				},
			},
		},
	}
	return testPod
}

func makeCudaAdditionDevicePluginTestPod() *v1.Pod {
	podName := testPodNamePrefix + string(uuid.NewUUID())
	testPod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: podName,
		},
		Spec: v1.PodSpec{
			RestartPolicy: v1.RestartPolicyNever,
			Containers: []v1.Container{
				{
					Name:  "vector-addition",
					Image: imageutils.GetE2EImage(imageutils.CudaVectorAdd),
					Resources: v1.ResourceRequirements{
						Limits: v1.ResourceList{
							gpuResourceName: *resource.NewQuantity(1, resource.DecimalSI),
						},
					},
				},
			},
		},
	}
	return testPod
}

func isClusterRunningCOS(f *framework.Framework) bool {
	nodeList, err := f.ClientSet.CoreV1().Nodes().List(metav1.ListOptions{})
	framework.ExpectNoError(err, "getting node list")
	for _, node := range nodeList.Items {
		if !strings.Contains(node.Status.NodeInfo.OSImage, cosOSImage) {
			return false
		}
	}
	return true
}

func areGPUsAvailableOnAllSchedulableNodes(f *framework.Framework) bool {
	framework.Logf("Getting list of Nodes from API server")
	nodeList, err := f.ClientSet.CoreV1().Nodes().List(metav1.ListOptions{})
	framework.ExpectNoError(err, "getting node list")
	for _, node := range nodeList.Items {
		if node.Spec.Unschedulable {
			continue
		}
		framework.Logf("gpuResourceName %s", gpuResourceName)
		if val, ok := node.Status.Capacity[gpuResourceName]; !ok || val.Value() == 0 {
			framework.Logf("Nvidia GPUs not available on Node: %q", node.Name)
			return false
		}
	}
	framework.Logf("Nvidia GPUs exist on all schedulable nodes")
	return true
}

func getGPUsAvailable(f *framework.Framework) int64 {
	nodeList, err := f.ClientSet.CoreV1().Nodes().List(metav1.ListOptions{})
	framework.ExpectNoError(err, "getting node list")
	var gpusAvailable int64
	for _, node := range nodeList.Items {
		if val, ok := node.Status.Capacity[gpuResourceName]; ok {
			gpusAvailable += (&val).Value()
		}
	}
	return gpusAvailable
}

func testNvidiaGPUsOnCOS(f *framework.Framework) {
	// Skip the test if the base image is not COS.
	// TODO: Add support for other base images.
	// CUDA apps require host mounts which is not portable across base images (yet).
	framework.Logf("Checking base image")
	if !isClusterRunningCOS(f) {
		Skip("Nvidia GPU tests are supproted only on Container Optimized OS image currently")
	}
	framework.Logf("Cluster is running on COS. Proceeding with test")

	if f.BaseName == "device-plugin-gpus" {
		dsYamlUrl = "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/device-plugin-daemonset.yaml"
		gpuResourceName = "nvidia.com/gpu"
		podCreationFunc = makeCudaAdditionDevicePluginTestPod
	} else {
		dsYamlUrl = "https://raw.githubusercontent.com/ContainerEngine/accelerators/master/cos-nvidia-gpu-installer/daemonset.yaml"
		gpuResourceName = v1.ResourceNvidiaGPU
		podCreationFunc = makeCudaAdditionTestPod
	}

	// GPU drivers might have already been installed.
	if !areGPUsAvailableOnAllSchedulableNodes(f) {
		// Install Nvidia Drivers.
		ds := dsFromManifest(dsYamlUrl)
		ds.Namespace = f.Namespace.Name
		_, err := f.ClientSet.Extensions().DaemonSets(f.Namespace.Name).Create(ds)
		framework.ExpectNoError(err, "failed to create daemonset")
		framework.Logf("Successfully created daemonset to install Nvidia drivers. Waiting for drivers to be installed and GPUs to be available in Node Capacity...")
		// Wait for Nvidia GPUs to be available on nodes
		Eventually(func() bool {
			return areGPUsAvailableOnAllSchedulableNodes(f)
		}, driverInstallTimeout, time.Second).Should(BeTrue())
	}
	framework.Logf("Creating as many pods as there are Nvidia GPUs and have the pods run a CUDA app")
	podList := []*v1.Pod{}
	for i := int64(0); i < getGPUsAvailable(f); i++ {
		podList = append(podList, f.PodClient().Create(podCreationFunc()))
	}
	framework.Logf("Wait for all test pods to succeed")
	// Wait for all pods to succeed
	for _, po := range podList {
		f.PodClient().WaitForSuccess(po.Name, 5*time.Minute)
	}
}

// dsFromManifest reads a .json/yaml file and returns the daemonset in it.
func dsFromManifest(url string) *extensions.DaemonSet {
	var controller extensions.DaemonSet
	framework.Logf("Parsing ds from %v", url)

	var response *http.Response
	var err error
	for i := 1; i <= 5; i++ {
		response, err = http.Get(url)
		if err == nil && response.StatusCode == 200 {
			break
		}
		time.Sleep(time.Duration(i) * time.Second)
	}
	Expect(err).NotTo(HaveOccurred())
	Expect(response.StatusCode).To(Equal(200))
	defer response.Body.Close()

	data, err := ioutil.ReadAll(response.Body)
	Expect(err).NotTo(HaveOccurred())

	json, err := utilyaml.ToJSON(data)
	Expect(err).NotTo(HaveOccurred())

	Expect(runtime.DecodeInto(api.Codecs.UniversalDecoder(), json, &controller)).NotTo(HaveOccurred())
	return &controller
}

var _ = SIGDescribe("[Feature:GPU]", func() {
	f := framework.NewDefaultFramework("gpus")
	It("run Nvidia GPU tests on Container Optimized OS only", func() {
		testNvidiaGPUsOnCOS(f)
	})
})

var _ = SIGDescribe("[Feature:GPUDevicePlugin]", func() {
	f := framework.NewDefaultFramework("device-plugin-gpus")
	It("run Nvidia GPU Device Plugin tests on Container Optimized OS only", func() {
		// 1. Verifies GPU resource is successfully advertised on the nodes
		// and we can run pods using GPUs.
		By("Starting device plugin daemonset and running GPU pods")
		testNvidiaGPUsOnCOS(f)

		// 2. Verifies that when the device plugin DaemonSet is removed, resource capacity drops to zero.
		By("Deleting device plugin daemonset")
		ds := dsFromManifest(dsYamlUrl)
		falseVar := false
		err := f.ClientSet.Extensions().DaemonSets(f.Namespace.Name).Delete(ds.Name, &metav1.DeleteOptions{OrphanDependents: &falseVar})
		framework.ExpectNoError(err, "failed to delete daemonset")
		framework.Logf("Successfully deleted device plugin daemonset. Wait for resource to be removed.")
		// Wait for Nvidia GPUs to be not available on nodes
		Eventually(func() bool {
			return !areGPUsAvailableOnAllSchedulableNodes(f)
		}, 5*time.Minute, time.Second).Should(BeTrue())

		// 3. Restarts the device plugin DaemonSet. Verifies GPU resource is successfully advertised
		// on the nodes and we can run pods using GPUs.
		By("Restarting device plugin daemonset and running GPU pods")
		testNvidiaGPUsOnCOS(f)
	})
})
<|endoftext|>


<commit_before>package builds

import (
	"context"
	"fmt"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	e2e "k8s.io/kubernetes/test/e2e/framework"

	buildv1 "github.com/openshift/api/build/v1"
	eximages "github.com/openshift/origin/test/extended/images"
	exutil "github.com/openshift/origin/test/extended/util"
	"github.com/openshift/origin/test/extended/util/image"
)

var _ = g.Describe("[sig-builds][Feature:Builds] Multi-stage image builds", func() {
	defer g.GinkgoRecover()
	var (
		oc             = exutil.NewCLI("build-multistage")
		testDockerfile = fmt.Sprintf(`
FROM scratch as test
USER 1001
FROM %[1]s as other
COPY --from=test /usr/bin/curl /test/
COPY --from=%[2]s /bin/echo /test/
COPY --from=%[2]s /bin/ping /test/
`, image.LimitedShellImage(), image.ShellImage())
	)

	g.AfterEach(func() {
		if g.CurrentGinkgoTestDescription().Failed {
			exutil.DumpPodStates(oc)
			exutil.DumpConfigMapStates(oc)
			exutil.DumpPodLogsStartingWith("", oc)
		}
	})

	g.It("should succeed", func() {
		g.By("creating a build directly")
		registryURL, err := eximages.GetDockerRegistryURL(oc)
		o.Expect(err).NotTo(o.HaveOccurred())

		build, err := oc.BuildClient().BuildV1().Builds(oc.Namespace()).Create(context.Background(), &buildv1.Build{
			ObjectMeta: metav1.ObjectMeta{
				Name: "multi-stage",
			},
			Spec: buildv1.BuildSpec{
				CommonSpec: buildv1.CommonSpec{
					Source: buildv1.BuildSource{
						Dockerfile: &testDockerfile,
						Images: []buildv1.ImageSource{
							{From: corev1.ObjectReference{Kind: "DockerImage", Name: image.ShellImage()}, As: []string{"scratch"}},
						},
					},
					Strategy: buildv1.BuildStrategy{
						DockerStrategy: &buildv1.DockerBuildStrategy{},
					},
					Output: buildv1.BuildOutput{
						To: &corev1.ObjectReference{
							Kind: "DockerImage",
							Name: fmt.Sprintf("%s/%s/multi-stage:v1", registryURL, oc.Namespace()),
						},
					},
				},
			},
		}, metav1.CreateOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		result := exutil.NewBuildResult(oc, build)
		err = exutil.WaitForBuildResult(oc.AdminBuildClient().BuildV1().Builds(oc.Namespace()), result)
		o.Expect(err).NotTo(o.HaveOccurred())

		pod, err := oc.KubeClient().CoreV1().Pods(oc.Namespace()).Get(context.Background(), build.Name+"-build", metav1.GetOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		o.Expect(result.BuildSuccess).To(o.BeTrue(), "Build did not succeed: %#v", result)

		s, err := result.Logs()
		o.Expect(err).NotTo(o.HaveOccurred())
		o.Expect(s).ToNot(o.ContainSubstring("--> FROM scratch"))
		o.Expect(s).ToNot(o.ContainSubstring("FROM busybox"))
		o.Expect(s).To(o.ContainSubstring(fmt.Sprintf("STEP 1: FROM %s AS test", image.ShellImage())))
		o.Expect(s).To(o.ContainSubstring("COPY --from"))
		o.Expect(s).To(o.ContainSubstring(fmt.Sprintf("\"OPENSHIFT_BUILD_NAMESPACE\"=\"%s\"", oc.Namespace())))
		e2e.Logf("Build logs:\n%s", result)

		c := oc.KubeFramework().PodClient()
		pod = c.Create(&corev1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "test",
			},
			Spec: corev1.PodSpec{
				RestartPolicy: corev1.RestartPolicyNever,
				Containers: []corev1.Container{
					{
						Name:    "run",
						Image:   fmt.Sprintf("%s/%s/multi-stage:v1", registryURL, oc.Namespace()),
						Command: []string{"/test/curl", "-k", "https://kubernetes.default.svc"},
					},
					{
						Name:    "check",
						Image:   fmt.Sprintf("%s/%s/multi-stage:v1", registryURL, oc.Namespace()),
						Command: []string{"ls", "/test/"},
					},
				},
			},
		})
		c.WaitForSuccess(pod.Name, e2e.PodStartTimeout)
		data, err := oc.Run("logs").Args("-f", "test", "-c", "run").Output()
		o.Expect(err).NotTo(o.HaveOccurred())
		m, err := oc.Run("logs").Args("-f", "test", "-c", "check").Output()
		o.Expect(err).NotTo(o.HaveOccurred())
		o.Expect(m).To(o.ContainSubstring("echo"))
		o.Expect(m).To(o.ContainSubstring("ping"))
		e2e.Logf("Pod logs:\n%s\n%s", string(data), string(m))
	})
})
<commit_msg>tests/extended/builds: handle new step logging<commit_after>package builds

import (
	"context"
	"fmt"
	"regexp"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	e2e "k8s.io/kubernetes/test/e2e/framework"

	buildv1 "github.com/openshift/api/build/v1"
	eximages "github.com/openshift/origin/test/extended/images"
	exutil "github.com/openshift/origin/test/extended/util"
	"github.com/openshift/origin/test/extended/util/image"
)

var _ = g.Describe("[sig-builds][Feature:Builds] Multi-stage image builds", func() {
	defer g.GinkgoRecover()
	var (
		oc             = exutil.NewCLI("build-multistage")
		testDockerfile = fmt.Sprintf(`
FROM scratch as test
USER 1001
FROM %[1]s as other
COPY --from=test /usr/bin/curl /test/
COPY --from=%[2]s /bin/echo /test/
COPY --from=%[2]s /bin/ping /test/
`, image.LimitedShellImage(), image.ShellImage())
	)

	g.AfterEach(func() {
		if g.CurrentGinkgoTestDescription().Failed {
			exutil.DumpPodStates(oc)
			exutil.DumpConfigMapStates(oc)
			exutil.DumpPodLogsStartingWith("", oc)
		}
	})

	g.It("should succeed", func() {
		g.By("creating a build directly")
		registryURL, err := eximages.GetDockerRegistryURL(oc)
		o.Expect(err).NotTo(o.HaveOccurred())

		build, err := oc.BuildClient().BuildV1().Builds(oc.Namespace()).Create(context.Background(), &buildv1.Build{
			ObjectMeta: metav1.ObjectMeta{
				Name: "multi-stage",
			},
			Spec: buildv1.BuildSpec{
				CommonSpec: buildv1.CommonSpec{
					Source: buildv1.BuildSource{
						Dockerfile: &testDockerfile,
						Images: []buildv1.ImageSource{
							{From: corev1.ObjectReference{Kind: "DockerImage", Name: image.ShellImage()}, As: []string{"scratch"}},
						},
					},
					Strategy: buildv1.BuildStrategy{
						DockerStrategy: &buildv1.DockerBuildStrategy{},
					},
					Output: buildv1.BuildOutput{
						To: &corev1.ObjectReference{
							Kind: "DockerImage",
							Name: fmt.Sprintf("%s/%s/multi-stage:v1", registryURL, oc.Namespace()),
						},
					},
				},
			},
		}, metav1.CreateOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		result := exutil.NewBuildResult(oc, build)
		err = exutil.WaitForBuildResult(oc.AdminBuildClient().BuildV1().Builds(oc.Namespace()), result)
		o.Expect(err).NotTo(o.HaveOccurred())

		pod, err := oc.KubeClient().CoreV1().Pods(oc.Namespace()).Get(context.Background(), build.Name+"-build", metav1.GetOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		o.Expect(result.BuildSuccess).To(o.BeTrue(), "Build did not succeed: %#v", result)

		s, err := result.Logs()
		o.Expect(err).NotTo(o.HaveOccurred())
		o.Expect(s).ToNot(o.ContainSubstring("--> FROM scratch"))
		o.Expect(s).ToNot(o.ContainSubstring("FROM busybox"))
		o.Expect(s).To(o.MatchRegexp("(\\[1/2\\] STEP 1/2|STEP 1): FROM %s AS test", regexp.QuoteMeta(image.ShellImage())))
		o.Expect(s).To(o.ContainSubstring("COPY --from"))
		o.Expect(s).To(o.ContainSubstring("\"OPENSHIFT_BUILD_NAMESPACE\"=\"%s\"", oc.Namespace()))
		e2e.Logf("Build logs:\n%s", result)

		c := oc.KubeFramework().PodClient()
		pod = c.Create(&corev1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "test",
			},
			Spec: corev1.PodSpec{
				RestartPolicy: corev1.RestartPolicyNever,
				Containers: []corev1.Container{
					{
						Name:    "run",
						Image:   fmt.Sprintf("%s/%s/multi-stage:v1", registryURL, oc.Namespace()),
						Command: []string{"/test/curl", "-k", "https://kubernetes.default.svc"},
					},
					{
						Name:    "check",
						Image:   fmt.Sprintf("%s/%s/multi-stage:v1", registryURL, oc.Namespace()),
						Command: []string{"ls", "/test/"},
					},
				},
			},
		})
		c.WaitForSuccess(pod.Name, e2e.PodStartTimeout)
		data, err := oc.Run("logs").Args("-f", "test", "-c", "run").Output()
		o.Expect(err).NotTo(o.HaveOccurred())
		m, err := oc.Run("logs").Args("-f", "test", "-c", "check").Output()
		o.Expect(err).NotTo(o.HaveOccurred())
		o.Expect(m).To(o.ContainSubstring("echo"))
		o.Expect(m).To(o.ContainSubstring("ping"))
		e2e.Logf("Pod logs:\n%s\n%s", string(data), string(m))
	})
})
<|endoftext|>


<commit_before>// +build integration,!no-etcd

/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

// This file tests the scheduler.

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/resource"
	"k8s.io/kubernetes/pkg/api/testapi"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apiserver"
	"k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/client/record"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/master"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/plugin/pkg/admission/admit"
	"k8s.io/kubernetes/plugin/pkg/scheduler"
	_ "k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider"
	"k8s.io/kubernetes/plugin/pkg/scheduler/factory"
	"k8s.io/kubernetes/test/integration/framework"
)

type nodeMutationFunc func(t *testing.T, n *api.Node, nodeStore cache.Store, c *client.Client)

type nodeStateManager struct {
	makeSchedulable   nodeMutationFunc
	makeUnSchedulable nodeMutationFunc
}

func TestUnschedulableNodes(t *testing.T) {
	etcdStorage, err := framework.NewEtcdStorage()
	if err != nil {
		t.Fatalf("Couldn't create etcd storage: %v", err)
	}
	storageDestinations := master.NewStorageDestinations()
	storageDestinations.AddAPIGroup("", etcdStorage)
	framework.DeleteAllEtcdKeys()

	var m *master.Master
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		m.Handler.ServeHTTP(w, req)
	}))
	defer s.Close()

	m = master.New(&master.Config{
		StorageDestinations:   storageDestinations,
		KubeletClient:         client.FakeKubeletClient{},
		EnableCoreControllers: true,
		EnableLogsSupport:     false,
		EnableUISupport:       false,
		EnableIndex:           true,
		APIPrefix:             "/api",
		Authorizer:            apiserver.NewAlwaysAllowAuthorizer(),
		AdmissionControl:      admit.NewAlwaysAdmit(),
		StorageVersions:       map[string]string{"": testapi.Default.Version()},
	})

	restClient := client.NewOrDie(&client.Config{Host: s.URL, Version: testapi.Default.Version()})

	schedulerConfigFactory := factory.NewConfigFactory(restClient, nil)
	schedulerConfig, err := schedulerConfigFactory.Create()
	if err != nil {
		t.Fatalf("Couldn't create scheduler config: %v", err)
	}
	eventBroadcaster := record.NewBroadcaster()
	schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "scheduler"})
	eventBroadcaster.StartRecordingToSink(restClient.Events(""))
	scheduler.New(schedulerConfig).Run()

	defer close(schedulerConfig.StopEverything)

	DoTestUnschedulableNodes(t, restClient, schedulerConfigFactory.NodeLister.Store)
}

func podScheduled(c *client.Client, podNamespace, podName string) wait.ConditionFunc {
	return func() (bool, error) {
		pod, err := c.Pods(podNamespace).Get(podName)
		if errors.IsNotFound(err) {
			return false, nil
		}
		if err != nil {
			// This could be a connection error so we want to retry.
			return false, nil
		}
		if pod.Spec.NodeName == "" {
			return false, nil
		}
		return true, nil
	}
}

// Wait till the passFunc confirms that the object it expects to see is in the store.
// Used to observe reflected events.
func waitForReflection(s cache.Store, key string, passFunc func(n interface{}) bool) error {
	return wait.Poll(time.Millisecond*10, time.Second*20, func() (bool, error) {
		if n, _, err := s.GetByKey(key); err == nil && passFunc(n) {
			return true, nil
		}
		return false, nil
	})
}

func DoTestUnschedulableNodes(t *testing.T, restClient *client.Client, nodeStore cache.Store) {
	goodCondition := api.NodeCondition{
		Type:              api.NodeReady,
		Status:            api.ConditionTrue,
		Reason:            fmt.Sprintf("schedulable condition"),
		LastHeartbeatTime: unversioned.Time{time.Now()},
	}
	badCondition := api.NodeCondition{
		Type:              api.NodeReady,
		Status:            api.ConditionUnknown,
		Reason:            fmt.Sprintf("unschedulable condition"),
		LastHeartbeatTime: unversioned.Time{time.Now()},
	}
	// Create a new schedulable node, since we're first going to apply
	// the unschedulable condition and verify that pods aren't scheduled.
	node := &api.Node{
		ObjectMeta: api.ObjectMeta{Name: "node-scheduling-test-node"},
		Spec:       api.NodeSpec{Unschedulable: false},
		Status: api.NodeStatus{
			Capacity: api.ResourceList{
				api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI),
			},
			Conditions: []api.NodeCondition{goodCondition},
		},
	}
	nodeKey, err := cache.MetaNamespaceKeyFunc(node)
	if err != nil {
		t.Fatalf("Couldn't retrieve key for node %v", node.Name)
	}

	// The test does the following for each nodeStateManager in this list:
	//	1. Create a new node
	//	2. Apply the makeUnSchedulable function
	//	3. Create a new pod
	//  4. Check that the pod doesn't get assigned to the node
	//  5. Apply the schedulable function
	//  6. Check that the pod *does* get assigned to the node
	//  7. Delete the pod and node.

	nodeModifications := []nodeStateManager{
		// Test node.Spec.Unschedulable=true/false
		{
			makeUnSchedulable: func(t *testing.T, n *api.Node, s cache.Store, c *client.Client) {
				n.Spec.Unschedulable = true
				if _, err := c.Nodes().Update(n); err != nil {
					t.Fatalf("Failed to update node with unschedulable=true: %v", err)
				}
				err = waitForReflection(s, nodeKey, func(node interface{}) bool {
					// An unschedulable node should get deleted from the store
					return node == nil
				})
				if err != nil {
					t.Fatalf("Failed to observe reflected update for setting unschedulable=true: %v", err)
				}
			},
			makeSchedulable: func(t *testing.T, n *api.Node, s cache.Store, c *client.Client) {
				n.Spec.Unschedulable = false
				if _, err := c.Nodes().Update(n); err != nil {
					t.Fatalf("Failed to update node with unschedulable=false: %v", err)
				}
				err = waitForReflection(s, nodeKey, func(node interface{}) bool {
					return node != nil && node.(*api.Node).Spec.Unschedulable == false
				})
				if err != nil {
					t.Fatalf("Failed to observe reflected update for setting unschedulable=false: %v", err)
				}
			},
		},
		// Test node.Status.Conditions=ConditionTrue/Unknown
		{
			makeUnSchedulable: func(t *testing.T, n *api.Node, s cache.Store, c *client.Client) {
				n.Status = api.NodeStatus{
					Capacity: api.ResourceList{
						api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI),
					},
					Conditions: []api.NodeCondition{badCondition},
				}
				if _, err = c.Nodes().UpdateStatus(n); err != nil {
					t.Fatalf("Failed to update node with bad status condition: %v", err)
				}
				err = waitForReflection(s, nodeKey, func(node interface{}) bool {
					return node != nil && node.(*api.Node).Status.Conditions[0].Status == api.ConditionUnknown
				})
				if err != nil {
					t.Fatalf("Failed to observe reflected update for status condition update: %v", err)
				}
			},
			makeSchedulable: func(t *testing.T, n *api.Node, s cache.Store, c *client.Client) {
				n.Status = api.NodeStatus{
					Capacity: api.ResourceList{
						api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI),
					},
					Conditions: []api.NodeCondition{goodCondition},
				}
				if _, err = c.Nodes().UpdateStatus(n); err != nil {
					t.Fatalf("Failed to update node with healthy status condition: %v", err)
				}
				waitForReflection(s, nodeKey, func(node interface{}) bool {
					return node != nil && node.(*api.Node).Status.Conditions[0].Status == api.ConditionTrue
				})
				if err != nil {
					t.Fatalf("Failed to observe reflected update for status condition update: %v", err)
				}
			},
		},
	}

	for i, mod := range nodeModifications {
		unSchedNode, err := restClient.Nodes().Create(node)
		if err != nil {
			t.Fatalf("Failed to create node: %v", err)
		}

		// Apply the unschedulable modification to the node, and wait for the reflection
		mod.makeUnSchedulable(t, unSchedNode, nodeStore, restClient)

		// Create the new pod, note that this needs to happen post unschedulable
		// modification or we have a race in the test.
		pod := &api.Pod{
			ObjectMeta: api.ObjectMeta{Name: "node-scheduling-test-pod"},
			Spec: api.PodSpec{
				Containers: []api.Container{{Name: "container", Image: "kubernetes/pause:go"}},
			},
		}
		myPod, err := restClient.Pods(api.NamespaceDefault).Create(pod)
		if err != nil {
			t.Fatalf("Failed to create pod: %v", err)
		}

		// There are no schedulable nodes - the pod shouldn't be scheduled.
		err = wait.Poll(time.Second, util.ForeverTestTimeout, podScheduled(restClient, myPod.Namespace, myPod.Name))
		if err == nil {
			t.Errorf("Pod scheduled successfully on unschedulable nodes")
		}
		if err != wait.ErrWaitTimeout {
			t.Errorf("Test %d: failed while trying to confirm the pod does not get scheduled on the node: %v", i, err)
		} else {
			t.Logf("Test %d: Pod did not get scheduled on an unschedulable node", i)
		}

		// Apply the schedulable modification to the node, and wait for the reflection
		schedNode, err := restClient.Nodes().Get(unSchedNode.Name)
		if err != nil {
			t.Fatalf("Failed to get node: %v", err)
		}
		mod.makeSchedulable(t, schedNode, nodeStore, restClient)

		// Wait until the pod is scheduled.
		err = wait.Poll(time.Second, util.ForeverTestTimeout, podScheduled(restClient, myPod.Namespace, myPod.Name))
		if err != nil {
			t.Errorf("Test %d: failed to schedule a pod: %v", i, err)
		} else {
			t.Logf("Test %d: Pod got scheduled on a schedulable node", i)
		}

		err = restClient.Pods(api.NamespaceDefault).Delete(myPod.Name, api.NewDeleteOptions(0))
		if err != nil {
			t.Errorf("Failed to delete pod: %v", err)
		}
		err = restClient.Nodes().Delete(schedNode.Name)
		if err != nil {
			t.Errorf("Failed to delete node: %v", err)
		}
	}
}
<commit_msg>add scheduler integration benchmark<commit_after>// +build integration,!no-etcd

/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

// This file tests the scheduler.

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"sync"
	"testing"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/resource"
	"k8s.io/kubernetes/pkg/api/testapi"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apiserver"
	"k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/client/record"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/master"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/plugin/pkg/admission/admit"
	"k8s.io/kubernetes/plugin/pkg/scheduler"
	_ "k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider"
	"k8s.io/kubernetes/plugin/pkg/scheduler/factory"
	"k8s.io/kubernetes/test/integration/framework"
)

type nodeMutationFunc func(t *testing.T, n *api.Node, nodeStore cache.Store, c *client.Client)

type nodeStateManager struct {
	makeSchedulable   nodeMutationFunc
	makeUnSchedulable nodeMutationFunc
}

func TestUnschedulableNodes(t *testing.T) {
	etcdStorage, err := framework.NewEtcdStorage()
	if err != nil {
		t.Fatalf("Couldn't create etcd storage: %v", err)
	}
	storageDestinations := master.NewStorageDestinations()
	storageDestinations.AddAPIGroup("", etcdStorage)
	framework.DeleteAllEtcdKeys()

	var m *master.Master
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		m.Handler.ServeHTTP(w, req)
	}))
	defer s.Close()

	m = master.New(&master.Config{
		StorageDestinations:   storageDestinations,
		KubeletClient:         client.FakeKubeletClient{},
		EnableCoreControllers: true,
		EnableLogsSupport:     false,
		EnableUISupport:       false,
		EnableIndex:           true,
		APIPrefix:             "/api",
		Authorizer:            apiserver.NewAlwaysAllowAuthorizer(),
		AdmissionControl:      admit.NewAlwaysAdmit(),
		StorageVersions:       map[string]string{"": testapi.Default.Version()},
	})

	restClient := client.NewOrDie(&client.Config{Host: s.URL, Version: testapi.Default.Version()})

	schedulerConfigFactory := factory.NewConfigFactory(restClient, nil)
	schedulerConfig, err := schedulerConfigFactory.Create()
	if err != nil {
		t.Fatalf("Couldn't create scheduler config: %v", err)
	}
	eventBroadcaster := record.NewBroadcaster()
	schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "scheduler"})
	eventBroadcaster.StartRecordingToSink(restClient.Events(""))
	scheduler.New(schedulerConfig).Run()

	defer close(schedulerConfig.StopEverything)

	DoTestUnschedulableNodes(t, restClient, schedulerConfigFactory.NodeLister.Store)
}

func podScheduled(c *client.Client, podNamespace, podName string) wait.ConditionFunc {
	return func() (bool, error) {
		pod, err := c.Pods(podNamespace).Get(podName)
		if errors.IsNotFound(err) {
			return false, nil
		}
		if err != nil {
			// This could be a connection error so we want to retry.
			return false, nil
		}
		if pod.Spec.NodeName == "" {
			return false, nil
		}
		return true, nil
	}
}

// Wait till the passFunc confirms that the object it expects to see is in the store.
// Used to observe reflected events.
func waitForReflection(s cache.Store, key string, passFunc func(n interface{}) bool) error {
	return wait.Poll(time.Millisecond*10, time.Second*20, func() (bool, error) {
		if n, _, err := s.GetByKey(key); err == nil && passFunc(n) {
			return true, nil
		}
		return false, nil
	})
}

func DoTestUnschedulableNodes(t *testing.T, restClient *client.Client, nodeStore cache.Store) {
	goodCondition := api.NodeCondition{
		Type:              api.NodeReady,
		Status:            api.ConditionTrue,
		Reason:            fmt.Sprintf("schedulable condition"),
		LastHeartbeatTime: unversioned.Time{time.Now()},
	}
	badCondition := api.NodeCondition{
		Type:              api.NodeReady,
		Status:            api.ConditionUnknown,
		Reason:            fmt.Sprintf("unschedulable condition"),
		LastHeartbeatTime: unversioned.Time{time.Now()},
	}
	// Create a new schedulable node, since we're first going to apply
	// the unschedulable condition and verify that pods aren't scheduled.
	node := &api.Node{
		ObjectMeta: api.ObjectMeta{Name: "node-scheduling-test-node"},
		Spec:       api.NodeSpec{Unschedulable: false},
		Status: api.NodeStatus{
			Capacity: api.ResourceList{
				api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI),
			},
			Conditions: []api.NodeCondition{goodCondition},
		},
	}
	nodeKey, err := cache.MetaNamespaceKeyFunc(node)
	if err != nil {
		t.Fatalf("Couldn't retrieve key for node %v", node.Name)
	}

	// The test does the following for each nodeStateManager in this list:
	//	1. Create a new node
	//	2. Apply the makeUnSchedulable function
	//	3. Create a new pod
	//  4. Check that the pod doesn't get assigned to the node
	//  5. Apply the schedulable function
	//  6. Check that the pod *does* get assigned to the node
	//  7. Delete the pod and node.

	nodeModifications := []nodeStateManager{
		// Test node.Spec.Unschedulable=true/false
		{
			makeUnSchedulable: func(t *testing.T, n *api.Node, s cache.Store, c *client.Client) {
				n.Spec.Unschedulable = true
				if _, err := c.Nodes().Update(n); err != nil {
					t.Fatalf("Failed to update node with unschedulable=true: %v", err)
				}
				err = waitForReflection(s, nodeKey, func(node interface{}) bool {
					// An unschedulable node should get deleted from the store
					return node == nil
				})
				if err != nil {
					t.Fatalf("Failed to observe reflected update for setting unschedulable=true: %v", err)
				}
			},
			makeSchedulable: func(t *testing.T, n *api.Node, s cache.Store, c *client.Client) {
				n.Spec.Unschedulable = false
				if _, err := c.Nodes().Update(n); err != nil {
					t.Fatalf("Failed to update node with unschedulable=false: %v", err)
				}
				err = waitForReflection(s, nodeKey, func(node interface{}) bool {
					return node != nil && node.(*api.Node).Spec.Unschedulable == false
				})
				if err != nil {
					t.Fatalf("Failed to observe reflected update for setting unschedulable=false: %v", err)
				}
			},
		},
		// Test node.Status.Conditions=ConditionTrue/Unknown
		{
			makeUnSchedulable: func(t *testing.T, n *api.Node, s cache.Store, c *client.Client) {
				n.Status = api.NodeStatus{
					Capacity: api.ResourceList{
						api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI),
					},
					Conditions: []api.NodeCondition{badCondition},
				}
				if _, err = c.Nodes().UpdateStatus(n); err != nil {
					t.Fatalf("Failed to update node with bad status condition: %v", err)
				}
				err = waitForReflection(s, nodeKey, func(node interface{}) bool {
					return node != nil && node.(*api.Node).Status.Conditions[0].Status == api.ConditionUnknown
				})
				if err != nil {
					t.Fatalf("Failed to observe reflected update for status condition update: %v", err)
				}
			},
			makeSchedulable: func(t *testing.T, n *api.Node, s cache.Store, c *client.Client) {
				n.Status = api.NodeStatus{
					Capacity: api.ResourceList{
						api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI),
					},
					Conditions: []api.NodeCondition{goodCondition},
				}
				if _, err = c.Nodes().UpdateStatus(n); err != nil {
					t.Fatalf("Failed to update node with healthy status condition: %v", err)
				}
				waitForReflection(s, nodeKey, func(node interface{}) bool {
					return node != nil && node.(*api.Node).Status.Conditions[0].Status == api.ConditionTrue
				})
				if err != nil {
					t.Fatalf("Failed to observe reflected update for status condition update: %v", err)
				}
			},
		},
	}

	for i, mod := range nodeModifications {
		unSchedNode, err := restClient.Nodes().Create(node)
		if err != nil {
			t.Fatalf("Failed to create node: %v", err)
		}

		// Apply the unschedulable modification to the node, and wait for the reflection
		mod.makeUnSchedulable(t, unSchedNode, nodeStore, restClient)

		// Create the new pod, note that this needs to happen post unschedulable
		// modification or we have a race in the test.
		pod := &api.Pod{
			ObjectMeta: api.ObjectMeta{Name: "node-scheduling-test-pod"},
			Spec: api.PodSpec{
				Containers: []api.Container{{Name: "container", Image: "kubernetes/pause:go"}},
			},
		}
		myPod, err := restClient.Pods(api.NamespaceDefault).Create(pod)
		if err != nil {
			t.Fatalf("Failed to create pod: %v", err)
		}

		// There are no schedulable nodes - the pod shouldn't be scheduled.
		err = wait.Poll(time.Second, util.ForeverTestTimeout, podScheduled(restClient, myPod.Namespace, myPod.Name))
		if err == nil {
			t.Errorf("Pod scheduled successfully on unschedulable nodes")
		}
		if err != wait.ErrWaitTimeout {
			t.Errorf("Test %d: failed while trying to confirm the pod does not get scheduled on the node: %v", i, err)
		} else {
			t.Logf("Test %d: Pod did not get scheduled on an unschedulable node", i)
		}

		// Apply the schedulable modification to the node, and wait for the reflection
		schedNode, err := restClient.Nodes().Get(unSchedNode.Name)
		if err != nil {
			t.Fatalf("Failed to get node: %v", err)
		}
		mod.makeSchedulable(t, schedNode, nodeStore, restClient)

		// Wait until the pod is scheduled.
		err = wait.Poll(time.Second, util.ForeverTestTimeout, podScheduled(restClient, myPod.Namespace, myPod.Name))
		if err != nil {
			t.Errorf("Test %d: failed to schedule a pod: %v", i, err)
		} else {
			t.Logf("Test %d: Pod got scheduled on a schedulable node", i)
		}

		err = restClient.Pods(api.NamespaceDefault).Delete(myPod.Name, api.NewDeleteOptions(0))
		if err != nil {
			t.Errorf("Failed to delete pod: %v", err)
		}
		err = restClient.Nodes().Delete(schedNode.Name)
		if err != nil {
			t.Errorf("Failed to delete node: %v", err)
		}
	}
}

func BenchmarkScheduling(b *testing.B) {
	etcdStorage, err := framework.NewEtcdStorage()
	if err != nil {
		b.Fatalf("Couldn't create etcd storage: %v", err)
	}
	storageDestinations := master.NewStorageDestinations()
	storageDestinations.AddAPIGroup("", etcdStorage)
	framework.DeleteAllEtcdKeys()

	var m *master.Master
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		m.Handler.ServeHTTP(w, req)
	}))
	defer s.Close()

	m = master.New(&master.Config{
		StorageDestinations:   storageDestinations,
		KubeletClient:         client.FakeKubeletClient{},
		EnableCoreControllers: true,
		EnableLogsSupport:     false,
		EnableUISupport:       false,
		EnableIndex:           true,
		APIPrefix:             "/api",
		Authorizer:            apiserver.NewAlwaysAllowAuthorizer(),
		AdmissionControl:      admit.NewAlwaysAdmit(),
		StorageVersions:       map[string]string{"": testapi.Default.Version()},
	})

	c := client.NewOrDie(&client.Config{
		Host:    s.URL,
		Version: testapi.Default.Version(),
		QPS:     5000.0,
		Burst:   5000,
	})

	schedulerConfigFactory := factory.NewConfigFactory(c, nil)
	schedulerConfig, err := schedulerConfigFactory.Create()
	if err != nil {
		b.Fatalf("Couldn't create scheduler config: %v", err)
	}
	eventBroadcaster := record.NewBroadcaster()
	schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "scheduler"})
	eventBroadcaster.StartRecordingToSink(c.Events(""))
	scheduler.New(schedulerConfig).Run()

	defer close(schedulerConfig.StopEverything)

	makeNNodes(c, 1000)
	N := b.N
	b.ResetTimer()
	makeNPods(c, N)
	for {
		objs := schedulerConfigFactory.ScheduledPodLister.Store.List()
		if len(objs) >= N {
			fmt.Printf("%v pods scheduled.\n", len(objs))
			/* // To prove that this actually works:
			for _, o := range objs {
				fmt.Printf("%s\n", o.(*api.Pod).Spec.NodeName)
			}
			*/
			break
		}
		time.Sleep(time.Millisecond)
	}
	b.StopTimer()
}

func makeNNodes(c client.Interface, N int) {
	baseNode := &api.Node{
		ObjectMeta: api.ObjectMeta{
			GenerateName: "scheduler-test-node-",
		},
		Spec: api.NodeSpec{
			ExternalID: "foobar",
		},
		Status: api.NodeStatus{
			Capacity: api.ResourceList{
				api.ResourcePods:   *resource.NewQuantity(32, resource.DecimalSI),
				api.ResourceCPU:    resource.MustParse("4"),
				api.ResourceMemory: resource.MustParse("32Gi"),
			},
			Phase: api.NodeRunning,
			Conditions: []api.NodeCondition{
				{Type: api.NodeReady, Status: api.ConditionTrue},
			},
		},
	}
	for i := 0; i < N; i++ {
		if _, err := c.Nodes().Create(baseNode); err != nil {
			panic("error creating node: " + err.Error())
		}
	}
}

func makeNPods(c client.Interface, N int) {
	basePod := &api.Pod{
		ObjectMeta: api.ObjectMeta{
			GenerateName: "scheduler-test-pod-",
		},
		Spec: api.PodSpec{
			Containers: []api.Container{{
				Name:  "pause",
				Image: "gcr.io/google_containers/pause:1.0",
				Resources: api.ResourceRequirements{
					Limits: api.ResourceList{
						api.ResourceCPU:    resource.MustParse("100m"),
						api.ResourceMemory: resource.MustParse("500Mi"),
					},
					Requests: api.ResourceList{
						api.ResourceCPU:    resource.MustParse("100m"),
						api.ResourceMemory: resource.MustParse("500Mi"),
					},
				},
			}},
		},
	}
	wg := sync.WaitGroup{}
	threads := 30
	wg.Add(threads)
	remaining := make(chan int, N)
	go func() {
		for i := 0; i < N; i++ {
			remaining <- i
		}
		close(remaining)
	}()
	for i := 0; i < threads; i++ {
		go func() {
			defer wg.Done()
			for {
				_, ok := <-remaining
				if !ok {
					return
				}
				for {
					_, err := c.Pods("default").Create(basePod)
					if err == nil {
						break
					}
				}
			}
		}()
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>/*
xslate is an extremely powerful template engine, based on Perl5's Text::Xslate
module. Xslate uses a virtual machine to execute pre-compiled template bytecode,
which gives its flexibility while maitaining a very fast execution speed.

Note that RenderString() DOES NOT CACHE THE GENERATED BYTECODE. This has
significant effect on performance if you repeatedly call the same template

*/
package xslate

import (
  "errors"
  "fmt"
  "io/ioutil"
  "os"
  "reflect"

  "github.com/lestrrat/go-xslate/compiler"
  "github.com/lestrrat/go-xslate/loader"
  "github.com/lestrrat/go-xslate/parser"
  "github.com/lestrrat/go-xslate/parser/tterse"
  "github.com/lestrrat/go-xslate/vm"
)

type Vars vm.Vars
type Xslate struct {
  Flags    int32
  Vm       *vm.VM
  Compiler compiler.Compiler
  Parser   parser.Parser
  Loader   loader.ByteCodeLoader
  // XXX Need to make syntax pluggable
}

type ConfigureArgs interface {
  Get(string) (interface {}, bool)
}

type Args map[string]interface {}

// Given an unconfigured Xslate instance and arguments, sets up
// the compiler of said Xslate instance. Current implementation
// just uses compiler.New()
func DefaultCompiler(tx *Xslate, args Args) error {
  tx.Compiler = compiler.New()
  return nil
}

func DefaultParser(tx *Xslate, args Args) error {
  syntax, ok := args.Get("Syntax")
  if ! ok {
    syntax = "TTerse"
  }

  switch syntax {
  case "TTerse":
    tx.Parser = tterse.New()
  default:
    return errors.New(fmt.Sprintf("Syntax '%s' not available", syntax))
  }
  return nil
}

func DefaultLoader(tx *Xslate, args Args) error {
  var tmp interface {}

  tmp, ok := args.Get("CacheDir")
  if !ok {
    tmp, _ = ioutil.TempDir("", "go-xslate-cache-")
  }
  cacheDir := tmp.(string)

  tmp, ok = args.Get("LoadPaths")
  if !ok {
    cwd, _ := os.Getwd()
    tmp = []string { cwd }
  }
  paths := tmp.([]string)

  cache, err := loader.NewFileCache(cacheDir)
  if err != nil {
    return err
  }
  fetcher, err := loader.NewFileTemplateFetcher(paths)
  if err != nil {
    return err
  }
  tx.Loader = loader.NewCachedByteCodeLoader(cache, fetcher, tx.Parser, tx.Compiler)
  return nil
}

func DefaultVm(tx *Xslate, args Args) error {
  tx.Vm = vm.NewVM()
  tx.Vm.Loader = tx.Loader
  return nil
}

func (args Args) Get(key string) (interface {}, bool) {
  ret, ok := args[key]
  return ret, ok
}

func (tx *Xslate) configureGeneric(configuror interface {}, args Args) error {
  ref := reflect.ValueOf(configuror)
  switch ref.Type().Kind() {
  case reflect.Func:
    // If this is a function, it better take our Xslate instance as the
    // sole argument, and initialize it as it pleases
    if ref.Type().NumIn() != 2 && (ref.Type().In(0).Name() != "Xslate" || ref.Type().In(1).Name() != "Args") {
      panic(fmt.Sprintf(`Expected function initializer "func (tx *Xslate ", but instead of %s`, ref.Type.String()))
    }
    cb := configuror.(func(*Xslate, Args) error)
    err := cb(tx, args)
    return err
  }
  return errors.New("Bad configurator")
}

func (tx *Xslate) Configure(args ConfigureArgs) error {
  // The compiler currently does not have any configurable options, but
  // one may want to replace the entire compiler struct
  defaults := map[string]func(*Xslate, Args) error {
    "Compiler": DefaultCompiler,
    "Parser":   DefaultParser,
    "Loader":   DefaultLoader,
    "Vm":       DefaultVm,
  }

  for _, key := range []string { "Parser", "Compiler", "Loader", "Vm" } {
    configKey := "Configure" + key
    configuror, ok := args.Get(configKey);
    if !ok {
      configuror = defaults[key]
    }

    args, ok := args.Get(key)
    if !ok {
      args = Args {}
    }

    err := tx.configureGeneric(configuror, args.(Args))
    if err != nil {
      return err
    }
  }

  return nil
}

func New(args ...Args) (*Xslate, error) {
  tx := &Xslate {}

  // We jump through hoops because there are A LOT of configuration options
  // but most of them only need to use the default values
  if len(args) <= 0 {
    args = []Args { Args {} }
  }
  err := tx.Configure(args[0])
  if err != nil {
    return nil, err
  }
  return tx, nil
}

func (tx *Xslate) DumpAST(b bool) {
  tx.Loader.DumpAST(b)
}

func (tx *Xslate) DumpByteCode(b bool) {
  tx.Loader.DumpByteCode(b)
}

func (x *Xslate) Render(name string, vars Vars) (string, error) {
  bc, err := x.Loader.Load(name)
  if err != nil {
    return "", err
  }
  x.Vm.Run(bc, vm.Vars(vars))
  return x.Vm.OutputString()
}

func (x *Xslate) RenderString(template string, vars Vars) (string, error) {
  bc, err := x.Loader.LoadString(template)
  if err != nil {
    return "", err
  }

  x.Vm.Run(bc, vm.Vars(vars))
  return x.Vm.OutputString()
}
<commit_msg>Revert "Respect go vet"<commit_after>/*
xslate is an extremely powerful template engine, based on Perl5's Text::Xslate
module. Xslate uses a virtual machine to execute pre-compiled template bytecode,
which gives its flexibility while maitaining a very fast execution speed.

Note that RenderString() DOES NOT CACHE THE GENERATED BYTECODE. This has
significant effect on performance if you repeatedly call the same template

*/
package xslate

import (
  "errors"
  "fmt"
  "io/ioutil"
  "os"
  "reflect"

  "github.com/lestrrat/go-xslate/compiler"
  "github.com/lestrrat/go-xslate/loader"
  "github.com/lestrrat/go-xslate/parser"
  "github.com/lestrrat/go-xslate/parser/tterse"
  "github.com/lestrrat/go-xslate/vm"
)

type Vars vm.Vars
type Xslate struct {
  Flags    int32
  Vm       *vm.VM
  Compiler compiler.Compiler
  Parser   parser.Parser
  Loader   loader.ByteCodeLoader
  // XXX Need to make syntax pluggable
}

type ConfigureArgs interface {
  Get(string) (interface {}, bool)
}

type Args map[string]interface {}

// Given an unconfigured Xslate instance and arguments, sets up
// the compiler of said Xslate instance. Current implementation
// just uses compiler.New()
func DefaultCompiler(tx *Xslate, args Args) error {
  tx.Compiler = compiler.New()
  return nil
}

func DefaultParser(tx *Xslate, args Args) error {
  syntax, ok := args.Get("Syntax")
  if ! ok {
    syntax = "TTerse"
  }

  switch syntax {
  case "TTerse":
    tx.Parser = tterse.New()
  default:
    return errors.New(fmt.Sprintf("Syntax '%s' not available", syntax))
  }
  return nil
}

func DefaultLoader(tx *Xslate, args Args) error {
  var tmp interface {}

  tmp, ok := args.Get("CacheDir")
  if !ok {
    tmp, _ = ioutil.TempDir("", "go-xslate-cache-")
  }
  cacheDir := tmp.(string)

  tmp, ok = args.Get("LoadPaths")
  if !ok {
    cwd, _ := os.Getwd()
    tmp = []string { cwd }
  }
  paths := tmp.([]string)

  cache, err := loader.NewFileCache(cacheDir)
  if err != nil {
    return err
  }
  fetcher, err := loader.NewFileTemplateFetcher(paths)
  if err != nil {
    return err
  }
  tx.Loader = loader.NewCachedByteCodeLoader(cache, fetcher, tx.Parser, tx.Compiler)
  return nil
}

func DefaultVm(tx *Xslate, args Args) error {
  tx.Vm = vm.NewVM()
  tx.Vm.Loader = tx.Loader
  return nil
}

func (args Args) Get(key string) (interface {}, bool) {
  ret, ok := args[key]
  return ret, ok
}

func (tx *Xslate) configureGeneric(configuror interface {}, args Args) error {
  ref := reflect.ValueOf(configuror)
  switch ref.Type().Kind() {
  case reflect.Func:
    // If this is a function, it better take our Xslate instance as the
    // sole argument, and initialize it as it pleases
    if ref.Type().NumIn() != 2 && (ref.Type().In(0).Name() != "Xslate" || ref.Type().In(1).Name() != "Args") {
      panic(fmt.Sprintf(`Expected function initializer "func (tx *Xslate ", but instead of %s`, ref.Type))
    }
    cb := configuror.(func(*Xslate, Args) error)
    err := cb(tx, args)
    return err
  }
  return errors.New("Bad configurator")
}

func (tx *Xslate) Configure(args ConfigureArgs) error {
  // The compiler currently does not have any configurable options, but
  // one may want to replace the entire compiler struct
  defaults := map[string]func(*Xslate, Args) error {
    "Compiler": DefaultCompiler,
    "Parser":   DefaultParser,
    "Loader":   DefaultLoader,
    "Vm":       DefaultVm,
  }

  for _, key := range []string { "Parser", "Compiler", "Loader", "Vm" } {
    configKey := "Configure" + key
    configuror, ok := args.Get(configKey);
    if !ok {
      configuror = defaults[key]
    }

    args, ok := args.Get(key)
    if !ok {
      args = Args {}
    }

    err := tx.configureGeneric(configuror, args.(Args))
    if err != nil {
      return err
    }
  }

  return nil
}

func New(args ...Args) (*Xslate, error) {
  tx := &Xslate {}

  // We jump through hoops because there are A LOT of configuration options
  // but most of them only need to use the default values
  if len(args) <= 0 {
    args = []Args { Args {} }
  }
  err := tx.Configure(args[0])
  if err != nil {
    return nil, err
  }
  return tx, nil
}

func (tx *Xslate) DumpAST(b bool) {
  tx.Loader.DumpAST(b)
}

func (tx *Xslate) DumpByteCode(b bool) {
  tx.Loader.DumpByteCode(b)
}

func (x *Xslate) Render(name string, vars Vars) (string, error) {
  bc, err := x.Loader.Load(name)
  if err != nil {
    return "", err
  }
  x.Vm.Run(bc, vm.Vars(vars))
  return x.Vm.OutputString()
}

func (x *Xslate) RenderString(template string, vars Vars) (string, error) {
  bc, err := x.Loader.LoadString(template)
  if err != nil {
    return "", err
  }

  x.Vm.Run(bc, vm.Vars(vars))
  return x.Vm.OutputString()
}
<|endoftext|>


<commit_before>package servactive

import (
	"fmt"
	"strings"

	"github.com/containerum/chkit/pkg/model/service"
	"github.com/containerum/chkit/pkg/util/namegen"

	"github.com/containerum/chkit/pkg/chkitErrors"
)

const (
	ErrUserStoppedSession   chkitErrors.Err = "user stopped session"
	ErrInvalidSymbolInLabel chkitErrors.Err = "invalid symbol in label"
	defaultString                           = "undefined"
)

type ConstructorConfig struct {
	Force bool
}

func RunInteractveConstructor(config ConstructorConfig) (service.Service, error) {
	fmt.Printf("Hi there!\n")
	if !config.Force {
		ok, _ := yes("Do you want to create service?")
		if !ok {
			return service.Service{}, ErrUserStoppedSession
		}
	}
	fmt.Printf("OK\n")
	serv, err := fillServiceField()
	return serv, err
}

func fillServiceField() (service.Service, error) {
	const (
		name = iota
		domain
		ips
		ports
		deploy
	)
	serv := defaultService()
	for {
		fields := []string{
			fmt.Sprintf("Name  : %s", serv.Name),
			fmt.Sprintf("Domain: %s", serv.Domain),
			fmt.Sprintf("IPs   : [%s]", strings.Join(serv.IPs, ", ")),
			fmt.Sprintf("Ports : %v", service.PortList(serv.Ports)),
			fmt.Sprintf("Deploy: %s", serv.Deploy),
		}
		field, ok := askFieldToChange(fields)
		if !ok {
			return serv, ErrUserStoppedSession
		}
		switch field {
		case name:
			name, err := getName(serv.Name)
			if err != nil {
				return serv, err
			}
			serv.Name = name
		case ports:
			ports, err := getPorts()
			if err != nil {
				return serv, err
			}
			serv.Ports = ports
		case domain:
		case ips:
			IPs, err := getIPs()
			if err != nil {
				return serv, err
			}
			serv.IPs = IPs
		case deploy:
		default:
			panic("[service interactive constructor] unreacheable state in field selection func")
		}
	}
	return serv, nil
}

func defaultService() service.Service {
	return service.Service{
		Name:   namegen.ColoredPhysics(),
		Domain: defaultString,
		IPs:    nil,
		Ports:  nil,
		Deploy: defaultString,
	}
}
<commit_msg>add domain getters<commit_after>package servactive

import (
	"fmt"
	"strings"

	"github.com/containerum/chkit/pkg/model/service"
	"github.com/containerum/chkit/pkg/util/namegen"

	"github.com/containerum/chkit/pkg/chkitErrors"
)

const (
	ErrUserStoppedSession   chkitErrors.Err = "user stopped session"
	ErrInvalidSymbolInLabel chkitErrors.Err = "invalid symbol in label"
	defaultString                           = "undefined"
)

type ConstructorConfig struct {
	Force bool
}

func RunInteractveConstructor(config ConstructorConfig) (service.Service, error) {
	fmt.Printf("Hi there!\n")
	if !config.Force {
		ok, _ := yes("Do you want to create service?")
		if !ok {
			return service.Service{}, ErrUserStoppedSession
		}
	}
	fmt.Printf("OK\n")
	serv, err := fillServiceField()
	return serv, err
}

func fillServiceField() (service.Service, error) {
	const (
		name = iota
		domain
		ips
		ports
		deploy
	)
	serv := defaultService()
	for {
		fields := []string{
			fmt.Sprintf("Name  : %s", serv.Name),
			fmt.Sprintf("Domain: %s", serv.Domain),
			fmt.Sprintf("IPs   : [%s]", strings.Join(serv.IPs, ", ")),
			fmt.Sprintf("Ports : %v", service.PortList(serv.Ports)),
			fmt.Sprintf("Deploy: %s", serv.Deploy),
		}
		field, ok := askFieldToChange(fields)
		if !ok {
			return serv, ErrUserStoppedSession
		}
		switch field {
		case name:
			name, err := getName(serv.Name)
			if err != nil {
				return serv, err
			}
			serv.Name = name
		case ports:
			ports, err := getPorts()
			if err != nil {
				return serv, err
			}
			serv.Ports = ports
		case domain:
			domain, err := getDomain()
			if err != nil {
				return serv, err
			}
			serv.Domain = domain
		case ips:
			IPs, err := getIPs()
			if err != nil {
				return serv, err
			}
			serv.IPs = IPs
		case deploy:
			deploy, err := getDeploy()
			if err != nil {
				return serv, err
			}
			serv.Deploy = deploy
		default:
			panic("[service interactive constructor] unreacheable state in field selection func")
		}
	}
}

func defaultService() service.Service {
	return service.Service{
		Name:   namegen.ColoredPhysics(),
		Domain: "undefined (optional)",
		IPs:    nil,
		Ports:  nil,
		Deploy: "undefined (required)",
	}
}
<|endoftext|>