<commit_before>package main

import (
	rdb "github.com/dancannon/gorethink"
	"github.com/go-martini/martini"
	"github.com/martini-contrib/render"
	"github.com/martini-contrib/sessions"
	"net/http"
	"os"
	"strings"
	"time"
)

// Middleware function hooks the RethinkDB to be accessible for Martini routes.
// By default the middleware spawns a session pool of 10 connections.
// Typical connection options on development environment would be
//		Address: "localhost:28015"
//		Database: "test"
func middleware() martini.Handler {
	session, err := rdb.Connect(rdb.ConnectOpts{
		Address:     os.Getenv("rDB"),
		Database:    os.Getenv("rNAME"),
		MaxIdle:     10,
		IdleTimeout: time.Second * 10,
	})

	if err != nil {
		panic(err)
	}

	return func(c martini.Context) {
		c.Map(session)
	}
}

// Checks that session cookie with label "user" exists and is valid.
func sessionIsAlive(session sessions.Session) bool {
	data := session.Get("user")
	_, exists := data.(string)
	if exists {
		return true
	}
	return false
}

// Checks whther session cookie with label "user" exists and is valid.
// If true, redirects to user's profile root.
// Useful for redirecting from pages which are only visible when logged out,
// for example login and register pages.
func SessionRedirect(c martini.Context, res http.ResponseWriter, req *http.Request, r render.Render, session sessions.Session) {
	if sessionIsAlive(session) {
		http.Redirect(res, req, "/user", 302)
	}
}

// Makes sure that the user is logged in. Use on pages which need authentication
// or which have to deal with user structure later on.
func ProtectedPage(res http.ResponseWriter, req *http.Request, session sessions.Session) {
	if !sessionIsAlive(session) {
		session.Delete("user")
		http.Redirect(res, req, "/", 302)
	}
}

// Returns request "root".
// For example, calling it with http.Request which has URL of /api/user/5348482a2142dfb84ca41085
// would return "api". This function is used to route both JSON API and frontend requests in the same function.
func root(req *http.Request) string {
	return strings.Split(strings.TrimPrefix(req.URL.String(), "/"), "/")[0]
}
<commit_msg>misc: delete unused middleware parameters and dependencies<commit_after>package main

import (
	r "github.com/dancannon/gorethink"
	"github.com/go-martini/martini"
	"github.com/martini-contrib/sessions"
	"net/http"
	"os"
	"strings"
	"time"
)

// Middleware function hooks the RethinkDB to be accessible for Martini routes.
// By default the middleware spawns a session pool of 10 connections.
// Typical connection options on development environment would be
//		Address: "localhost:28015"
//		Database: "test"
func middleware() martini.Handler {
	session, err := r.Connect(r.ConnectOpts{
		Address:     os.Getenv("rDB"),
		Database:    os.Getenv("rNAME"),
		MaxIdle:     10,
		IdleTimeout: time.Second * 10,
	})

	if err != nil {
		panic(err)
	}

	return func(c martini.Context) {
		c.Map(session)
	}
}

// Checks that session cookie with label "user" exists and is valid.
func sessionIsAlive(session sessions.Session) bool {
	data := session.Get("user")
	_, exists := data.(string)
	if exists {
		return true
	}
	return false
}

// Checks whther session cookie with label "user" exists and is valid.
// If true, redirects to user's profile root.
// Useful for redirecting from pages which are only visible when logged out,
// for example login and register pages.
func SessionRedirect(res http.ResponseWriter, req *http.Request, session sessions.Session) {
	if sessionIsAlive(session) {
		http.Redirect(res, req, "/user", 302)
	}
}

// Makes sure that the user is logged in. Use on pages which need authentication
// or which have to deal with user structure later on.
func ProtectedPage(res http.ResponseWriter, req *http.Request, session sessions.Session) {
	if !sessionIsAlive(session) {
		session.Delete("user")
		http.Redirect(res, req, "/", 302)
	}
}

// Returns request "root".
// For example, calling it with http.Request which has URL of /api/user/5348482a2142dfb84ca41085
// would return "api". This function is used to route both JSON API and frontend requests in the same function.
func root(req *http.Request) string {
	return strings.Split(strings.TrimPrefix(req.URL.String(), "/"), "/")[0]
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"github.com/albertyw/devops-reactions-index/tumblr"
	_ "github.com/joho/godotenv/autoload"
	"io/ioutil"
	"net/http"
	"os"
)

const dataURLPath = "/data.json"

var templateDir = os.Getenv("SERVER_TEMPLATES")
var indexPath = fmt.Sprintf("%s/index.htm", templateDir)
var uRLFilePaths = map[string]func() (string, error){}
var posts []tumblr.Post

// exactURL is a closure that checks that the http match is an exact url path
// match instead of allowing for net/http's loose match
func exactURL(
	targetFunc func(http.ResponseWriter, *http.Request),
	requestedPath string,
) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != requestedPath {
			http.NotFound(w, r)
			return
		}
		targetFunc(w, r)
		return
	}
}

// readFile returns a function that reads the file at a given path and makes a
// response from it
func readFile(p string) func(http.ResponseWriter, *http.Request) {
	path := p
	return func(w http.ResponseWriter, r *http.Request) {
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return
		}
		html := string(data)
		fmt.Fprintf(w, html)
	}
}

// dataURLHandler is an http handler for the dataURLPath response
func dataURLHandler(w http.ResponseWriter, r *http.Request) {
	html := tumblr.PostsToJSON(posts)
	fmt.Fprintf(w, html)
}

// Run starts up the HTTP server
func Run(p []tumblr.Post) {
	posts = p
	address := ":" + os.Getenv("PORT")
	fmt.Println("server listening on", address)
	http.HandleFunc("/", exactURL(readFile(indexPath), "/"))
	http.HandleFunc(dataURLPath, dataURLHandler)
	http.ListenAndServe(address, nil)
}
<commit_msg>Fix lint errors<commit_after>package server

import (
	"fmt"
	"github.com/albertyw/devops-reactions-index/tumblr"
	// Used for getting tumblr env vars
	_ "github.com/joho/godotenv/autoload"
	"io/ioutil"
	"net/http"
	"os"
)

const dataURLPath = "/data.json"

var templateDir = os.Getenv("SERVER_TEMPLATES")
var indexPath = fmt.Sprintf("%s/index.htm", templateDir)
var uRLFilePaths = map[string]func() (string, error){}
var posts []tumblr.Post

// exactURL is a closure that checks that the http match is an exact url path
// match instead of allowing for net/http's loose match
func exactURL(
	targetFunc func(http.ResponseWriter, *http.Request),
	requestedPath string,
) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != requestedPath {
			http.NotFound(w, r)
			return
		}
		targetFunc(w, r)
		return
	}
}

// readFile returns a function that reads the file at a given path and makes a
// response from it
func readFile(p string) func(http.ResponseWriter, *http.Request) {
	path := p
	return func(w http.ResponseWriter, r *http.Request) {
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return
		}
		html := string(data)
		fmt.Fprintf(w, html)
	}
}

// dataURLHandler is an http handler for the dataURLPath response
func dataURLHandler(w http.ResponseWriter, r *http.Request) {
	html := tumblr.PostsToJSON(posts)
	fmt.Fprintf(w, html)
}

// Run starts up the HTTP server
func Run(p []tumblr.Post) {
	posts = p
	address := ":" + os.Getenv("PORT")
	fmt.Println("server listening on", address)
	http.HandleFunc("/", exactURL(readFile(indexPath), "/"))
	http.HandleFunc(dataURLPath, dataURLHandler)
	http.ListenAndServe(address, nil)
}
<|endoftext|>


<commit_before>package release

import (
	"bytes"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/goreleaser/goreleaser/clients"
	"github.com/goreleaser/goreleaser/config"
	"github.com/goreleaser/goreleaser/context"
	"github.com/stretchr/testify/assert"
)

func TestPipeDescription(t *testing.T) {
	assert.NotEmpty(t, Pipe{}.Description())
}

func TestRunPipe(t *testing.T) {
	assert := assert.New(t)
	folder, err := ioutil.TempDir("", "gorelasertest")
	assert.NoError(err)
	_, err = os.Create(filepath.Join(folder, "bin.tar.gz"))
	assert.NoError(err)
	_, err = os.Create(filepath.Join(folder, "bin.deb"))
	assert.NoError(err)
	var ctx = &context.Context{
		Git: context.GitInfo{
			CurrentTag: "v1.0.0",
		},
		Config: config.Project{
			Dist: folder,
			Archive: config.Archive{
				Format: "tar.gz",
			},
			Release: config.Release{
				GitHub: config.Repo{
					Owner: "test",
					Name:  "test",
				},
			},
			FPM: config.FPM{
				Formats: []string{
					"deb",
				},
			},
		},
		Archives: map[string]string{
			"darwinamd64": "bin",
		},
	}
	client := &DummyClient{}
	assert.NoError(doRun(ctx, client))
	assert.True(client.CreatedRelease)
	assert.True(client.UploadedFile)
}

type DummyClient struct {
	CreatedRelease bool
	UploadedFile   bool
}

func (client *DummyClient) GetInfo(ctx *context.Context) (info clients.Info, err error) {
	return
}

func (client *DummyClient) CreateRelease(ctx *context.Context) (releaseID int, err error) {
	client.CreatedRelease = true
	return
}

func (client *DummyClient) CreateFile(ctx *context.Context, content bytes.Buffer, path string) (err error) {
	return
}

func (client *DummyClient) Upload(ctx *context.Context, releaseID int, name string, file *os.File) (err error) {
	client.UploadedFile = true
	return
}
<commit_msg>fixed broken test<commit_after>package release

import (
	"bytes"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/goreleaser/goreleaser/clients"
	"github.com/goreleaser/goreleaser/config"
	"github.com/goreleaser/goreleaser/context"
	"github.com/stretchr/testify/assert"
)

func TestPipeDescription(t *testing.T) {
	assert.NotEmpty(t, Pipe{}.Description())
}

func TestRunPipe(t *testing.T) {
	assert := assert.New(t)
	folder, err := ioutil.TempDir("", "gorelasertest")
	assert.NoError(err)
	tarfile, err := os.Create(filepath.Join(folder, "bin.tar.gz"))
	assert.NoError(err)
	debfile, err := os.Create(filepath.Join(folder, "bin.deb"))
	assert.NoError(err)
	var ctx = &context.Context{
		Git: context.GitInfo{
			CurrentTag: "v1.0.0",
		},
		Config: config.Project{
			Dist: folder,
			Release: config.Release{
				GitHub: config.Repo{
					Owner: "test",
					Name:  "test",
				},
			},
		},
	}
	ctx.AddArtifact(tarfile.Name())
	ctx.AddArtifact(debfile.Name())
	client := &DummyClient{}
	assert.NoError(doRun(ctx, client))
	assert.True(client.CreatedRelease)
	assert.True(client.UploadedFile)
}

type DummyClient struct {
	CreatedRelease bool
	UploadedFile   bool
}

func (client *DummyClient) GetInfo(ctx *context.Context) (info clients.Info, err error) {
	return
}

func (client *DummyClient) CreateRelease(ctx *context.Context) (releaseID int, err error) {
	client.CreatedRelease = true
	return
}

func (client *DummyClient) CreateFile(ctx *context.Context, content bytes.Buffer, path string) (err error) {
	return
}

func (client *DummyClient) Upload(ctx *context.Context, releaseID int, name string, file *os.File) (err error) {
	client.UploadedFile = true
	return
}
<|endoftext|>


<commit_before>package veneur

import (
	"bytes"
	"compress/zlib"
	"encoding/json"
	"math/rand"
	"net/http"
	"net/http/httptest"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

// set up a boilerplate local config for later use
func localConfig() Config {
	return Config{
		APIHostname:         "http://localhost",
		Debug:               false,
		Hostname:            "localhost",
		Interval:            100 * time.Millisecond,
		Key:                 "",
		MetricMaxLength:     4096,
		Percentiles:         []float64{.5, .75, .99},
		ReadBufferSizeBytes: 2097152,
		UDPAddr:             "localhost:8126",
		HTTPAddr:            "localhost:8127",
		ForwardAddr:         "http://localhost",
		NumWorkers:          96,
		NumReaders:          4,
		StatsAddr:           "localhost:8125",
		Tags:                []string{},
		SentryDSN:           "",
		FlushLimit:          1024,
	}
}

// assertMetrics checks that all expected metrics are present
// and have the correct value
func assertMetrics(t *testing.T, metrics DDMetricsRequest, expectedMetrics map[string]float64) {
	// it doesn't count as accidentally quadratic if it's intentional
	for metricName, expectedValue := range expectedMetrics {
		assertMetric(t, metrics, metricName, expectedValue)
	}
}

func assertMetric(t *testing.T, metrics DDMetricsRequest, metricName string, value float64) {
	defer func() {
		if r := recover(); r != nil {
			assert.FailNow(t, "error extracting metrics", r)
		}
	}()
	for _, metric := range metrics.Series {
		if metric.Name == metricName {
			assert.Equal(t, metric.Value[0][1], value)
			return
		}
	}
	assert.FailNow(t, "did not find expected metric", metricName)
}

// setupLocalServer creates a local server from the specified config
// and starts listening for requests. It returns the server for inspection.
func setupLocalServer(t *testing.T, config Config) Server {
	server, err := NewFromConfig(config)
	if err != nil {
		t.Fatal(err)
	}

	packetPool := &sync.Pool{
		New: func() interface{} {
			return make([]byte, config.MetricMaxLength)
		},
	}

	for i := 0; i < config.NumReaders; i++ {
		go func() {
			defer func() {
				server.ConsumePanic(recover())
			}()
			server.ReadSocket(packetPool, config.NumReaders != 1)
		}()
	}

	go func() {
		defer func() {
			t.Fatal(recover())
		}()
		ticker := time.NewTicker(config.Interval)
		for range ticker.C {
			server.Flush(config.Interval, config.FlushLimit)
		}
	}()

	go server.HTTPServe()
	return server
}

// DDMetricsRequest represents the body of the POST request
// for sending metrics data to Datadog
type DDMetricsRequest struct {
	Series []DDMetric
}

func TestLocalServer(t *testing.T) {
	var ddmetrics DDMetricsRequest
	globalServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		zr, err := zlib.NewReader(r.Body)
		if err != nil {
			t.Fatal(err)
		}
		err = json.NewDecoder(zr).Decode(&ddmetrics)
		if err != nil {
			t.Fatal(err)
		}
		expectedMetrics := map[string]float64{
			"a.b.c":              10,
			"a.b.c.max":          1,
			"a.b.c.min":          1,
			"a.b.c.count":        10,
			"a.b.c.50percentile": 1,
			"a.b.c.75percentile": 1,
			"a.b.c.99percentile": 1,
		}
		assert.Len(t, ddmetrics.Series, 7, "number of elements in the flushed series")
		assertMetric(t, ddmetrics, "a.b.c.50percentile", 1)
		assertMetrics(t, ddmetrics, expectedMetrics)
		w.WriteHeader(http.StatusAccepted)
	}))
	config := localConfig()
	config.APIHostname = globalServer.URL

	server := setupLocalServer(t, config)

	metrics := []UDPMetric{
		{
			MetricKey: MetricKey{
				Name: "a.b.c",
				Type: "counter",
			},
			Value:      1.0,
			Digest:     12345,
			SampleRate: 1.0,
		},
		{

			MetricKey: MetricKey{
				Name: "a.b.c",
				Type: "histogram",
			},
			Value:      1.0,
			Digest:     12345,
			SampleRate: 1.0,
			LocalOnly:  true,
		}}

	for _, metric := range metrics {
		server.Workers[0].ProcessMetric(&metric)
	}

	server.Flush(config.Interval, config.FlushLimit)
}

func TestSplitBytes(t *testing.T) {
	rand.Seed(time.Now().Unix())
	buf := make([]byte, 1000)

	for i := 0; i < 1000; i++ {
		// we construct a string of random length which is approximately 1/3rd A
		// and the other 2/3rds B
		buf = buf[:rand.Intn(cap(buf))]
		for i := range buf {
			if rand.Intn(3) == 0 {
				buf[i] = 'A'
			} else {
				buf[i] = 'B'
			}
		}
		checkBufferSplit(t, buf)
		buf = buf[:cap(buf)]
	}

	// also test pathological cases that the fuzz is unlikely to find
	checkBufferSplit(t, nil)
	checkBufferSplit(t, []byte{})
}

func checkBufferSplit(t *testing.T, buf []byte) {
	var testSplit [][]byte
	sb := NewSplitBytes(buf, 'A')
	for sb.Next() {
		testSplit = append(testSplit, sb.Chunk())
	}

	// now compare our split to the "real" implementation of split
	assert.EqualValues(t, bytes.Split(buf, []byte{'A'}), testSplit, "should have split %s correctly", buf)
}
<commit_msg>Expand tests to include larger histogram<commit_after>package veneur

import (
	"bytes"
	"compress/zlib"
	"encoding/json"
	"math/rand"
	"net/http"
	"net/http/httptest"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

// set up a boilerplate local config for later use
func localConfig() Config {
	return Config{
		APIHostname:         "http://localhost",
		Debug:               false,
		Hostname:            "localhost",
		Interval:            100 * time.Millisecond,
		Key:                 "",
		MetricMaxLength:     4096,
		Percentiles:         []float64{.5, .75, .99},
		ReadBufferSizeBytes: 2097152,
		UDPAddr:             "localhost:8126",
		HTTPAddr:            "localhost:8127",
		ForwardAddr:         "http://localhost",
		NumWorkers:          96,
		NumReaders:          4,
		StatsAddr:           "localhost:8125",
		Tags:                []string{},
		SentryDSN:           "",
		FlushLimit:          1024,
	}
}

// assertMetrics checks that all expected metrics are present
// and have the correct value
func assertMetrics(t *testing.T, metrics DDMetricsRequest, expectedMetrics map[string]float64) {
	// it doesn't count as accidentally quadratic if it's intentional
	for metricName, expectedValue := range expectedMetrics {
		assertMetric(t, metrics, metricName, expectedValue)
	}
}

func assertMetric(t *testing.T, metrics DDMetricsRequest, metricName string, value float64) {
	defer func() {
		if r := recover(); r != nil {
			assert.Fail(t, "error extracting metrics", r)
		}
	}()
	for _, metric := range metrics.Series {
		if metric.Name == metricName {
			assert.Equal(t, int(value+.5), int(metric.Value[0][1]+.5), metricName)
			return
		}
	}
	assert.Fail(t, "did not find expected metric", metricName)
}

// setupLocalServer creates a local server from the specified config
// and starts listening for requests. It returns the server for inspection.
func setupLocalServer(t *testing.T, config Config) Server {
	server, err := NewFromConfig(config)
	if err != nil {
		t.Fatal(err)
	}

	packetPool := &sync.Pool{
		New: func() interface{} {
			return make([]byte, config.MetricMaxLength)
		},
	}

	for i := 0; i < config.NumReaders; i++ {
		go func() {
			defer func() {
				server.ConsumePanic(recover())
			}()
			server.ReadSocket(packetPool, config.NumReaders != 1)
		}()
	}

	go func() {
		defer func() {
			t.Fatal(recover())
		}()
		ticker := time.NewTicker(config.Interval)
		for range ticker.C {
			server.Flush(config.Interval, config.FlushLimit)
		}
	}()

	go server.HTTPServe()
	return server
}

// DDMetricsRequest represents the body of the POST request
// for sending metrics data to Datadog
type DDMetricsRequest struct {
	Series []DDMetric
}

func TestLocalServer(t *testing.T) {
	var ddmetrics DDMetricsRequest
	globalServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		zr, err := zlib.NewReader(r.Body)
		if err != nil {
			t.Fatal(err)
		}
		err = json.NewDecoder(zr).Decode(&ddmetrics)
		if err != nil {
			t.Fatal(err)
		}
		expectedMetrics := map[string]float64{
			"a.b.c.max": 100,
			"a.b.c.min": 1,

			// There are five items, but we multiply by 10 when sampling
			"a.b.c.count": 50,

			// tdigest approximation causes this to be off by 1
			"a.b.c.50percentile": 6,
			"a.b.c.75percentile": 42,
			"a.b.c.99percentile": 98,
		}
		assert.Equal(t, 6, len(ddmetrics.Series), "incorrect number of elements in the flushed series")
		assertMetrics(t, ddmetrics, expectedMetrics)
		w.WriteHeader(http.StatusAccepted)
	}))
	config := localConfig()
	config.APIHostname = globalServer.URL

	server := setupLocalServer(t, config)

	metrics := []UDPMetric{
		{
			MetricKey: MetricKey{
				Name: "a.b.c",
				Type: "histogram",
			},
			Value:      1.0,
			Digest:     12345,
			SampleRate: 1.0,
			LocalOnly:  true,
		},
		{

			MetricKey: MetricKey{
				Name: "a.b.c",
				Type: "histogram",
			},
			Value:      2.0,
			Digest:     12345,
			SampleRate: 1.0,
			LocalOnly:  true,
		},
		{

			MetricKey: MetricKey{
				Name: "a.b.c",
				Type: "histogram",
			},
			Value:      7.0,
			Digest:     12345,
			SampleRate: 1.0,
			LocalOnly:  true,
		},
		{

			MetricKey: MetricKey{
				Name: "a.b.c",
				Type: "histogram",
			},
			Value:      8.0,
			Digest:     12345,
			SampleRate: 1.0,
			LocalOnly:  true,
		},
		{

			MetricKey: MetricKey{
				Name: "a.b.c",
				Type: "histogram",
			},
			Value:      100.0,
			Digest:     12345,
			SampleRate: 1.0,
			LocalOnly:  true,
		},
	}

	for _, metric := range metrics {
		server.Workers[0].ProcessMetric(&metric)
	}

	server.Flush(config.Interval, config.FlushLimit)
}

func TestSplitBytes(t *testing.T) {
	rand.Seed(time.Now().Unix())
	buf := make([]byte, 1000)

	for i := 0; i < 1000; i++ {
		// we construct a string of random length which is approximately 1/3rd A
		// and the other 2/3rds B
		buf = buf[:rand.Intn(cap(buf))]
		for i := range buf {
			if rand.Intn(3) == 0 {
				buf[i] = 'A'
			} else {
				buf[i] = 'B'
			}
		}
		checkBufferSplit(t, buf)
		buf = buf[:cap(buf)]
	}

	// also test pathological cases that the fuzz is unlikely to find
	checkBufferSplit(t, nil)
	checkBufferSplit(t, []byte{})
}

func checkBufferSplit(t *testing.T, buf []byte) {
	var testSplit [][]byte
	sb := NewSplitBytes(buf, 'A')
	for sb.Next() {
		testSplit = append(testSplit, sb.Chunk())
	}

	// now compare our split to the "real" implementation of split
	assert.EqualValues(t, bytes.Split(buf, []byte{'A'}), testSplit, "should have split %s correctly", buf)
}
<|endoftext|>


<commit_before>package connector

import (
	"context"
	"fmt"
	"net"
	"sort"
	"strings"
	"time"

	"github.com/datawire/ambassador/pkg/kates"
	"github.com/datawire/dlib/dgroup"
	"github.com/datawire/dlib/dlog"
	"github.com/datawire/dlib/dutil"
	"github.com/datawire/telepresence2/rpc/v2/daemon"
)

// k8sWatcher watcher of pods and services.
type k8sWatcher struct {
	Pods      []*kates.Pod
	Services  []*kates.Service
	Endpoints []*kates.Endpoints
	cancel    context.CancelFunc
	namespace string
}

// startWatchers initializes a set of Kubernetes watchers that provide information from the cluster
// which is then used for controlling the outbound connectivity of the cluster.
//
// Initially, a watcher that watches the namespaces is created. Once it produces its first
// snapshot, some filtering is done on the list in that snapshot and then one watcher for each
// resulting namespace is created that watches pods and services. When their snapshot arrives,
// these pods and services are used when creating IP-tables that are sent to the NAT-logic in the
// daemon.
//
// The filtered list of namespaces is also used for creating a DNS search path which is propagated to
// the DNS-resolver in the daemon.
//
// When an update arrives in the namespace watcher, it will refresh the DNS-search path and the current
// set of watchers so that new watchers are added for added namespaces and watchers for namespaces that
// have been will be cancelled.
//
// If a pods and services watcher receives an update, it will send an updated IP-table to the daemon.
func (kc *k8sCluster) startWatchers(c context.Context, accWait chan struct{}) (err error) {
	defer func() {
		if r := dutil.PanicToError(recover()); r != nil {
			err = r
		}
	}()

	acc := kc.client.Watch(c,
		kates.Query{
			Name: "Namespaces",
			Kind: "namespace",
		})

	g := dgroup.ParentGroup(c)
	g.Go("watch-k8s-namespaces", func(c context.Context) error {
		for {
			select {
			case <-c.Done():
				return nil
			case <-acc.Changed():
				if kc.onNamespacesChange(c, acc, accWait) {
					accWait = nil // accWait is one-shot
				}
			}
		}
	})

	g.Go("watch-k8s", func(c context.Context) error {
		for {
			select {
			case <-c.Done():
				return nil
			case <-kc.watcherChanged:
				kc.updateDaemonTable(c)
			}
		}
	})
	return nil
}

func (kc *k8sCluster) onNamespacesChange(c context.Context, acc *kates.Accumulator, accWait chan<- struct{}) bool {
	changed := func() bool {
		kc.accLock.Lock()
		defer kc.accLock.Unlock()
		return acc.Update(kc)
	}()
	if changed {
		changed = kc.refreshNamespaces(c, accWait)
	}
	return changed
}

func (kc *k8sCluster) setMappedNamespaces(c context.Context, namespaces []string) {
	sort.Strings(namespaces)
	kc.accLock.Lock()
	kc.mappedNamespaces = namespaces
	kc.accLock.Unlock()
	kc.refreshNamespaces(c, nil)
}

func (kc *k8sCluster) refreshNamespaces(c context.Context, accWait chan<- struct{}) bool {
	kc.accLock.Lock()
	namespaces := make([]string, 0, len(kc.Namespaces))
	for _, ns := range kc.Namespaces {
		if kc.shouldBeWatched(ns.Name) {
			namespaces = append(namespaces, ns.Name)
		}
	}
	sort.Strings(namespaces)

	nsChange := len(namespaces) != len(kc.lastNamespaces)
	if !nsChange {
		for i, ns := range namespaces {
			if ns != kc.lastNamespaces[i] {
				nsChange = true
				break
			}
		}
	}
	if nsChange {
		kc.lastNamespaces = namespaces
	}
	kc.accLock.Unlock()

	if nsChange {
		kc.refreshWatchers(c, namespaces, accWait)
		kc.updateDaemonNamespaces(c)
	}
	return nsChange
}

func (kc *k8sCluster) shouldBeWatched(namespace string) bool {
	if len(kc.mappedNamespaces) == 0 {
		return !(namespace == managerNamespace || strings.HasPrefix(namespace, "kube-"))
	}
	for _, n := range kc.mappedNamespaces {
		if n == namespace {
			return true
		}
	}
	return false
}

// refreshWatchersLocked ensures that the current set of namespaces are watched.
//
// The accWait channel should only be passed once to this function and will be closed once all watchers have
// received their initial snapshot.
func (kc *k8sCluster) refreshWatchers(c context.Context, namespaces []string, accWait chan<- struct{}) {
	var onChange func(int)
	if accWait == nil {
		onChange = func(_ int) {}
	} else {
		// close accWait when all watchers have received an update
		initialReceived := make([]bool, len(namespaces))
		onChange = func(index int) {
			kc.accLock.Lock()
			defer kc.accLock.Unlock()
			if accWait == nil || initialReceived[index] {
				return
			}
			initialReceived[index] = true
			for _, r := range initialReceived {
				if !r {
					return
				}
			}
			al := accWait
			accWait = nil
			close(al)
		}
	}

	kc.accLock.Lock()
	if kc.watchers == nil {
		kc.watchers = make(map[string]*k8sWatcher, len(namespaces))
	}

	keep := make(map[string]struct{})
	for i, namespace := range namespaces {
		keep[namespace] = struct{}{}
		if _, ok := kc.watchers[namespace]; ok {
			continue
		}
		watcher := &k8sWatcher{namespace: namespace}
		var wc context.Context
		wc, watcher.cancel = context.WithCancel(c)
		kc.watchers[namespace] = watcher

		go kc.watchPodsAndServices(wc, watcher, i, onChange)
	}
	kc.accLock.Unlock()

	// Cancel watchers of undesired namespaces
	for namespace, watcher := range kc.watchers {
		if _, ok := keep[namespace]; !ok {
			watcher.cancel()
		}
	}
}

func (kc *k8sCluster) watchPodsAndServices(c context.Context, watcher *k8sWatcher, i int, onChange func(int)) {
	nAcc := kc.client.Watch(c,
		kates.Query{
			Name:      "Services",
			Namespace: watcher.namespace,
			Kind:      "service",
		},
		kates.Query{
			Name:      "Endpoints",
			Namespace: watcher.namespace,
			Kind:      "endpoints",
		},
		kates.Query{
			Name:      "Pods",
			Namespace: watcher.namespace,
			Kind:      "pod",
		})

	dlog.Infof(c, "Watching namespace %q", watcher.namespace)

	for {
		select {
		case <-c.Done():
			dlog.Infof(c, "Watch of namespace %q cancelled", watcher.namespace)
			return
		case x := <-nAcc.Changed():
			if func() bool {
				kc.accLock.Lock()
				defer kc.accLock.Unlock()
				return nAcc.Update(watcher)
			}() {
				kc.watcherChanged <- x
			}
			onChange(i)
		}
	}
}

func (kc *k8sCluster) setInterceptedNamespaces(c context.Context, interceptedNamespaces map[string]struct{}) {
	kc.accLock.Lock()
	kc.interceptedNamespaces = interceptedNamespaces
	kc.accLock.Unlock()
	kc.updateDaemonNamespaces(c)
}

const clusterServerSuffix = ".svc.cluster.local"

// updateDaemonNamespacesLocked will create a new DNS search path from the given namespaces and
// send it to the DNS-resolver in the daemon.
func (kc *k8sCluster) updateDaemonNamespaces(c context.Context) {
	if kc.daemon == nil {
		// NOTE! Some tests dont't set the daemon
		return
	}

	kc.accLock.Lock()
	namespaces := make([]string, 0, len(kc.interceptedNamespaces)+len(kc.localIntercepts))
	for ns := range kc.interceptedNamespaces {
		namespaces = append(namespaces, ns)
	}
	for ns := range kc.localInterceptedNamespaces {
		if _, found := kc.interceptedNamespaces[ns]; !found {
			namespaces = append(namespaces, ns)
		}
	}

	// Pass current mapped namespaces as plain names (no ending dot). The DNS-resolver will
	// create special mapping for those, allowing names like myservice.mynamespace to be resolved
	paths := make([]string, len(kc.lastNamespaces), len(kc.lastNamespaces)+len(namespaces))
	copy(paths, kc.lastNamespaces)

	// Avoid being locked for the remainder of this function.
	kc.accLock.Unlock()

	// Provide direct access to intercepted namespaces
	sort.Strings(namespaces)
	for _, ns := range namespaces {
		paths = append(paths, ns+clusterServerSuffix+".")
	}
	dlog.Debugf(c, "posting search paths to %s", strings.Join(paths, " "))
	if _, err := kc.daemon.SetDnsSearchPath(c, &daemon.Paths{Paths: paths}); err != nil {
		dlog.Errorf(c, "error posting search paths to %s: %v", strings.Join(paths, " "), err)
	}
}

const coalesceDelay = time.Second

// updateDaemonTable will create IP-tables based on the current snapshots of services and pods
// that the watchers have produced.
func (kc *k8sCluster) updateDaemonTable(c context.Context) {
	if kc.daemon == nil {
		// NOTE! Some tests don't set the daemon
		return
	}

	doUpdate := func() {
		kc.accLock.Lock()
		table := &daemon.Table{Name: "kubernetes"}
		for _, watcher := range kc.watchers {
			for _, svc := range watcher.Services {
				updateTableFromService(c, svc, watcher.Endpoints, table)
			}
			for _, pod := range watcher.Pods {
				updateTableFromPod(pod, table)
			}
		}
		kc.accLock.Unlock()

		// Send updated table to daemon
		dlog.Debugf(c, "posting update to IP-table %s with %d routes", table.Name, len(table.Routes))
		if _, err := kc.daemon.Update(c, table); err != nil {
			dlog.Errorf(c, "error posting update to %s: %v", table.Name, err)
		}
	}

	// No update is sent to the daemon until it has been quiet with updates from the watchers for
	// at least one second.
	kc.accLock.Lock()
	if kc.svcPodTimer == nil {
		kc.svcPodTimer = time.AfterFunc(coalesceDelay, doUpdate)
	} else {
		kc.svcPodTimer.Reset(coalesceDelay)
	}
	kc.accLock.Unlock()
}

func updateTableFromService(c context.Context, svc *kates.Service, endpoints []*kates.Endpoints, table *daemon.Table) {
	spec := &svc.Spec
	clusterIP := spec.ClusterIP
	var ips []string
	var err error
	// Handle headless services separately since they don't have cluster IPs
	if clusterIP == "" || clusterIP == "None" {
		ips, err = getIPsFromHeadlessService(svc, spec.ExternalName, endpoints)
		if err != nil {
			dlog.Errorf(c, "Error finding IPs for service %s: %s", svc.Name, err)
		}
		if len(ips) == 0 {
			dlog.Errorf(c, "Found no IPs for service %s", svc.Name)
		}
	} else {
		ips = append(ips, clusterIP)
	}
	qName := svc.Name + "." + svc.Namespace + clusterServerSuffix

	ports := ""
	for _, port := range spec.Ports {
		if ports == "" {
			ports = fmt.Sprintf("%d", port.Port)
		} else {
			ports = fmt.Sprintf("%s,%d", ports, port.Port)
		}

		// Kubernetes creates records for all named ports, of the form
		// _my-port-name._my-port-protocol.my-svc.my-namespace.svc.cluster-domain.example
		// https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#srv-records
		if port.Name != "" {
			proto := strings.ToLower(string(port.Protocol))
			table.Routes = append(table.Routes, &daemon.Route{
				Name:  fmt.Sprintf("_%v._%v.%v", port.Name, proto, qName),
				Ips:   ips,
				Port:  ports,
				Proto: proto,
			})
		}
	}

	table.Routes = append(table.Routes, &daemon.Route{
		Name:  qName,
		Ips:   ips,
		Port:  ports,
		Proto: "tcp",
	})
}

// Headless Services don't have a cluster IP so we need to get their IP(s) separately.
// Docs on headless services can be found here
// https://kubernetes.io/docs/concepts/services-networking/service/#headless-services
func getIPsFromHeadlessService(svc *kates.Service, externalName string, endpoints []*kates.Endpoints) ([]string, error) {
	serviceName := svc.Name
	var ips []string
	// ExternalName services don't have endpoints so we handle them separately.
	if externalName != "" {
		domainIPs, err := net.LookupIP(externalName)
		if err != nil {
			return nil, err
		}
		if len(domainIPs) == 0 {
			return nil, nil
		}
		for _, ip := range domainIPs {
			ips = append(ips, ip.String())
		}
	} else {
		// see if there's an associated endpoint with the headless service
		for i := range endpoints {
			if endpoints[i].ObjectMeta.Name == serviceName {
				matchingEndpoint := endpoints[i]
				if len(matchingEndpoint.Subsets) > 0 {
					endpointSubset := matchingEndpoint.Subsets[0]

					// Endpoints have two sets of addresses: Addresses and NotReadyAddresses
					// We only want to forward traffic to pods that are ready one so we only
					// loop through the ready addresses
					for _, address := range endpointSubset.Addresses {
						ips = append(ips, address.IP)
					}
					if len(ips) == 0 {
						return nil, nil
					}
				} else {
					return nil, nil
				}
				break
			}
		}
	}
	return ips, nil
}

func updateTableFromPod(pod *kates.Pod, table *daemon.Table) {
	qname := ""

	hostname := pod.Spec.Hostname
	if hostname != "" {
		qname += hostname
	}

	subdomain := pod.Spec.Subdomain
	if subdomain != "" {
		qname += "." + subdomain
	}

	if qname == "" {
		// Note: this is a departure from kubernetes, kubernetes will
		// simply not publish a dns name in this case.
		qname = pod.Name + "." + pod.Namespace + ".pod.cluster.local"
	} else {
		qname += clusterServerSuffix
	}

	ips := []string{pod.Status.PodIP}
	if len(ips) > 0 {
		table.Routes = append(table.Routes, &daemon.Route{
			Name:  qname,
			Ips:   ips,
			Proto: "tcp",
		})
	}
}
<commit_msg>Don't create routes unless at least one IP is found.<commit_after>package connector

import (
	"context"
	"fmt"
	"net"
	"sort"
	"strings"
	"time"

	"github.com/datawire/ambassador/pkg/kates"
	"github.com/datawire/dlib/dgroup"
	"github.com/datawire/dlib/dlog"
	"github.com/datawire/dlib/dutil"
	"github.com/datawire/telepresence2/rpc/v2/daemon"
)

// k8sWatcher watcher of pods and services.
type k8sWatcher struct {
	Pods      []*kates.Pod
	Services  []*kates.Service
	Endpoints []*kates.Endpoints
	cancel    context.CancelFunc
	namespace string
}

// startWatchers initializes a set of Kubernetes watchers that provide information from the cluster
// which is then used for controlling the outbound connectivity of the cluster.
//
// Initially, a watcher that watches the namespaces is created. Once it produces its first
// snapshot, some filtering is done on the list in that snapshot and then one watcher for each
// resulting namespace is created that watches pods and services. When their snapshot arrives,
// these pods and services are used when creating IP-tables that are sent to the NAT-logic in the
// daemon.
//
// The filtered list of namespaces is also used for creating a DNS search path which is propagated to
// the DNS-resolver in the daemon.
//
// When an update arrives in the namespace watcher, it will refresh the DNS-search path and the current
// set of watchers so that new watchers are added for added namespaces and watchers for namespaces that
// have been will be cancelled.
//
// If a pods and services watcher receives an update, it will send an updated IP-table to the daemon.
func (kc *k8sCluster) startWatchers(c context.Context, accWait chan struct{}) (err error) {
	defer func() {
		if r := dutil.PanicToError(recover()); r != nil {
			err = r
		}
	}()

	acc := kc.client.Watch(c,
		kates.Query{
			Name: "Namespaces",
			Kind: "namespace",
		})

	g := dgroup.ParentGroup(c)
	g.Go("watch-k8s-namespaces", func(c context.Context) error {
		for {
			select {
			case <-c.Done():
				return nil
			case <-acc.Changed():
				if kc.onNamespacesChange(c, acc, accWait) {
					accWait = nil // accWait is one-shot
				}
			}
		}
	})

	g.Go("watch-k8s", func(c context.Context) error {
		for {
			select {
			case <-c.Done():
				return nil
			case <-kc.watcherChanged:
				kc.updateDaemonTable(c)
			}
		}
	})
	return nil
}

func (kc *k8sCluster) onNamespacesChange(c context.Context, acc *kates.Accumulator, accWait chan<- struct{}) bool {
	changed := func() bool {
		kc.accLock.Lock()
		defer kc.accLock.Unlock()
		return acc.Update(kc)
	}()
	if changed {
		changed = kc.refreshNamespaces(c, accWait)
	}
	return changed
}

func (kc *k8sCluster) setMappedNamespaces(c context.Context, namespaces []string) {
	sort.Strings(namespaces)
	kc.accLock.Lock()
	kc.mappedNamespaces = namespaces
	kc.accLock.Unlock()
	kc.refreshNamespaces(c, nil)
}

func (kc *k8sCluster) refreshNamespaces(c context.Context, accWait chan<- struct{}) bool {
	kc.accLock.Lock()
	namespaces := make([]string, 0, len(kc.Namespaces))
	for _, ns := range kc.Namespaces {
		if kc.shouldBeWatched(ns.Name) {
			namespaces = append(namespaces, ns.Name)
		}
	}
	sort.Strings(namespaces)

	nsChange := len(namespaces) != len(kc.lastNamespaces)
	if !nsChange {
		for i, ns := range namespaces {
			if ns != kc.lastNamespaces[i] {
				nsChange = true
				break
			}
		}
	}
	if nsChange {
		kc.lastNamespaces = namespaces
	}
	kc.accLock.Unlock()

	if nsChange {
		kc.refreshWatchers(c, namespaces, accWait)
		kc.updateDaemonNamespaces(c)
	}
	return nsChange
}

func (kc *k8sCluster) shouldBeWatched(namespace string) bool {
	if len(kc.mappedNamespaces) == 0 {
		return !(namespace == managerNamespace || strings.HasPrefix(namespace, "kube-"))
	}
	for _, n := range kc.mappedNamespaces {
		if n == namespace {
			return true
		}
	}
	return false
}

// refreshWatchersLocked ensures that the current set of namespaces are watched.
//
// The accWait channel should only be passed once to this function and will be closed once all watchers have
// received their initial snapshot.
func (kc *k8sCluster) refreshWatchers(c context.Context, namespaces []string, accWait chan<- struct{}) {
	var onChange func(int)
	if accWait == nil {
		onChange = func(_ int) {}
	} else {
		// close accWait when all watchers have received an update
		initialReceived := make([]bool, len(namespaces))
		onChange = func(index int) {
			kc.accLock.Lock()
			defer kc.accLock.Unlock()
			if accWait == nil || initialReceived[index] {
				return
			}
			initialReceived[index] = true
			for _, r := range initialReceived {
				if !r {
					return
				}
			}
			al := accWait
			accWait = nil
			close(al)
		}
	}

	kc.accLock.Lock()
	if kc.watchers == nil {
		kc.watchers = make(map[string]*k8sWatcher, len(namespaces))
	}

	keep := make(map[string]struct{})
	for i, namespace := range namespaces {
		keep[namespace] = struct{}{}
		if _, ok := kc.watchers[namespace]; ok {
			continue
		}
		watcher := &k8sWatcher{namespace: namespace}
		var wc context.Context
		wc, watcher.cancel = context.WithCancel(c)
		kc.watchers[namespace] = watcher

		go kc.watchPodsAndServices(wc, watcher, i, onChange)
	}
	kc.accLock.Unlock()

	// Cancel watchers of undesired namespaces
	for namespace, watcher := range kc.watchers {
		if _, ok := keep[namespace]; !ok {
			watcher.cancel()
		}
	}
}

func (kc *k8sCluster) watchPodsAndServices(c context.Context, watcher *k8sWatcher, i int, onChange func(int)) {
	nAcc := kc.client.Watch(c,
		kates.Query{
			Name:      "Services",
			Namespace: watcher.namespace,
			Kind:      "service",
		},
		kates.Query{
			Name:      "Endpoints",
			Namespace: watcher.namespace,
			Kind:      "endpoints",
		},
		kates.Query{
			Name:      "Pods",
			Namespace: watcher.namespace,
			Kind:      "pod",
		})

	dlog.Infof(c, "Watching namespace %q", watcher.namespace)

	for {
		select {
		case <-c.Done():
			dlog.Infof(c, "Watch of namespace %q cancelled", watcher.namespace)
			return
		case x := <-nAcc.Changed():
			if func() bool {
				kc.accLock.Lock()
				defer kc.accLock.Unlock()
				return nAcc.Update(watcher)
			}() {
				kc.watcherChanged <- x
			}
			onChange(i)
		}
	}
}

func (kc *k8sCluster) setInterceptedNamespaces(c context.Context, interceptedNamespaces map[string]struct{}) {
	kc.accLock.Lock()
	kc.interceptedNamespaces = interceptedNamespaces
	kc.accLock.Unlock()
	kc.updateDaemonNamespaces(c)
}

const clusterServerSuffix = ".svc.cluster.local"

// updateDaemonNamespacesLocked will create a new DNS search path from the given namespaces and
// send it to the DNS-resolver in the daemon.
func (kc *k8sCluster) updateDaemonNamespaces(c context.Context) {
	if kc.daemon == nil {
		// NOTE! Some tests dont't set the daemon
		return
	}

	kc.accLock.Lock()
	namespaces := make([]string, 0, len(kc.interceptedNamespaces)+len(kc.localIntercepts))
	for ns := range kc.interceptedNamespaces {
		namespaces = append(namespaces, ns)
	}
	for ns := range kc.localInterceptedNamespaces {
		if _, found := kc.interceptedNamespaces[ns]; !found {
			namespaces = append(namespaces, ns)
		}
	}

	// Pass current mapped namespaces as plain names (no ending dot). The DNS-resolver will
	// create special mapping for those, allowing names like myservice.mynamespace to be resolved
	paths := make([]string, len(kc.lastNamespaces), len(kc.lastNamespaces)+len(namespaces))
	copy(paths, kc.lastNamespaces)

	// Avoid being locked for the remainder of this function.
	kc.accLock.Unlock()

	// Provide direct access to intercepted namespaces
	sort.Strings(namespaces)
	for _, ns := range namespaces {
		paths = append(paths, ns+clusterServerSuffix+".")
	}
	dlog.Debugf(c, "posting search paths to %s", strings.Join(paths, " "))
	if _, err := kc.daemon.SetDnsSearchPath(c, &daemon.Paths{Paths: paths}); err != nil {
		dlog.Errorf(c, "error posting search paths to %s: %v", strings.Join(paths, " "), err)
	}
}

const coalesceDelay = time.Second

// updateDaemonTable will create IP-tables based on the current snapshots of services and pods
// that the watchers have produced.
func (kc *k8sCluster) updateDaemonTable(c context.Context) {
	if kc.daemon == nil {
		// NOTE! Some tests don't set the daemon
		return
	}

	doUpdate := func() {
		kc.accLock.Lock()
		table := &daemon.Table{Name: "kubernetes"}
		for _, watcher := range kc.watchers {
			for _, svc := range watcher.Services {
				updateTableFromService(c, svc, watcher.Endpoints, table)
			}
			for _, pod := range watcher.Pods {
				updateTableFromPod(pod, table)
			}
		}
		kc.accLock.Unlock()

		// Send updated table to daemon
		dlog.Debugf(c, "posting update to IP-table %s with %d routes", table.Name, len(table.Routes))
		if _, err := kc.daemon.Update(c, table); err != nil {
			dlog.Errorf(c, "error posting update to %s: %v", table.Name, err)
		}
	}

	// No update is sent to the daemon until it has been quiet with updates from the watchers for
	// at least one second.
	kc.accLock.Lock()
	if kc.svcPodTimer == nil {
		kc.svcPodTimer = time.AfterFunc(coalesceDelay, doUpdate)
	} else {
		kc.svcPodTimer.Reset(coalesceDelay)
	}
	kc.accLock.Unlock()
}

func updateTableFromService(c context.Context, svc *kates.Service, endpoints []*kates.Endpoints, table *daemon.Table) {
	spec := &svc.Spec
	clusterIP := spec.ClusterIP
	var ips []string
	var err error
	// Handle headless services separately since they don't have cluster IPs
	if clusterIP == "" || clusterIP == "None" {
		ips, err = getIPsFromHeadlessService(svc, spec.ExternalName, endpoints)
		if err != nil {
			dlog.Errorf(c, "Error finding IPs for service %s: %s", svc.Name, err)
			return
		}
		if len(ips) == 0 {
			if svc.Name != "traffic-manager" {
				dlog.Errorf(c, "Found no IPs for service %s", svc.Name)
			}
			return
		}
	} else {
		ips = append(ips, clusterIP)
	}
	qName := svc.Name + "." + svc.Namespace + clusterServerSuffix

	ports := ""
	for _, port := range spec.Ports {
		if ports == "" {
			ports = fmt.Sprintf("%d", port.Port)
		} else {
			ports = fmt.Sprintf("%s,%d", ports, port.Port)
		}

		// Kubernetes creates records for all named ports, of the form
		// _my-port-name._my-port-protocol.my-svc.my-namespace.svc.cluster-domain.example
		// https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#srv-records
		if port.Name != "" {
			proto := strings.ToLower(string(port.Protocol))
			table.Routes = append(table.Routes, &daemon.Route{
				Name:  fmt.Sprintf("_%v._%v.%v", port.Name, proto, qName),
				Ips:   ips,
				Port:  ports,
				Proto: proto,
			})
		}
	}

	table.Routes = append(table.Routes, &daemon.Route{
		Name:  qName,
		Ips:   ips,
		Port:  ports,
		Proto: "tcp",
	})
}

// Headless Services don't have a cluster IP so we need to get their IP(s) separately.
// Docs on headless services can be found here
// https://kubernetes.io/docs/concepts/services-networking/service/#headless-services
func getIPsFromHeadlessService(svc *kates.Service, externalName string, endpoints []*kates.Endpoints) ([]string, error) {
	serviceName := svc.Name
	var ips []string
	// ExternalName services don't have endpoints so we handle them separately.
	if externalName != "" {
		domainIPs, err := net.LookupIP(externalName)
		if err != nil {
			return nil, err
		}
		if len(domainIPs) == 0 {
			return nil, nil
		}
		for _, ip := range domainIPs {
			ips = append(ips, ip.String())
		}
	} else {
		// see if there's an associated endpoint with the headless service
		for i := range endpoints {
			if endpoints[i].ObjectMeta.Name == serviceName {
				matchingEndpoint := endpoints[i]
				if len(matchingEndpoint.Subsets) > 0 {
					endpointSubset := matchingEndpoint.Subsets[0]

					// Endpoints have two sets of addresses: Addresses and NotReadyAddresses
					// We only want to forward traffic to pods that are ready one so we only
					// loop through the ready addresses
					for _, address := range endpointSubset.Addresses {
						ips = append(ips, address.IP)
					}
					if len(ips) == 0 {
						return nil, nil
					}
				} else {
					return nil, nil
				}
				break
			}
		}
	}
	return ips, nil
}

func updateTableFromPod(pod *kates.Pod, table *daemon.Table) {
	qname := ""

	hostname := pod.Spec.Hostname
	if hostname != "" {
		qname += hostname
	}

	subdomain := pod.Spec.Subdomain
	if subdomain != "" {
		qname += "." + subdomain
	}

	if qname == "" {
		// Note: this is a departure from kubernetes, kubernetes will
		// simply not publish a dns name in this case.
		qname = pod.Name + "." + pod.Namespace + ".pod.cluster.local"
	} else {
		qname += clusterServerSuffix
	}

	ips := []string{pod.Status.PodIP}
	if len(ips) > 0 {
		table.Routes = append(table.Routes, &daemon.Route{
			Name:  qname,
			Ips:   ips,
			Proto: "tcp",
		})
	}
}
<|endoftext|>


<commit_before>package add

import (
	"compress/gzip"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"runtime"
	"time"

	"github.com/docker/distribution"
	"github.com/docker/distribution/manifest/schema2"
	digest "github.com/opencontainers/go-digest"

	"github.com/openshift/origin/pkg/image/apis/image/docker10"
	"github.com/openshift/origin/pkg/image/dockerlayer"
)

// get base manifest
// check that I can access base layers
// find the input file (assume I can stream)
// start a streaming upload of the layer to the remote registry, while calculating digests
// get back the final digest
// build the new image manifest and config.json
// upload config.json
// upload the rest of the layers
// tag the image

const (
	// dockerV2Schema2LayerMediaType is the MIME type used for schema 2 layers.
	dockerV2Schema2LayerMediaType = "application/vnd.docker.image.rootfs.diff.tar.gzip"
	// dockerV2Schema2ConfigMediaType is the MIME type used for schema 2 config blobs.
	dockerV2Schema2ConfigMediaType = "application/vnd.docker.container.image.v1+json"
)

// DigestCopy reads all of src into dst, where src is a gzipped stream. It will return the
// sha256 sum of the underlying content (the layerDigest) and the sha256 sum of the
// tar archive (the blobDigest) or an error. If the gzip layer has a modification time
// it will be returned.
// TODO: use configurable digests
func DigestCopy(dst io.ReaderFrom, src io.Reader) (layerDigest, blobDigest digest.Digest, modTime *time.Time, size int64, err error) {
	algo := digest.Canonical
	// calculate the blob digest as the sha256 sum of the uploaded contents
	blobhash := algo.Hash()
	// calculate the diffID as the sha256 sum of the layer contents
	pr, pw := io.Pipe()
	layerhash := algo.Hash()
	ch := make(chan error)
	go func() {
		defer close(ch)
		gr, err := gzip.NewReader(pr)
		if err != nil {
			ch <- fmt.Errorf("unable to create gzip reader layer upload: %v", err)
			return
		}
		if !gr.Header.ModTime.IsZero() {
			modTime = &gr.Header.ModTime
		}
		_, err = io.Copy(layerhash, gr)
		ch <- err
	}()

	n, err := dst.ReadFrom(io.TeeReader(src, io.MultiWriter(blobhash, pw)))
	if err != nil {
		return "", "", nil, 0, fmt.Errorf("unable to upload new layer (%d): %v", n, err)
	}
	if err := pw.Close(); err != nil {
		return "", "", nil, 0, fmt.Errorf("unable to complete writing diffID: %v", err)
	}
	if err := <-ch; err != nil {
		return "", "", nil, 0, fmt.Errorf("unable to calculate layer diffID: %v", err)
	}

	layerDigest = digest.NewDigestFromBytes(algo, layerhash.Sum(make([]byte, 0, layerhash.Size())))
	blobDigest = digest.NewDigestFromBytes(algo, blobhash.Sum(make([]byte, 0, blobhash.Size())))
	return layerDigest, blobDigest, modTime, n, nil
}

func NewEmptyConfig() *docker10.DockerImageConfig {
	config := &docker10.DockerImageConfig{
		DockerVersion: "",
		// Created must be non-zero
		Created:      (time.Time{}).Add(1 * time.Second),
		OS:           runtime.GOOS,
		Architecture: runtime.GOARCH,
	}
	return config
}

func AddScratchLayerToConfig(config *docker10.DockerImageConfig) distribution.Descriptor {
	layer := distribution.Descriptor{
		MediaType: dockerV2Schema2LayerMediaType,
		Digest:    digest.Digest(dockerlayer.GzippedEmptyLayerDigest),
		Size:      int64(len(dockerlayer.GzippedEmptyLayer)),
	}
	AddLayerToConfig(config, layer, dockerlayer.EmptyLayerDiffID)
	return layer
}

func AddLayerToConfig(config *docker10.DockerImageConfig, layer distribution.Descriptor, diffID string) {
	if config.RootFS == nil {
		config.RootFS = &docker10.DockerConfigRootFS{Type: "layers"}
	}
	config.RootFS.DiffIDs = append(config.RootFS.DiffIDs, diffID)
	config.Size += layer.Size
}

func UploadSchema2Config(ctx context.Context, blobs distribution.BlobService, config *docker10.DockerImageConfig, layers []distribution.Descriptor) (*schema2.DeserializedManifest, error) {
	// ensure the image size is correct before persisting
	config.Size = 0
	for _, layer := range layers {
		config.Size += layer.Size
	}
	configJSON, err := json.Marshal(config)
	if err != nil {
		return nil, err
	}
	return putSchema2ImageConfig(ctx, blobs, dockerV2Schema2ConfigMediaType, configJSON, layers)
}

// putSchema2ImageConfig uploads the provided configJSON to the blob store and returns the generated manifest
// for the requested image.
func putSchema2ImageConfig(ctx context.Context, blobs distribution.BlobService, mediaType string, configJSON []byte, layers []distribution.Descriptor) (*schema2.DeserializedManifest, error) {
	b := schema2.NewManifestBuilder(blobs, mediaType, configJSON)
	for _, layer := range layers {
		if err := b.AppendReference(layer); err != nil {
			return nil, err
		}
	}
	m, err := b.Build(ctx)
	if err != nil {
		return nil, err
	}
	manifest, ok := m.(*schema2.DeserializedManifest)
	if !ok {
		return nil, fmt.Errorf("unable to turn %T into a DeserializedManifest, unable to store image", m)
	}
	return manifest, nil
}
<commit_msg>When a docker layer has a gzip error, read after error<commit_after>package add

import (
	"compress/gzip"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"runtime"
	"time"

	"github.com/docker/distribution"
	"github.com/docker/distribution/manifest/schema2"
	digest "github.com/opencontainers/go-digest"

	"github.com/openshift/origin/pkg/image/apis/image/docker10"
	"github.com/openshift/origin/pkg/image/dockerlayer"
)

// get base manifest
// check that I can access base layers
// find the input file (assume I can stream)
// start a streaming upload of the layer to the remote registry, while calculating digests
// get back the final digest
// build the new image manifest and config.json
// upload config.json
// upload the rest of the layers
// tag the image

const (
	// dockerV2Schema2LayerMediaType is the MIME type used for schema 2 layers.
	dockerV2Schema2LayerMediaType = "application/vnd.docker.image.rootfs.diff.tar.gzip"
	// dockerV2Schema2ConfigMediaType is the MIME type used for schema 2 config blobs.
	dockerV2Schema2ConfigMediaType = "application/vnd.docker.container.image.v1+json"
)

// DigestCopy reads all of src into dst, where src is a gzipped stream. It will return the
// sha256 sum of the underlying content (the layerDigest) and the sha256 sum of the
// tar archive (the blobDigest) or an error. If the gzip layer has a modification time
// it will be returned.
// TODO: use configurable digests
func DigestCopy(dst io.ReaderFrom, src io.Reader) (layerDigest, blobDigest digest.Digest, modTime *time.Time, size int64, err error) {
	algo := digest.Canonical
	// calculate the blob digest as the sha256 sum of the uploaded contents
	blobhash := algo.Hash()
	// calculate the diffID as the sha256 sum of the layer contents
	pr, pw := io.Pipe()
	layerhash := algo.Hash()
	ch := make(chan error)
	go func() {
		defer close(ch)
		gr, err := gzip.NewReader(pr)
		if err != nil {
			ch <- fmt.Errorf("unable to create gzip reader layer upload: %v", err)
			return
		}
		if !gr.Header.ModTime.IsZero() {
			modTime = &gr.Header.ModTime
		}
		_, err = io.Copy(layerhash, gr)
		if err != nil {
			io.Copy(ioutil.Discard, pr)
		}
		ch <- err
	}()

	n, err := dst.ReadFrom(io.TeeReader(src, io.MultiWriter(blobhash, pw)))
	if err != nil {
		return "", "", nil, 0, fmt.Errorf("unable to upload new layer (%d): %v", n, err)
	}
	if err := pw.Close(); err != nil {
		return "", "", nil, 0, fmt.Errorf("unable to complete writing diffID: %v", err)
	}
	if err := <-ch; err != nil {
		return "", "", nil, 0, fmt.Errorf("unable to calculate layer diffID: %v", err)
	}

	layerDigest = digest.NewDigestFromBytes(algo, layerhash.Sum(make([]byte, 0, layerhash.Size())))
	blobDigest = digest.NewDigestFromBytes(algo, blobhash.Sum(make([]byte, 0, blobhash.Size())))
	return layerDigest, blobDigest, modTime, n, nil
}

func NewEmptyConfig() *docker10.DockerImageConfig {
	config := &docker10.DockerImageConfig{
		DockerVersion: "",
		// Created must be non-zero
		Created:      (time.Time{}).Add(1 * time.Second),
		OS:           runtime.GOOS,
		Architecture: runtime.GOARCH,
	}
	return config
}

func AddScratchLayerToConfig(config *docker10.DockerImageConfig) distribution.Descriptor {
	layer := distribution.Descriptor{
		MediaType: dockerV2Schema2LayerMediaType,
		Digest:    digest.Digest(dockerlayer.GzippedEmptyLayerDigest),
		Size:      int64(len(dockerlayer.GzippedEmptyLayer)),
	}
	AddLayerToConfig(config, layer, dockerlayer.EmptyLayerDiffID)
	return layer
}

func AddLayerToConfig(config *docker10.DockerImageConfig, layer distribution.Descriptor, diffID string) {
	if config.RootFS == nil {
		config.RootFS = &docker10.DockerConfigRootFS{Type: "layers"}
	}
	config.RootFS.DiffIDs = append(config.RootFS.DiffIDs, diffID)
	config.Size += layer.Size
}

func UploadSchema2Config(ctx context.Context, blobs distribution.BlobService, config *docker10.DockerImageConfig, layers []distribution.Descriptor) (*schema2.DeserializedManifest, error) {
	// ensure the image size is correct before persisting
	config.Size = 0
	for _, layer := range layers {
		config.Size += layer.Size
	}
	configJSON, err := json.Marshal(config)
	if err != nil {
		return nil, err
	}
	return putSchema2ImageConfig(ctx, blobs, dockerV2Schema2ConfigMediaType, configJSON, layers)
}

// putSchema2ImageConfig uploads the provided configJSON to the blob store and returns the generated manifest
// for the requested image.
func putSchema2ImageConfig(ctx context.Context, blobs distribution.BlobService, mediaType string, configJSON []byte, layers []distribution.Descriptor) (*schema2.DeserializedManifest, error) {
	b := schema2.NewManifestBuilder(blobs, mediaType, configJSON)
	for _, layer := range layers {
		if err := b.AppendReference(layer); err != nil {
			return nil, err
		}
	}
	m, err := b.Build(ctx)
	if err != nil {
		return nil, err
	}
	manifest, ok := m.(*schema2.DeserializedManifest)
	if !ok {
		return nil, fmt.Errorf("unable to turn %T into a DeserializedManifest, unable to store image", m)
	}
	return manifest, nil
}
<|endoftext|>


<commit_before>package configmap

import (
	"time"

	"github.com/containerum/chkit/pkg/model"
	kubeModels "github.com/containerum/kube-client/pkg/model"
)

type ConfigMap kubeModels.ConfigMap

func ConfigMapFromKube(kubeConfigMap kubeModels.ConfigMap) ConfigMap {
	return ConfigMap(kubeConfigMap).Copy()
}

func (config ConfigMap) ToKube() kubeModels.ConfigMap {
	return kubeModels.ConfigMap(config.Copy())
}

func (config ConfigMap) Copy() ConfigMap {
	var cm = config
	cm.Data = make(kubeModels.ConfigMapData, len(config.Data))
	for k, v := range config.Data {
		cm.Data[k] = v
	}
	return config
}

func (config ConfigMap) Set(key string, value string) ConfigMap {
	config = config.Copy()
	config.Data[key] = value
	return config
}

func (config ConfigMap) Add(data map[string]string) ConfigMap {
	config = config.Copy()
	for k, v := range data {
		config.Data[k] = v
	}
	return config
}

func (config ConfigMap) AddItems(items ...Item) ConfigMap {
	config = config.Copy()
	for _, item := range items {
		config.Data[item.Key] = item.Value
	}
	return config
}

func (config ConfigMap) Items() Items {
	var items = make([]Item, 0, len(config.Data))
	for k, v := range config.Data {
		items = append(items, Item{
			Key:   k,
			Value: v,
		})
	}
	return items
}

//  Get -- if defaultValues passed, then first return
func (config ConfigMap) Get(key string, defaultValues ...string) (string, bool) {
	value, ok := config.Data[key]
	if !ok {
		for _, defaultValue := range defaultValues {
			return defaultValue, ok
		}
	}
	return value, ok
}

func (config ConfigMap) Delete(key string) ConfigMap {
	config = config.Copy()
	delete(config.Data, key)
	return config
}

func (config ConfigMap) SetName(name string) ConfigMap {
	config.Name = name
	return config
}

func (config ConfigMap) Age() string {
	if timestamp, err := time.Parse(model.TimestampFormat, config.CreatedAt); err == nil {
		return model.Age(timestamp)
	}
	return "undefined"
}

func (config ConfigMap) New() ConfigMap {
	return ConfigMap{
		Data: make(kubeModels.ConfigMapData, len(config.Data)),
	}
}
<commit_msg>make items sorted<commit_after>package configmap

import (
	"time"

	"github.com/containerum/chkit/pkg/model"
	kubeModels "github.com/containerum/kube-client/pkg/model"
)

type ConfigMap kubeModels.ConfigMap

func ConfigMapFromKube(kubeConfigMap kubeModels.ConfigMap) ConfigMap {
	return ConfigMap(kubeConfigMap).Copy()
}

func (config ConfigMap) ToKube() kubeModels.ConfigMap {
	return kubeModels.ConfigMap(config.Copy())
}

func (config ConfigMap) Copy() ConfigMap {
	var cm = config
	cm.Data = make(kubeModels.ConfigMapData, len(config.Data))
	for k, v := range config.Data {
		cm.Data[k] = v
	}
	return config
}

func (config ConfigMap) Set(key string, value string) ConfigMap {
	config = config.Copy()
	config.Data[key] = value
	return config
}

func (config ConfigMap) Add(data map[string]string) ConfigMap {
	config = config.Copy()
	for k, v := range data {
		config.Data[k] = v
	}
	return config
}

func (config ConfigMap) AddItems(items ...Item) ConfigMap {
	config = config.Copy()
	for _, item := range items {
		config.Data[item.Key] = item.Value
	}
	return config
}

func (config ConfigMap) Items() Items {
	var items = make(Items, 0, len(config.Data))
	for k, v := range config.Data {
		items = append(items, Item{
			Key:   k,
			Value: v,
		})
	}
	return items.Sorted()
}

//  Get -- if defaultValues passed, then first return
func (config ConfigMap) Get(key string, defaultValues ...string) (string, bool) {
	value, ok := config.Data[key]
	if !ok {
		for _, defaultValue := range defaultValues {
			return defaultValue, ok
		}
	}
	return value, ok
}

func (config ConfigMap) Delete(key string) ConfigMap {
	config = config.Copy()
	delete(config.Data, key)
	return config
}

func (config ConfigMap) SetName(name string) ConfigMap {
	config.Name = name
	return config
}

func (config ConfigMap) Age() string {
	if timestamp, err := time.Parse(model.TimestampFormat, config.CreatedAt); err == nil {
		return model.Age(timestamp)
	}
	return "undefined"
}

func (config ConfigMap) New() ConfigMap {
	return ConfigMap{
		Data: make(kubeModels.ConfigMapData, len(config.Data)),
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resource

import (
	"errors"
	"fmt"

	openapi_v2 "github.com/googleapis/gnostic/openapiv2"
	yaml "gopkg.in/yaml.v2"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/discovery"
	"k8s.io/client-go/dynamic"
)

// VerifyDryRun returns nil if a resource group-version-kind supports
// server-side dry-run. Otherwise, an error is returned.
func VerifyDryRun(gvk schema.GroupVersionKind, dynamicClient dynamic.Interface, discoveryClient discovery.DiscoveryInterface) error {
	verifier := NewDryRunVerifier(dynamicClient, discoveryClient)
	return verifier.HasSupport(gvk)
}

func NewDryRunVerifier(dynamicClient dynamic.Interface, discoveryClient discovery.DiscoveryInterface) *DryRunVerifier {
	return &DryRunVerifier{
		finder:        NewCRDFinder(CRDFromDynamic(dynamicClient)),
		openAPIGetter: discoveryClient,
	}
}

func hasGVKExtension(extensions []*openapi_v2.NamedAny, gvk schema.GroupVersionKind) bool {
	for _, extension := range extensions {
		if extension.GetValue().GetYaml() == "" ||
			extension.GetName() != "x-kubernetes-group-version-kind" {
			continue
		}
		var value map[string]string
		err := yaml.Unmarshal([]byte(extension.GetValue().GetYaml()), &value)
		if err != nil {
			continue
		}

		if value["group"] == gvk.Group && value["kind"] == gvk.Kind && value["version"] == gvk.Version {
			return true
		}
		return false
	}
	return false
}

// DryRunVerifier verifies if a given group-version-kind supports DryRun
// against the current server. Sending dryRun requests to apiserver that
// don't support it will result in objects being unwillingly persisted.
//
// It reads the OpenAPI to see if the given GVK supports dryRun. If the
// GVK can not be found, we assume that CRDs will have the same level of
// support as "namespaces", and non-CRDs will not be supported. We
// delay the check for CRDs as much as possible though, since it
// requires an extra round-trip to the server.
type DryRunVerifier struct {
	finder        CRDFinder
	openAPIGetter discovery.OpenAPISchemaInterface
}

// HasSupport verifies if the given gvk supports DryRun. An error is
// returned if it doesn't.
func (v *DryRunVerifier) HasSupport(gvk schema.GroupVersionKind) error {
	oapi, err := v.openAPIGetter.OpenAPISchema()
	if err != nil {
		return fmt.Errorf("failed to download openapi: %v", err)
	}
	supports, err := supportsDryRun(oapi, gvk)
	if err != nil {
		// We assume that we couldn't find the type, then check for namespace:
		supports, _ = supportsDryRun(oapi, schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"})
		// If namespace supports dryRun, then we will support dryRun for CRDs only.
		if supports {
			supports, err = v.finder.HasCRD(gvk.GroupKind())
			if err != nil {
				return fmt.Errorf("failed to check CRD: %v", err)
			}
		}
	}
	if !supports {
		return fmt.Errorf("%v doesn't support dry-run", gvk)
	}
	return nil
}

// supportsDryRun is a method that let's us look in the OpenAPI if the
// specific group-version-kind supports the dryRun query parameter for
// the PATCH end-point.
func supportsDryRun(doc *openapi_v2.Document, gvk schema.GroupVersionKind) (bool, error) {
	for _, path := range doc.GetPaths().GetPath() {
		// Is this describing the gvk we're looking for?
		if !hasGVKExtension(path.GetValue().GetPatch().GetVendorExtension(), gvk) {
			continue
		}
		for _, param := range path.GetValue().GetPatch().GetParameters() {
			if param.GetParameter().GetNonBodyParameter().GetQueryParameterSubSchema().GetName() == "dryRun" {
				return true, nil
			}
		}
		return false, nil
	}

	return false, errors.New("couldn't find GVK in openapi")
}
<commit_msg>Cache the OpenAPI schema for server-side dry-run.<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resource

import (
	"errors"
	"fmt"

	openapi_v2 "github.com/googleapis/gnostic/openapiv2"
	yaml "gopkg.in/yaml.v2"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/discovery"
	"k8s.io/client-go/dynamic"
)

func NewDryRunVerifier(dynamicClient dynamic.Interface, openAPIGetter discovery.OpenAPISchemaInterface) *DryRunVerifier {
	return &DryRunVerifier{
		finder:        NewCRDFinder(CRDFromDynamic(dynamicClient)),
		openAPIGetter: openAPIGetter,
	}
}

func hasGVKExtension(extensions []*openapi_v2.NamedAny, gvk schema.GroupVersionKind) bool {
	for _, extension := range extensions {
		if extension.GetValue().GetYaml() == "" ||
			extension.GetName() != "x-kubernetes-group-version-kind" {
			continue
		}
		var value map[string]string
		err := yaml.Unmarshal([]byte(extension.GetValue().GetYaml()), &value)
		if err != nil {
			continue
		}

		if value["group"] == gvk.Group && value["kind"] == gvk.Kind && value["version"] == gvk.Version {
			return true
		}
		return false
	}
	return false
}

// DryRunVerifier verifies if a given group-version-kind supports DryRun
// against the current server. Sending dryRun requests to apiserver that
// don't support it will result in objects being unwillingly persisted.
//
// It reads the OpenAPI to see if the given GVK supports dryRun. If the
// GVK can not be found, we assume that CRDs will have the same level of
// support as "namespaces", and non-CRDs will not be supported. We
// delay the check for CRDs as much as possible though, since it
// requires an extra round-trip to the server.
type DryRunVerifier struct {
	finder        CRDFinder
	openAPIGetter discovery.OpenAPISchemaInterface
}

// HasSupport verifies if the given gvk supports DryRun. An error is
// returned if it doesn't.
func (v *DryRunVerifier) HasSupport(gvk schema.GroupVersionKind) error {
	oapi, err := v.openAPIGetter.OpenAPISchema()
	if err != nil {
		return fmt.Errorf("failed to download openapi: %v", err)
	}
	supports, err := supportsDryRun(oapi, gvk)
	if err != nil {
		// We assume that we couldn't find the type, then check for namespace:
		supports, _ = supportsDryRun(oapi, schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"})
		// If namespace supports dryRun, then we will support dryRun for CRDs only.
		if supports {
			supports, err = v.finder.HasCRD(gvk.GroupKind())
			if err != nil {
				return fmt.Errorf("failed to check CRD: %v", err)
			}
		}
	}
	if !supports {
		return fmt.Errorf("%v doesn't support dry-run", gvk)
	}
	return nil
}

// supportsDryRun is a method that let's us look in the OpenAPI if the
// specific group-version-kind supports the dryRun query parameter for
// the PATCH end-point.
func supportsDryRun(doc *openapi_v2.Document, gvk schema.GroupVersionKind) (bool, error) {
	for _, path := range doc.GetPaths().GetPath() {
		// Is this describing the gvk we're looking for?
		if !hasGVKExtension(path.GetValue().GetPatch().GetVendorExtension(), gvk) {
			continue
		}
		for _, param := range path.GetValue().GetPatch().GetParameters() {
			if param.GetParameter().GetNonBodyParameter().GetQueryParameterSubSchema().GetName() == "dryRun" {
				return true, nil
			}
		}
		return false, nil
	}

	return false, errors.New("couldn't find GVK in openapi")
}
<|endoftext|>


<commit_before>package eve

import (
	"encoding/xml"
	"fmt"
	"net/http"
	"io/ioutil"
)
const eveUrl string = "https://api.eveonline.com"

type Character struct {
	Name	string	`xml:"name,attr"`
	ID	string	`xml:"characterID,attr"`
	Corporation	string	`xml:"corporationName,attr"`
	CorporationID	string	`xml:"corporationID,attr"`
}

type CharactersResult struct {
	XMLName		xml.Name	`xml:"eveapi"`
	CurrentTime	string		`xml:"currentTime"`
	Characters	[]Character	`xml:"result>rowset>row"`
	CachedUntil	string		`xml:"cachedUntil"`
}

// Characters fetches, parses and returns a set of Eve Online characters.
func Characters(keyID string, vCode string) (res CharactersResult) {
	var v CharactersResult

	url := fmt.Sprintf("%s/account/Characters.xml.aspx?keyID=%s&%s", eveUrl, keyID, vCode)
	data, err := fetch(url)
	if err != nil {
		fmt.Printf("Fetch error: %v", err)
		return
	}

	err := xml.Unmarshal(data, &v)
	if err != nil {
		fmt.Printf("Unmarshal error: %v", err)
		return
	}

	return v
}

func fetch(url string) ([]byte, error) {
	res, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	data, err := ioutil.ReadAll(res.Body)
	res.Body.Close()

	if err != nil {
		return nil, err
	}
	return []byte(data), nil
}
<commit_msg>Cleanup<commit_after>package eve

import (
	"encoding/xml"
	"fmt"
	"net/http"
	"io/ioutil"
)
const eveUrl string = "https://api.eveonline.com"

type Character struct {
	Name	string	`xml:"name,attr"`
	ID	string	`xml:"characterID,attr"`
	Corporation	string	`xml:"corporationName,attr"`
	CorporationID	string	`xml:"corporationID,attr"`
}

type CharactersResult struct {
	XMLName		xml.Name	`xml:"eveapi"`
	CurrentTime	string		`xml:"currentTime"`
	Characters	[]Character	`xml:"result>rowset>row"`
	CachedUntil	string		`xml:"cachedUntil"`
}

// Characters fetches, parses and returns a set of Eve Online characters.
func Characters(keyID string, vCode string) (res CharactersResult) {
	var v CharactersResult

	url := fmt.Sprintf("%s/account/Characters.xml.aspx?keyID=%s&%s", eveUrl, keyID, vCode)
	data, err := fetch(url)
	if err != nil {
		fmt.Printf("Fetch error: %v", err)
		return
	}

	err = xml.Unmarshal(data, &v)
	if err != nil {
		fmt.Printf("Unmarshal error: %v", err)
		return
	}

	return v
}

func fetch(url string) ([]byte, error) {
	res, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	data, err := ioutil.ReadAll(res.Body)
	res.Body.Close()

	if err != nil {
		return nil, err
	}
	return []byte(data), nil
}
<|endoftext|>


<commit_before>package main

import(
	"crypto/md5"
	"database/sql"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	_ "github.com/mattn/go-sqlite3"
)

var(
	verp = flag.Bool("version", false, "Show version info")
	facetp = flag.Bool("facets", false, "List facets")
	scanp = flag.Bool("scan", false, "Check for new/modified files and sweep db for orphan records")
	cleanp = flag.Bool("cleandb", false, "Just clean db (no file scan)")
	tagp = flag.Bool("tag", false, "Tag [dir] with [facet]")
	getp = flag.Bool("get", false, "Get filenames for tracks tagged with [facet]")
	mdflag = flag.String("musicdir", "", "Set location of your mpd music directory")
	musicdir = ""
	seen      int64
	touched   int64
)

func init() {
	flag.Parse()
	config := os.Getenv("HOME") + "/.mpcf"
	if *mdflag != "" {
		err := ioutil.WriteFile(config, []byte(*mdflag), 0644)
		if err != nil {
			log.Fatal(err)
		}
		musicdir = *mdflag
	} else {
		mdbytes, err := ioutil.ReadFile(config)
		if err != nil {
			log.Fatal("Please run 'mpcf -musicdir /path/to/music' to set your musicdir path.")
		}
		musicdir = string(mdbytes)
	}
}

func main() {
	db, err := sql.Open("sqlite3", musicdir + "/.mpcf.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	// create db if needed
	var tracks int
	res := db.QueryRow("select count(id) from tracks")
	err = res.Scan(&tracks)
	if err != nil {
		db.Exec("PRAGMA synchronous=0")
		log.Println("Creating db")
		createdb(db)
		log.Println("Updating track list")
		scandir("", db)
	}

	if *verp {
		fmt.Println("This is mpcf v0.5.3")
		os.Exit(0)
	}
	if *scanp {
		db.Exec("PRAGMA synchronous=0")
		scandir("", db)
		cleandb(db)
		os.Exit(0)
	}
	if *cleanp {
		cleandb(db)
		os.Exit(0)
	}
	if *tagp {
		tagdir(flag.Args(), db)
		os.Exit(0)
	}
	if *getp {
		getfacettracks(flag.Args(), db)
		os.Exit(0)
	}
	if *facetp {
		lsfacets(db)
		os.Exit(0)
	}

	var tags, facets int
	db.QueryRow("select count(tid) from t2f").Scan(&tags)
	db.QueryRow("select count(id) from facets").Scan(&facets)
	fmt.Printf("%v tracks; %v tagged, with %v facets\n", tracks, tags, facets)
}

func lsfacets(db *sql.DB) {
	rows, err := db.Query("SELECT facet FROM facets ORDER BY facet")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	var f string
	for rows.Next() {
		if err := rows.Scan(&f); err != nil {
			log.Fatal(err)
		}
		fmt.Println(f)
	}
	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
}

func getfacettracks(args []string, db *sql.DB) {
	if len(args) != 1 {
		log.Fatal("Too many/few arguments to -get; need a facet name")
	}
	var fid int
	db.QueryRow("select id from facets where facet = ?", args[0]).Scan(&fid)
	if fid == 0 {
		return
	}
	rows, err := db.Query("SELECT filename FROM tracks WHERE id IN (SELECT DISTINCT tid FROM t2f WHERE fid = ?)", fid)
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	var name string
	for rows.Next() {
		if err := rows.Scan(&name); err != nil {
			log.Fatal(err)
		}
		fmt.Println(name)
	}
	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
}

func tagdir(args []string, db *sql.DB) {
	if len(args) != 2 {
		log.Fatal("Too many/few arguments to -tag; need a directory and a facet")
	}
	// create the tag if it doesn't exist
	var fid int
	db.QueryRow("select id from facets where facet = ?", args[1]).Scan(&fid)
	if fid == 0 {
		db.Exec("insert into facets (facet) values (?)", args[1])
		db.QueryRow("select id from facets where facet = ?", args[1]).Scan(&fid)
	}
	// now actually tag tracks under this dir
	args[0] = strings.TrimRight(args[0], "/")
	args[0] = strings.TrimLeft(args[0], "./")
	args[0] = strings.TrimLeft(args[0], musicdir)
	tagdir2(args[0], fid, db)
}

func tagdir2(dir string, fid int, db *sql.DB) {
	err := os.Chdir(musicdir + "/" + dir)
	if err != nil {
		log.Fatalf("Can't chdir to %v", dir)
	}
	ls, err := ioutil.ReadDir(".")
	for _, direntry := range ls {
		name := dir + "/" + direntry.Name()
		if direntry.IsDir() {
			tagdir2(name, fid, db)
		} else {
			var tid, fcnt int
			db.QueryRow("select id from tracks where filename = ?", name).Scan(&tid)
			db.QueryRow("select count(tid) from t2f where tid = ? and fid = ?", tid, fid).Scan(&fcnt)
			if fcnt > 0 {
				continue
			}
			db.Exec("insert into t2f (tid, fid) values (?, ?)", tid, fid)
		}
	}
}

func createdb(db *sql.DB) {
	var err error
	var stmts = []string{
		"create table tracks (id integer primary key, filename text unique, hash text unique)",
		"create table facets (id integer primary key, facet text)",
		"create table t2f (tid integer, fid integer)",
		"create index fididx on t2f(fid)",
		"create table config (key text, value text)",
		"insert into config (key, value) values('mpdconf', '/etc/mpd.conf')",
	}
	for _, stmt := range stmts {
		if err != nil {
			break
		}
		_, err = db.Exec(stmt)
	}
	if err != nil {
		log.Fatal(err)
	}
}

func scandir(dir string, db *sql.DB) {
	os.Chdir(musicdir + "/" + dir)
	ls, err := ioutil.ReadDir(".")
	if err != nil {
		log.Fatal(err, dir)
	}
	for _, direntry := range ls {
		if direntry.IsDir() {
			if dir == "" {
				scandir(direntry.Name(), db)
			} else {
				scandir(dir + "/" + direntry.Name(), db)
			}
		} else {
			seen ++
			if seen % 100 == 0 {
				log.Printf("Processed %v tracks; updated %v\n", seen, touched)
			}
			name := dir + "/" + direntry.Name()
			md5 := fmt.Sprintf("%x", calcMD5(direntry.Name()))
			// _, err := db.Exec("INSERT OR REPLACE INTO tracks (filename, hash) VALUES(COALESCE((SELECT filename FROM tracks WHERE filename = ?),?), COALESCE((SELECT hash FROM tracks WHERE hash = ?), ?))", name, name, md5, md5)
			r, err := db.Exec("INSERT OR IGNORE INTO tracks (filename, hash) VALUES(?, ?)", name, md5)
			if err != nil {
				log.Fatal(err)
			}
			touch, _ := r.RowsAffected()
			touched += touch
			//r, err = db.Exec("UPDATE tracks SET filename = ?, hash = ? WHERE filename = ?", name, md5, name)
			//if err != nil {
			//	log.Fatal(err)
			//}
		}
	}
}

func cleandb(db *sql.DB) {
	log.Printf("Scanning db for orphaned records")
	rows, err := db.Query("SELECT id, filename FROM tracks")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	var id int64
	var name string
	for rows.Next() {
		if err := rows.Scan(&id, &name); err != nil {
			log.Fatal(err)
		}
		_, err = os.Stat(musicdir + "/" + name)
		if err == nil {
			continue
		}
		// remove track entry
		_, err = db.Exec("delete from tracks where id = ?", id)
		if err != nil {
			log.Fatal(err)
		}
		// remove tag links
		_, err = db.Exec("delete from t2f where tid = ?", id)
		if err != nil {
			log.Fatal(err)
		}
		log.Printf("Removed orphan record for %v\n", name)
	}
	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
	_, err = db.Exec("vacuum")
}

func calcMD5(filename string) []byte {
	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	hash := md5.New()
	if _, err := io.CopyN(hash, file, 524288); err != nil && err != io.EOF {
		log.Fatal(err)
	}
	return hash.Sum(nil)
}
<commit_msg>better reporting in default usage<commit_after>package main

import(
	"crypto/md5"
	"database/sql"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	_ "github.com/mattn/go-sqlite3"
)

var(
	verp = flag.Bool("version", false, "Show version info")
	facetp = flag.Bool("facets", false, "List facets")
	scanp = flag.Bool("scan", false, "Check for new/modified files and sweep db for orphan records")
	cleanp = flag.Bool("cleandb", false, "Just clean db (no file scan)")
	tagp = flag.Bool("tag", false, "Tag [dir] with [facet]")
	getp = flag.Bool("get", false, "Get filenames for tracks tagged with [facet]")
	mdflag = flag.String("musicdir", "", "Set location of your mpd music directory")
	musicdir = ""
	seen      int64
	touched   int64
)

func init() {
	flag.Parse()
	config := os.Getenv("HOME") + "/.mpcf"
	if *mdflag != "" {
		err := ioutil.WriteFile(config, []byte(*mdflag), 0644)
		if err != nil {
			log.Fatal(err)
		}
		musicdir = *mdflag
	} else {
		mdbytes, err := ioutil.ReadFile(config)
		if err != nil {
			log.Fatal("Please run 'mpcf -musicdir /path/to/music' to set your musicdir path.")
		}
		musicdir = string(mdbytes)
	}
}

func main() {
	db, err := sql.Open("sqlite3", musicdir + "/.mpcf.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	// create db if needed
	var tracks int
	res := db.QueryRow("select count(id) from tracks")
	err = res.Scan(&tracks)
	if err != nil {
		db.Exec("PRAGMA synchronous=0")
		log.Println("Creating db")
		createdb(db)
		log.Println("Updating track list")
		scandir("", db)
	}

	if *verp {
		fmt.Println("This is mpcf v0.5.3")
		os.Exit(0)
	}
	if *scanp {
		db.Exec("PRAGMA synchronous=0")
		scandir("", db)
		cleandb(db)
		os.Exit(0)
	}
	if *cleanp {
		cleandb(db)
		os.Exit(0)
	}
	if *tagp {
		tagdir(flag.Args(), db)
		os.Exit(0)
	}
	if *getp {
		getfacettracks(flag.Args(), db)
		os.Exit(0)
	}
	if *facetp {
		lsfacets(db)
		os.Exit(0)
	}

	var taggedtracks, tags, facets int
	db.QueryRow("select count(tid) from t2f").Scan(&tags)
	db.QueryRow("select count(distinct tid) from t2f").Scan(&taggedtracks)
	db.QueryRow("select count(id) from facets").Scan(&facets)
	fmt.Printf("%v tracks (%v tagged); %v tags across %v facets\n", tracks, taggedtracks, tags, facets)
}

func lsfacets(db *sql.DB) {
	rows, err := db.Query("SELECT facet FROM facets ORDER BY facet")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	var f string
	for rows.Next() {
		if err := rows.Scan(&f); err != nil {
			log.Fatal(err)
		}
		fmt.Println(f)
	}
	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
}

func getfacettracks(args []string, db *sql.DB) {
	if len(args) != 1 {
		log.Fatal("Too many/few arguments to -get; need a facet name")
	}
	var fid int
	db.QueryRow("select id from facets where facet = ?", args[0]).Scan(&fid)
	if fid == 0 {
		return
	}
	rows, err := db.Query("SELECT filename FROM tracks WHERE id IN (SELECT DISTINCT tid FROM t2f WHERE fid = ?)", fid)
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	var name string
	for rows.Next() {
		if err := rows.Scan(&name); err != nil {
			log.Fatal(err)
		}
		fmt.Println(name)
	}
	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
}

func tagdir(args []string, db *sql.DB) {
	if len(args) != 2 {
		log.Fatal("Too many/few arguments to -tag; need a directory and a facet")
	}
	// create the tag if it doesn't exist
	var fid int
	db.QueryRow("select id from facets where facet = ?", args[1]).Scan(&fid)
	if fid == 0 {
		db.Exec("insert into facets (facet) values (?)", args[1])
		db.QueryRow("select id from facets where facet = ?", args[1]).Scan(&fid)
	}
	// now actually tag tracks under this dir
	args[0] = strings.TrimRight(args[0], "/")
	args[0] = strings.TrimLeft(args[0], "./")
	args[0] = strings.TrimLeft(args[0], musicdir)
	tagdir2(args[0], fid, db)
}

func tagdir2(dir string, fid int, db *sql.DB) {
	err := os.Chdir(musicdir + "/" + dir)
	if err != nil {
		log.Fatalf("Can't chdir to %v", dir)
	}
	ls, err := ioutil.ReadDir(".")
	for _, direntry := range ls {
		name := dir + "/" + direntry.Name()
		if direntry.IsDir() {
			tagdir2(name, fid, db)
		} else {
			var tid, fcnt int
			db.QueryRow("select id from tracks where filename = ?", name).Scan(&tid)
			db.QueryRow("select count(tid) from t2f where tid = ? and fid = ?", tid, fid).Scan(&fcnt)
			if fcnt > 0 {
				continue
			}
			db.Exec("insert into t2f (tid, fid) values (?, ?)", tid, fid)
		}
	}
}

func createdb(db *sql.DB) {
	var err error
	var stmts = []string{
		"create table tracks (id integer primary key, filename text unique, hash text unique)",
		"create table facets (id integer primary key, facet text)",
		"create table t2f (tid integer, fid integer)",
		"create index fididx on t2f(fid)",
		"create table config (key text, value text)",
		"insert into config (key, value) values('mpdconf', '/etc/mpd.conf')",
	}
	for _, stmt := range stmts {
		if err != nil {
			break
		}
		_, err = db.Exec(stmt)
	}
	if err != nil {
		log.Fatal(err)
	}
}

func scandir(dir string, db *sql.DB) {
	os.Chdir(musicdir + "/" + dir)
	ls, err := ioutil.ReadDir(".")
	if err != nil {
		log.Fatal(err, dir)
	}
	for _, direntry := range ls {
		if direntry.IsDir() {
			if dir == "" {
				scandir(direntry.Name(), db)
			} else {
				scandir(dir + "/" + direntry.Name(), db)
			}
		} else {
			seen ++
			if seen % 100 == 0 {
				log.Printf("Processed %v tracks; updated %v\n", seen, touched)
			}
			name := dir + "/" + direntry.Name()
			md5 := fmt.Sprintf("%x", calcMD5(direntry.Name()))
			// _, err := db.Exec("INSERT OR REPLACE INTO tracks (filename, hash) VALUES(COALESCE((SELECT filename FROM tracks WHERE filename = ?),?), COALESCE((SELECT hash FROM tracks WHERE hash = ?), ?))", name, name, md5, md5)
			r, err := db.Exec("INSERT OR IGNORE INTO tracks (filename, hash) VALUES(?, ?)", name, md5)
			if err != nil {
				log.Fatal(err)
			}
			touch, _ := r.RowsAffected()
			touched += touch
			//r, err = db.Exec("UPDATE tracks SET filename = ?, hash = ? WHERE filename = ?", name, md5, name)
			//if err != nil {
			//	log.Fatal(err)
			//}
		}
	}
}

func cleandb(db *sql.DB) {
	log.Printf("Scanning db for orphaned records")
	rows, err := db.Query("SELECT id, filename FROM tracks")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	var id int64
	var name string
	for rows.Next() {
		if err := rows.Scan(&id, &name); err != nil {
			log.Fatal(err)
		}
		_, err = os.Stat(musicdir + "/" + name)
		if err == nil {
			continue
		}
		// remove track entry
		_, err = db.Exec("delete from tracks where id = ?", id)
		if err != nil {
			log.Fatal(err)
		}
		// remove tag links
		_, err = db.Exec("delete from t2f where tid = ?", id)
		if err != nil {
			log.Fatal(err)
		}
		log.Printf("Removed orphan record for %v\n", name)
	}
	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
	_, err = db.Exec("vacuum")
}

func calcMD5(filename string) []byte {
	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	hash := md5.New()
	if _, err := io.CopyN(hash, file, 524288); err != nil && err != io.EOF {
		log.Fatal(err)
	}
	return hash.Sum(nil)
}
<|endoftext|>


<commit_before>// Copyright © 2015 Jerry Jacobs <jerry.jacobs@xor-gate.org>.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sftpfs

import (
	"os"
	"time"

	"github.com/pkg/sftp"
	"github.com/spf13/afero"
)

// Fs is a afero.Fs implementation that uses functions provided by the sftp package.
//
// For details in any method, check the documentation of the sftp package
// (github.com/pkg/sftp).
type Fs struct {
	client *sftp.Client
}

func New(client *sftp.Client) afero.Fs {
	return &Fs{client: client}
}

func (s Fs) Name() string { return "sftpfs" }

func (s Fs) Create(name string) (afero.File, error) {
	return FileCreate(s.client, name)
}

func (s Fs) Mkdir(name string, perm os.FileMode) error {
	err := s.client.Mkdir(name)
	if err != nil {
		return err
	}
	return s.client.Chmod(name, perm)
}

func (s Fs) MkdirAll(path string, perm os.FileMode) error {
	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
	dir, err := s.Stat(path)
	if err == nil {
		if dir.IsDir() {
			return nil
		}
		return err
	}

	// Slow path: make sure parent exists and then call Mkdir for path.
	i := len(path)
	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
		i--
	}

	j := i
	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
		j--
	}

	if j > 1 {
		// Create parent
		err = s.MkdirAll(path[0:j-1], perm)
		if err != nil {
			return err
		}
	}

	// Parent now exists; invoke Mkdir and use its result.
	err = s.Mkdir(path, perm)
	if err != nil {
		// Handle arguments like "foo/." by
		// double-checking that directory doesn't exist.
		dir, err1 := s.Lstat(path)
		if err1 == nil && dir.IsDir() {
			return nil
		}
		return err
	}
	return nil
}

func (s Fs) Open(name string) (afero.File, error) {
	return FileOpen(s.client, name)
}

func (s Fs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
	return nil, nil
}

func (s Fs) Remove(name string) error {
	return s.client.Remove(name)
}

func (s Fs) RemoveAll(path string) error {
	// TODO have a look at os.RemoveAll
	// https://github.com/golang/go/blob/master/src/os/path.go#L66
	return nil
}

func (s Fs) Rename(oldname, newname string) error {
	return s.client.Rename(oldname, newname)
}

func (s Fs) Stat(name string) (os.FileInfo, error) {
	return s.client.Stat(name)
}

func (s Fs) Lstat(p string) (os.FileInfo, error) {
	return s.client.Lstat(p)
}

func (s Fs) Chmod(name string, mode os.FileMode) error {
	return s.client.Chmod(name, mode)
}

func (s Fs) Chtimes(name string, atime time.Time, mtime time.Time) error {
	return s.client.Chtimes(name, atime, mtime)
}
<commit_msg>sftpfs: Implement OpenFile method<commit_after>// Copyright © 2015 Jerry Jacobs <jerry.jacobs@xor-gate.org>.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sftpfs

import (
	"os"
	"time"

	"github.com/pkg/sftp"
	"github.com/spf13/afero"
)

// Fs is a afero.Fs implementation that uses functions provided by the sftp package.
//
// For details in any method, check the documentation of the sftp package
// (github.com/pkg/sftp).
type Fs struct {
	client *sftp.Client
}

func New(client *sftp.Client) afero.Fs {
	return &Fs{client: client}
}

func (s Fs) Name() string { return "sftpfs" }

func (s Fs) Create(name string) (afero.File, error) {
	return FileCreate(s.client, name)
}

func (s Fs) Mkdir(name string, perm os.FileMode) error {
	err := s.client.Mkdir(name)
	if err != nil {
		return err
	}
	return s.client.Chmod(name, perm)
}

func (s Fs) MkdirAll(path string, perm os.FileMode) error {
	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
	dir, err := s.Stat(path)
	if err == nil {
		if dir.IsDir() {
			return nil
		}
		return err
	}

	// Slow path: make sure parent exists and then call Mkdir for path.
	i := len(path)
	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
		i--
	}

	j := i
	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
		j--
	}

	if j > 1 {
		// Create parent
		err = s.MkdirAll(path[0:j-1], perm)
		if err != nil {
			return err
		}
	}

	// Parent now exists; invoke Mkdir and use its result.
	err = s.Mkdir(path, perm)
	if err != nil {
		// Handle arguments like "foo/." by
		// double-checking that directory doesn't exist.
		dir, err1 := s.Lstat(path)
		if err1 == nil && dir.IsDir() {
			return nil
		}
		return err
	}
	return nil
}

func (s Fs) Open(name string) (afero.File, error) {
	return FileOpen(s.client, name)
}

// OpenFile calls the OpenFile method on the SSHFS connection. The mode argument
// is ignored because it's ignored by the github.com/pkg/sftp implementation.
func (s Fs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
	sshfsFile, err := s.client.OpenFile(name, flag)
	if err != nil {
		return nil, err
	}
	return &File{fd: sshfsFile}, nil
}

func (s Fs) Remove(name string) error {
	return s.client.Remove(name)
}

func (s Fs) RemoveAll(path string) error {
	// TODO have a look at os.RemoveAll
	// https://github.com/golang/go/blob/master/src/os/path.go#L66
	return nil
}

func (s Fs) Rename(oldname, newname string) error {
	return s.client.Rename(oldname, newname)
}

func (s Fs) Stat(name string) (os.FileInfo, error) {
	return s.client.Stat(name)
}

func (s Fs) Lstat(p string) (os.FileInfo, error) {
	return s.client.Lstat(p)
}

func (s Fs) Chmod(name string, mode os.FileMode) error {
	return s.client.Chmod(name, mode)
}

func (s Fs) Chtimes(name string, atime time.Time, mtime time.Time) error {
	return s.client.Chtimes(name, atime, mtime)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package repltracker

import (
	"fmt"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/mysql/fakesqldb"
	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/dbconfigs"
	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	"vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv"
)

var (
	now         = time.Now()
	mockNowFunc = func() time.Time {
		return now
	}
)

func TestCreateSchema(t *testing.T) {
	db := fakesqldb.New(t)
	defer db.Close()
	tw := newTestWriter(db, mockNowFunc)
	defer tw.Close()
	writes.Reset()

	db.OrderMatters()
	upsert := fmt.Sprintf("INSERT INTO %s.heartbeat (ts, tabletUid, keyspaceShard) VALUES (%d, %d, '%s') ON DUPLICATE KEY UPDATE ts=VALUES(ts), tabletUid=VALUES(tabletUid)",
		"_vt", now.UnixNano(), tw.tabletAlias.Uid, tw.keyspaceShard)
	failInsert := fakesqldb.ExpectedExecuteFetch{
		Query: upsert,
		Error: mysql.NewSQLError(mysql.ERBadDb, "", "bad db error"),
	}
	db.AddExpectedExecuteFetch(failInsert)
	db.AddExpectedQuery(fmt.Sprintf(sqlCreateSidecarDB, "_vt"), nil)
	db.AddExpectedQuery(fmt.Sprintf(sqlCreateHeartbeatTable, "_vt"), nil)
	db.AddExpectedQuery(upsert, nil)

	err := tw.write()
	require.NoError(t, err)
}

func TestWriteHeartbeat(t *testing.T) {
	db := fakesqldb.New(t)
	defer db.Close()

	tw := newTestWriter(db, mockNowFunc)
	upsert := fmt.Sprintf("INSERT INTO %s.heartbeat (ts, tabletUid, keyspaceShard) VALUES (%d, %d, '%s') ON DUPLICATE KEY UPDATE ts=VALUES(ts), tabletUid=VALUES(tabletUid)",
		"_vt", now.UnixNano(), tw.tabletAlias.Uid, tw.keyspaceShard)
	db.AddQuery(upsert, &sqltypes.Result{})

	writes.Reset()
	writeErrors.Reset()

	tw.writeHeartbeat()
	assert.Equal(t, int64(1), writes.Get())
	assert.Equal(t, int64(0), writeErrors.Get())
}

func TestWriteHeartbeatError(t *testing.T) {
	db := fakesqldb.New(t)
	defer db.Close()

	tw := newTestWriter(db, mockNowFunc)

	writes.Reset()
	writeErrors.Reset()

	tw.writeHeartbeat()
	assert.Equal(t, int64(0), writes.Get())
	assert.Equal(t, int64(1), writeErrors.Get())
}

func newTestWriter(db *fakesqldb.DB, nowFunc func() time.Time) *heartbeatWriter {
	config := tabletenv.NewDefaultConfig()
	config.ReplicationTracker.Mode = tabletenv.Heartbeat
	config.ReplicationTracker.HeartbeatIntervalSeconds = 1

	params, _ := db.ConnParams().MysqlParams()
	cp := *params
	dbc := dbconfigs.NewTestDBConfigs(cp, cp, "")

	tw := newHeartbeatWriter(tabletenv.NewEnv(config, "WriterTest"), &topodatapb.TabletAlias{Cell: "test", Uid: 1111})
	tw.keyspaceShard = "test:0"
	tw.now = nowFunc
	tw.pool.Open(dbc.DbaWithDB())

	return tw
}
<commit_msg>test: also change the tests to use the app user<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package repltracker

import (
	"fmt"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/mysql/fakesqldb"
	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/dbconfigs"
	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	"vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv"
)

var (
	now         = time.Now()
	mockNowFunc = func() time.Time {
		return now
	}
)

func TestCreateSchema(t *testing.T) {
	db := fakesqldb.New(t)
	defer db.Close()
	tw := newTestWriter(db, mockNowFunc)
	defer tw.Close()
	writes.Reset()

	db.OrderMatters()
	upsert := fmt.Sprintf("INSERT INTO %s.heartbeat (ts, tabletUid, keyspaceShard) VALUES (%d, %d, '%s') ON DUPLICATE KEY UPDATE ts=VALUES(ts), tabletUid=VALUES(tabletUid)",
		"_vt", now.UnixNano(), tw.tabletAlias.Uid, tw.keyspaceShard)
	failInsert := fakesqldb.ExpectedExecuteFetch{
		Query: upsert,
		Error: mysql.NewSQLError(mysql.ERBadDb, "", "bad db error"),
	}
	db.AddExpectedExecuteFetch(failInsert)
	db.AddExpectedQuery(fmt.Sprintf(sqlCreateSidecarDB, "_vt"), nil)
	db.AddExpectedQuery(fmt.Sprintf(sqlCreateHeartbeatTable, "_vt"), nil)
	db.AddExpectedQuery(upsert, nil)

	err := tw.write()
	require.NoError(t, err)
}

func TestWriteHeartbeat(t *testing.T) {
	db := fakesqldb.New(t)
	defer db.Close()

	tw := newTestWriter(db, mockNowFunc)
	upsert := fmt.Sprintf("INSERT INTO %s.heartbeat (ts, tabletUid, keyspaceShard) VALUES (%d, %d, '%s') ON DUPLICATE KEY UPDATE ts=VALUES(ts), tabletUid=VALUES(tabletUid)",
		"_vt", now.UnixNano(), tw.tabletAlias.Uid, tw.keyspaceShard)
	db.AddQuery(upsert, &sqltypes.Result{})

	writes.Reset()
	writeErrors.Reset()

	tw.writeHeartbeat()
	assert.Equal(t, int64(1), writes.Get())
	assert.Equal(t, int64(0), writeErrors.Get())
}

func TestWriteHeartbeatError(t *testing.T) {
	db := fakesqldb.New(t)
	defer db.Close()

	tw := newTestWriter(db, mockNowFunc)

	writes.Reset()
	writeErrors.Reset()

	tw.writeHeartbeat()
	assert.Equal(t, int64(0), writes.Get())
	assert.Equal(t, int64(1), writeErrors.Get())
}

func newTestWriter(db *fakesqldb.DB, nowFunc func() time.Time) *heartbeatWriter {
	config := tabletenv.NewDefaultConfig()
	config.ReplicationTracker.Mode = tabletenv.Heartbeat
	config.ReplicationTracker.HeartbeatIntervalSeconds = 1

	params, _ := db.ConnParams().MysqlParams()
	cp := *params
	dbc := dbconfigs.NewTestDBConfigs(cp, cp, "")

	tw := newHeartbeatWriter(tabletenv.NewEnv(config, "WriterTest"), &topodatapb.TabletAlias{Cell: "test", Uid: 1111})
	tw.keyspaceShard = "test:0"
	tw.now = nowFunc
	tw.pool.Open(dbc.AppWithDB())

	return tw
}
<|endoftext|>