<commit_before>// Copyright 2015 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package parser

import (
	"bytes"
	"fmt"
	"reflect"
	"strings"
	"testing"
)

func TestInterfaceToConfig(t *testing.T) {
	cases := []struct {
		input interface{}
		mark  byte
		want  []byte
		isErr bool
	}{
		// TOML
		{map[string]interface{}{}, TOMLLead[0], nil, false},
		{
			map[string]interface{}{"title": "test 1"},
			TOMLLead[0],
			[]byte("title = \"test 1\"\n"),
			false,
		},

		// YAML
		{map[string]interface{}{}, YAMLLead[0], []byte("{}\n"), false},
		{
			map[string]interface{}{"title": "test 1"},
			YAMLLead[0],
			[]byte("title: test 1\n"),
			false,
		},

		// JSON
		{map[string]interface{}{}, JSONLead[0], []byte("{}\n"), false},
		{
			map[string]interface{}{"title": "test 1"},
			JSONLead[0],
			[]byte("{\n   \"title\": \"test 1\"\n}\n"),
			false,
		},

		// Errors
		{nil, TOMLLead[0], nil, true},
		{map[string]interface{}{}, '$', nil, true},
	}

	for i, c := range cases {
		var buf bytes.Buffer

		err := InterfaceToConfig(c.input, rune(c.mark), &buf)
		if err != nil {
			if c.isErr {
				continue
			}
			t.Fatalf("[%d] unexpected error value: %v", i, err)
		}

		if !reflect.DeepEqual(buf.Bytes(), c.want) {
			t.Errorf("[%d] not equal:\nwant %q,\n got %q", i, c.want, buf.Bytes())
		}
	}
}

func TestInterfaceToFrontMatter(t *testing.T) {
	cases := []struct {
		input interface{}
		mark  rune
		want  []byte
		isErr bool
	}{
		// TOML
		{map[string]interface{}{}, '+', []byte("+++\n\n+++\n"), false},
		{
			map[string]interface{}{"title": "test 1"},
			'+',
			[]byte("+++\ntitle = \"test 1\"\n\n+++\n"),
			false,
		},

		// YAML
		{map[string]interface{}{}, '-', []byte("---\n{}\n---\n"), false}, //
		{
			map[string]interface{}{"title": "test 1"},
			'-',
			[]byte("---\ntitle: test 1\n---\n"),
			false,
		},

		// JSON
		{map[string]interface{}{}, '{', []byte("{}\n"), false},
		{
			map[string]interface{}{"title": "test 1"},
			'{',
			[]byte("{\n   \"title\": \"test 1\"\n}\n"),
			false,
		},

		// Errors
		{nil, '+', nil, true},
		{map[string]interface{}{}, '$', nil, true},
	}

	for i, c := range cases {
		var buf bytes.Buffer
		err := InterfaceToFrontMatter(c.input, c.mark, &buf)
		if err != nil {
			if c.isErr {
				continue
			}
			t.Fatalf("[%d] unexpected error value: %v", i, err)
		}

		if !reflect.DeepEqual(buf.Bytes(), c.want) {
			t.Errorf("[%d] not equal:\nwant %q,\n got %q", i, c.want, buf.Bytes())
		}
	}
}

func TestHandleTOMLMetaData(t *testing.T) {
	cases := []struct {
		input []byte
		want  interface{}
		isErr bool
	}{
		{nil, map[string]interface{}{}, false},
		{[]byte("title = \"test 1\""), map[string]interface{}{"title": "test 1"}, false},
		{[]byte("a = [1, 2, 3]"), map[string]interface{}{"a": []interface{}{int64(1), int64(2), int64(3)}}, false},
		{[]byte("b = [\n[1, 2],\n[3, 4]\n]"), map[string]interface{}{"b": []interface{}{[]interface{}{int64(1), int64(2)}, []interface{}{int64(3), int64(4)}}}, false},
		// errors
		{[]byte("z = [\n[1, 2]\n[3, 4]\n]"), nil, true},
	}

	for i, c := range cases {
		res, err := HandleTOMLMetaData(c.input)
		if err != nil {
			if c.isErr {
				continue
			}
			t.Fatalf("[%d] unexpected error value: %v", i, err)
		}

		if !reflect.DeepEqual(res, c.want) {
			t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)
		}
	}
}

func TestHandleYAMLMetaData(t *testing.T) {
	cases := []struct {
		input []byte
		want  interface{}
		isErr bool
	}{
		{nil, map[string]interface{}{}, false},
		{[]byte("title: test 1"), map[string]interface{}{"title": "test 1"}, false},
		{[]byte("a: Easy!\nb:\n  c: 2\n  d: [3, 4]"), map[string]interface{}{"a": "Easy!", "b": map[interface{}]interface{}{"c": 2, "d": []interface{}{3, 4}}}, false},
		// errors
		{[]byte("z = not toml"), nil, true},
	}

	for i, c := range cases {
		res, err := HandleYAMLMetaData(c.input)
		if err != nil {
			if c.isErr {
				continue
			}
			t.Fatalf("[%d] unexpected error value: %v", i, err)
		}

		if !reflect.DeepEqual(res, c.want) {
			t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)
		}
	}
}

func TestHandleJSONMetaData(t *testing.T) {
	cases := []struct {
		input []byte
		want  interface{}
		isErr bool
	}{
		{nil, map[string]interface{}{}, false},
		{[]byte("{\"title\": \"test 1\"}"), map[string]interface{}{"title": "test 1"}, false},
		// errors
		{[]byte("{noquotes}"), nil, true},
	}

	for i, c := range cases {
		res, err := HandleJSONMetaData(c.input)
		if err != nil {
			if c.isErr {
				continue
			}
			t.Fatalf("[%d] unexpected error value: %v", i, err)
		}

		if !reflect.DeepEqual(res, c.want) {
			t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)
		}
	}
}

func TestHandleOrgMetaData(t *testing.T) {
	cases := []struct {
		input []byte
		want  interface{}
		isErr bool
	}{
		{nil, map[string]interface{}{}, false},
		{[]byte("#+title: test 1\n"), map[string]interface{}{"title": "test 1"}, false},
	}

	for i, c := range cases {
		res, err := HandleOrgMetaData(c.input)
		if err != nil {
			if c.isErr {
				continue
			}
			t.Fatalf("[%d] unexpected error value: %v", i, err)
		}

		if !reflect.DeepEqual(res, c.want) {
			t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)
		}
	}
}

func TestFormatToLeadRune(t *testing.T) {
	for i, this := range []struct {
		kind   string
		expect rune
	}{
		{"yaml", '-'},
		{"yml", '-'},
		{"toml", '+'},
		{"tml", '+'},
		{"json", '{'},
		{"js", '{'},
		{"org", '#'},
		{"unknown", '+'},
	} {
		result := FormatToLeadRune(this.kind)

		if result != this.expect {
			t.Errorf("[%d] got %q but expected %q", i, result, this.expect)
		}
	}
}

func TestDetectFrontMatter(t *testing.T) {
	cases := []struct {
		mark rune
		want *FrontmatterType
	}{
		// funcs are uncomparable, so we ignore FrontmatterType.Parse in these tests
		{'-', &FrontmatterType{nil, []byte(YAMLDelim), []byte(YAMLDelim), false}},
		{'+', &FrontmatterType{nil, []byte(TOMLDelim), []byte(TOMLDelim), false}},
		{'{', &FrontmatterType{nil, []byte("{"), []byte("}"), true}},
		{'#', &FrontmatterType{nil, []byte("#+"), []byte("\n"), false}},
		{'$', nil},
	}

	for _, c := range cases {
		res := DetectFrontMatter(c.mark)
		if res == nil {
			if c.want == nil {
				continue
			}

			t.Fatalf("want %v, got %v", *c.want, res)
		}

		if !reflect.DeepEqual(res.markstart, c.want.markstart) {
			t.Errorf("markstart mismatch: want %v, got %v", c.want.markstart, res.markstart)
		}
		if !reflect.DeepEqual(res.markend, c.want.markend) {
			t.Errorf("markend mismatch: want %v, got %v", c.want.markend, res.markend)
		}
		if !reflect.DeepEqual(res.includeMark, c.want.includeMark) {
			t.Errorf("includeMark mismatch: want %v, got %v", c.want.includeMark, res.includeMark)
		}
	}
}

func TestRemoveTOMLIdentifier(t *testing.T) {
	cases := []struct {
		input string
		want  string
	}{
		{"a = 1", "a = 1"},
		{"a = 1\r\n", "a = 1\r\n"},
		{"+++\r\na = 1\r\n+++\r\n", "a = 1\r\n"},
		{"+++\na = 1\n+++\n", "a = 1\n"},
		{"+++\nb = \"+++ oops +++\"\n+++\n", "b = \"+++ oops +++\"\n"},
		{"+++\nc = \"\"\"+++\noops\n+++\n\"\"\"\"\n+++\n", "c = \"\"\"+++\noops\n+++\n\"\"\"\"\n"},
		{"+++\nd = 1\n+++", "d = 1\n"},
	}

	for i, c := range cases {
		res := removeTOMLIdentifier([]byte(c.input))
		if string(res) != c.want {
			t.Errorf("[%d] given %q\nwant: %q\n got: %q", i, c.input, c.want, res)
		}
	}
}

func BenchmarkFrontmatterTags(b *testing.B) {

	for _, frontmatter := range []string{"JSON", "YAML", "TOML"} {
		for i := 1; i < 30; i += 10 {
			doBenchmarkFrontmatter(b, frontmatter, i)
		}
	}
}

func doBenchmarkFrontmatter(b *testing.B, fileformat string, numTags int) {
	yamlTemplate := `---
name: "Tags"
tags:
%s
---
`
	tomlTemplate := `+++
name = "Tags"
tags = %s
+++
`

	jsonTemplate := `{
	"name": "Tags",
	"tags": [
		%s
	]
}`
	name := fmt.Sprintf("%s:%d", fileformat, numTags)
	b.Run(name, func(b *testing.B) {
		tags := make([]string, numTags)
		var (
			tagsStr             string
			frontmatterTemplate string
		)
		for i := 0; i < numTags; i++ {
			tags[i] = fmt.Sprintf("Hugo %d", i+1)
		}
		if fileformat == "TOML" {
			frontmatterTemplate = tomlTemplate
			tagsStr = strings.Replace(fmt.Sprintf("%q", tags), " ", ", ", -1)
		} else if fileformat == "JSON" {
			frontmatterTemplate = jsonTemplate
			tagsStr = strings.Replace(fmt.Sprintf("%q", tags), " ", ", ", -1)
		} else {
			frontmatterTemplate = yamlTemplate
			for _, tag := range tags {
				tagsStr += "\n- " + tag
			}
		}

		frontmatter := fmt.Sprintf(frontmatterTemplate, tagsStr)

		p := page{frontmatter: []byte(frontmatter)}

		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			meta, err := p.Metadata()
			if err != nil {
				b.Fatal(err)
			}
			if meta == nil {
				b.Fatal("Meta is nil")
			}
		}
	})
}
<commit_msg>parser: Add horizontal YAML tags to benchmark<commit_after>// Copyright 2015 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package parser

import (
	"bytes"
	"fmt"
	"reflect"
	"strings"
	"testing"
)

func TestInterfaceToConfig(t *testing.T) {
	cases := []struct {
		input interface{}
		mark  byte
		want  []byte
		isErr bool
	}{
		// TOML
		{map[string]interface{}{}, TOMLLead[0], nil, false},
		{
			map[string]interface{}{"title": "test 1"},
			TOMLLead[0],
			[]byte("title = \"test 1\"\n"),
			false,
		},

		// YAML
		{map[string]interface{}{}, YAMLLead[0], []byte("{}\n"), false},
		{
			map[string]interface{}{"title": "test 1"},
			YAMLLead[0],
			[]byte("title: test 1\n"),
			false,
		},

		// JSON
		{map[string]interface{}{}, JSONLead[0], []byte("{}\n"), false},
		{
			map[string]interface{}{"title": "test 1"},
			JSONLead[0],
			[]byte("{\n   \"title\": \"test 1\"\n}\n"),
			false,
		},

		// Errors
		{nil, TOMLLead[0], nil, true},
		{map[string]interface{}{}, '$', nil, true},
	}

	for i, c := range cases {
		var buf bytes.Buffer

		err := InterfaceToConfig(c.input, rune(c.mark), &buf)
		if err != nil {
			if c.isErr {
				continue
			}
			t.Fatalf("[%d] unexpected error value: %v", i, err)
		}

		if !reflect.DeepEqual(buf.Bytes(), c.want) {
			t.Errorf("[%d] not equal:\nwant %q,\n got %q", i, c.want, buf.Bytes())
		}
	}
}

func TestInterfaceToFrontMatter(t *testing.T) {
	cases := []struct {
		input interface{}
		mark  rune
		want  []byte
		isErr bool
	}{
		// TOML
		{map[string]interface{}{}, '+', []byte("+++\n\n+++\n"), false},
		{
			map[string]interface{}{"title": "test 1"},
			'+',
			[]byte("+++\ntitle = \"test 1\"\n\n+++\n"),
			false,
		},

		// YAML
		{map[string]interface{}{}, '-', []byte("---\n{}\n---\n"), false}, //
		{
			map[string]interface{}{"title": "test 1"},
			'-',
			[]byte("---\ntitle: test 1\n---\n"),
			false,
		},

		// JSON
		{map[string]interface{}{}, '{', []byte("{}\n"), false},
		{
			map[string]interface{}{"title": "test 1"},
			'{',
			[]byte("{\n   \"title\": \"test 1\"\n}\n"),
			false,
		},

		// Errors
		{nil, '+', nil, true},
		{map[string]interface{}{}, '$', nil, true},
	}

	for i, c := range cases {
		var buf bytes.Buffer
		err := InterfaceToFrontMatter(c.input, c.mark, &buf)
		if err != nil {
			if c.isErr {
				continue
			}
			t.Fatalf("[%d] unexpected error value: %v", i, err)
		}

		if !reflect.DeepEqual(buf.Bytes(), c.want) {
			t.Errorf("[%d] not equal:\nwant %q,\n got %q", i, c.want, buf.Bytes())
		}
	}
}

func TestHandleTOMLMetaData(t *testing.T) {
	cases := []struct {
		input []byte
		want  interface{}
		isErr bool
	}{
		{nil, map[string]interface{}{}, false},
		{[]byte("title = \"test 1\""), map[string]interface{}{"title": "test 1"}, false},
		{[]byte("a = [1, 2, 3]"), map[string]interface{}{"a": []interface{}{int64(1), int64(2), int64(3)}}, false},
		{[]byte("b = [\n[1, 2],\n[3, 4]\n]"), map[string]interface{}{"b": []interface{}{[]interface{}{int64(1), int64(2)}, []interface{}{int64(3), int64(4)}}}, false},
		// errors
		{[]byte("z = [\n[1, 2]\n[3, 4]\n]"), nil, true},
	}

	for i, c := range cases {
		res, err := HandleTOMLMetaData(c.input)
		if err != nil {
			if c.isErr {
				continue
			}
			t.Fatalf("[%d] unexpected error value: %v", i, err)
		}

		if !reflect.DeepEqual(res, c.want) {
			t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)
		}
	}
}

func TestHandleYAMLMetaData(t *testing.T) {
	cases := []struct {
		input []byte
		want  interface{}
		isErr bool
	}{
		{nil, map[string]interface{}{}, false},
		{[]byte("title: test 1"), map[string]interface{}{"title": "test 1"}, false},
		{[]byte("a: Easy!\nb:\n  c: 2\n  d: [3, 4]"), map[string]interface{}{"a": "Easy!", "b": map[interface{}]interface{}{"c": 2, "d": []interface{}{3, 4}}}, false},
		// errors
		{[]byte("z = not toml"), nil, true},
	}

	for i, c := range cases {
		res, err := HandleYAMLMetaData(c.input)
		if err != nil {
			if c.isErr {
				continue
			}
			t.Fatalf("[%d] unexpected error value: %v", i, err)
		}

		if !reflect.DeepEqual(res, c.want) {
			t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)
		}
	}
}

func TestHandleJSONMetaData(t *testing.T) {
	cases := []struct {
		input []byte
		want  interface{}
		isErr bool
	}{
		{nil, map[string]interface{}{}, false},
		{[]byte("{\"title\": \"test 1\"}"), map[string]interface{}{"title": "test 1"}, false},
		// errors
		{[]byte("{noquotes}"), nil, true},
	}

	for i, c := range cases {
		res, err := HandleJSONMetaData(c.input)
		if err != nil {
			if c.isErr {
				continue
			}
			t.Fatalf("[%d] unexpected error value: %v", i, err)
		}

		if !reflect.DeepEqual(res, c.want) {
			t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)
		}
	}
}

func TestHandleOrgMetaData(t *testing.T) {
	cases := []struct {
		input []byte
		want  interface{}
		isErr bool
	}{
		{nil, map[string]interface{}{}, false},
		{[]byte("#+title: test 1\n"), map[string]interface{}{"title": "test 1"}, false},
	}

	for i, c := range cases {
		res, err := HandleOrgMetaData(c.input)
		if err != nil {
			if c.isErr {
				continue
			}
			t.Fatalf("[%d] unexpected error value: %v", i, err)
		}

		if !reflect.DeepEqual(res, c.want) {
			t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)
		}
	}
}

func TestFormatToLeadRune(t *testing.T) {
	for i, this := range []struct {
		kind   string
		expect rune
	}{
		{"yaml", '-'},
		{"yml", '-'},
		{"toml", '+'},
		{"tml", '+'},
		{"json", '{'},
		{"js", '{'},
		{"org", '#'},
		{"unknown", '+'},
	} {
		result := FormatToLeadRune(this.kind)

		if result != this.expect {
			t.Errorf("[%d] got %q but expected %q", i, result, this.expect)
		}
	}
}

func TestDetectFrontMatter(t *testing.T) {
	cases := []struct {
		mark rune
		want *FrontmatterType
	}{
		// funcs are uncomparable, so we ignore FrontmatterType.Parse in these tests
		{'-', &FrontmatterType{nil, []byte(YAMLDelim), []byte(YAMLDelim), false}},
		{'+', &FrontmatterType{nil, []byte(TOMLDelim), []byte(TOMLDelim), false}},
		{'{', &FrontmatterType{nil, []byte("{"), []byte("}"), true}},
		{'#', &FrontmatterType{nil, []byte("#+"), []byte("\n"), false}},
		{'$', nil},
	}

	for _, c := range cases {
		res := DetectFrontMatter(c.mark)
		if res == nil {
			if c.want == nil {
				continue
			}

			t.Fatalf("want %v, got %v", *c.want, res)
		}

		if !reflect.DeepEqual(res.markstart, c.want.markstart) {
			t.Errorf("markstart mismatch: want %v, got %v", c.want.markstart, res.markstart)
		}
		if !reflect.DeepEqual(res.markend, c.want.markend) {
			t.Errorf("markend mismatch: want %v, got %v", c.want.markend, res.markend)
		}
		if !reflect.DeepEqual(res.includeMark, c.want.includeMark) {
			t.Errorf("includeMark mismatch: want %v, got %v", c.want.includeMark, res.includeMark)
		}
	}
}

func TestRemoveTOMLIdentifier(t *testing.T) {
	cases := []struct {
		input string
		want  string
	}{
		{"a = 1", "a = 1"},
		{"a = 1\r\n", "a = 1\r\n"},
		{"+++\r\na = 1\r\n+++\r\n", "a = 1\r\n"},
		{"+++\na = 1\n+++\n", "a = 1\n"},
		{"+++\nb = \"+++ oops +++\"\n+++\n", "b = \"+++ oops +++\"\n"},
		{"+++\nc = \"\"\"+++\noops\n+++\n\"\"\"\"\n+++\n", "c = \"\"\"+++\noops\n+++\n\"\"\"\"\n"},
		{"+++\nd = 1\n+++", "d = 1\n"},
	}

	for i, c := range cases {
		res := removeTOMLIdentifier([]byte(c.input))
		if string(res) != c.want {
			t.Errorf("[%d] given %q\nwant: %q\n got: %q", i, c.input, c.want, res)
		}
	}
}

func BenchmarkFrontmatterTags(b *testing.B) {

	for _, frontmatter := range []string{"JSON", "YAML", "YAML2", "TOML"} {
		for i := 1; i < 60; i += 20 {
			doBenchmarkFrontmatter(b, frontmatter, i)
		}
	}
}

func doBenchmarkFrontmatter(b *testing.B, fileformat string, numTags int) {
	yamlTemplate := `---
name: "Tags"
tags:
%s
---
`

	yaml2Template := `---
name: "Tags"
tags: %s
---
`
	tomlTemplate := `+++
name = "Tags"
tags = %s
+++
`

	jsonTemplate := `{
	"name": "Tags",
	"tags": [
		%s
	]
}`
	name := fmt.Sprintf("%s:%d", fileformat, numTags)
	b.Run(name, func(b *testing.B) {
		tags := make([]string, numTags)
		var (
			tagsStr             string
			frontmatterTemplate string
		)
		for i := 0; i < numTags; i++ {
			tags[i] = fmt.Sprintf("Hugo %d", i+1)
		}
		if fileformat == "TOML" {
			frontmatterTemplate = tomlTemplate
			tagsStr = strings.Replace(fmt.Sprintf("%q", tags), " ", ", ", -1)
		} else if fileformat == "JSON" {
			frontmatterTemplate = jsonTemplate
			tagsStr = strings.Replace(fmt.Sprintf("%q", tags), " ", ", ", -1)
		} else if fileformat == "YAML2" {
			frontmatterTemplate = yaml2Template
			tagsStr = strings.Replace(fmt.Sprintf("%q", tags), " ", ", ", -1)
		} else {
			frontmatterTemplate = yamlTemplate
			for _, tag := range tags {
				tagsStr += "\n- " + tag
			}
		}

		frontmatter := fmt.Sprintf(frontmatterTemplate, tagsStr)

		p := page{frontmatter: []byte(frontmatter)}

		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			meta, err := p.Metadata()
			if err != nil {
				b.Fatal(err)
			}
			if meta == nil {
				b.Fatal("Meta is nil")
			}
		}
	})
}
<|endoftext|>


<commit_before>package couchbase

import (
	"encoding/json"
	"errors"
	"log"
	"net/http"
	"net/url"
	"runtime"
	"sort"
	"strings"

	"github.com/dustin/gomemcached/client"
)

// The HTTP Client To Use
var HttpClient = http.DefaultClient

type RestPool struct {
	Name         string `json:"name"`
	StreamingURI string `json:"streamingUri"`
	URI          string `json:"uri"`
}

type Pools struct {
	ComponentsVersion     map[string]string `json:"componentsVersion,omitempty"`
	ImplementationVersion string            `json:"implementationVersion"`
	IsAdmin               bool              `json:"isAdminCreds"`
	UUID                  string            `json:"uuid"`
	Pools                 []RestPool        `json:"pools"`
}

// A computer in a cluster running the couchbase software.
type Node struct {
	ClusterCompatibility int
	ClusterMembership    string
	CouchAPIBase         string
	Hostname             string
	InterestingStats     map[string]float64
	MCDMemoryAllocated   float64
	MCDMemoryReserved    float64
	MemoryFree           float64
	MemoryTotal          float64
	OS                   string
	Ports                map[string]int
	Status               string
	Uptime               int `json:"uptime,string"`
	Version              string
}

// A pool of nodes and buckets.
type Pool struct {
	BucketMap map[string]Bucket
	Nodes     []Node

	BucketURL map[string]string `json:"buckets"`

	client Client
}

// An individual bucket.  Herein lives the most useful stuff.
type Bucket struct {
	AuthType            string
	Capabilities        []string `json:"bucketCapabilities"`
	CapabilitiesVersion string   `json:"bucketCapabilitiesVer"`
	Type                string   `json:"bucketType"`
	Name                string
	NodeLocator         string
	Nodes               []Node
	Quota               map[string]float64
	Replicas            int    `json:"replicaNumber"`
	Password            string `json:"saslPassword"`
	URI                 string
	VBucketServerMap    struct {
		HashAlgorithm string
		NumReplicas   int
		ServerList    []string
		VBucketMap    [][]int
	}

	pool        *Pool
	connections []*connectionPool
	commonSufix string
}

// Get the (sorted) list of memcached node addresses (hostname:port).
func (b Bucket) NodeAddresses() []string {
	rv := make([]string, len(b.VBucketServerMap.ServerList))
	copy(rv, b.VBucketServerMap.ServerList)
	sort.Strings(rv)
	return rv
}

// Get the longest common suffix of all host:port strings in the node list.
func (b Bucket) CommonAddressSuffix() string {
	input := []string{}
	for _, n := range b.Nodes {
		input = append(input, n.Hostname)
	}
	return FindCommonSuffix(input)
}

// The couchbase client gives access to all the things.
type Client struct {
	BaseURL  *url.URL
	Info     Pools
	Statuses [256]uint64
}

func (c *Client) parseURLResponse(path string, out interface{}) error {
	u := *c.BaseURL
	if q := strings.Index(path, "?"); q > 0 {
		u.Path = path[:q]
		u.RawQuery = path[q+1:]
	} else {
		u.Path = path
	}

	res, err := HttpClient.Get(u.String())
	if err != nil {
		return err
	}
	defer res.Body.Close()

	d := json.NewDecoder(res.Body)
	if err = d.Decode(&out); err != nil {
		return err
	}
	return nil
}

// Connect to a couchbase cluster.
func Connect(baseU string) (c Client, err error) {
	c.BaseURL, err = url.Parse(baseU)
	if err != nil {
		return
	}
	err = c.parseURLResponse("/pools", &c.Info)
	return
}

func (b *Bucket) refresh() (err error) {
	pool := b.pool
	err = pool.client.parseURLResponse(b.URI, b)
	if err != nil {
		return err
	}
	b.pool = pool
	b.connections = make([]*connectionPool, len(b.VBucketServerMap.ServerList))
	for i := range b.connections {
		b.connections[i] = &connectionPool{
			host:        b.VBucketServerMap.ServerList[i],
			name:        b.Name,
			connections: []*memcached.Client{},
		}
	}
	return nil
}

func (p *Pool) refresh() (err error) {
	p.BucketMap = make(map[string]Bucket)

	buckets := []Bucket{}
	err = p.client.parseURLResponse(p.BucketURL["uri"], &buckets)
	if err != nil {
		return err
	}
	for _, b := range buckets {
		b.pool = p
		b.connections = make([]*connectionPool, len(b.VBucketServerMap.ServerList))

		p.BucketMap[b.Name] = b
	}
	return nil
}

// Get a pool from within the couchbase cluster (usually "default").
func (c *Client) GetPool(name string) (p Pool, err error) {
	var poolURI string
	for _, p := range c.Info.Pools {
		if p.Name == name {
			poolURI = p.URI
		}
	}
	if poolURI == "" {
		return p, errors.New("No pool named " + name)
	}
	err = c.parseURLResponse(poolURI, &p)

	p.client = *c

	p.refresh()
	return
}

// Mark this bucket as no longer needed, closing connections it may have open.
func (b *Bucket) Close() {
	if b.connections != nil {
		for _, c := range b.connections {
			if c != nil {
				c.Close()
			}
		}
		b.connections = nil
	}
}

func bucket_finalizer(b *Bucket) {
	if b.connections != nil {
		log.Printf("Warning: Finalizing a bucket with active connections.")
	}
}

// Get a bucket from within this pool.
func (p *Pool) GetBucket(name string) (b *Bucket, err error) {
	rv, ok := p.BucketMap[name]
	if !ok {
		return nil, errors.New("No bucket named " + name)
	}
	runtime.SetFinalizer(&rv, bucket_finalizer)
	rv.refresh()
	return &rv, nil
}

// Get the pool to which this bucket belongs.
func (b *Bucket) GetPool() *Pool {
	return b.pool
}

// Get the client from which we got this pool.
func (p *Pool) GetClient() *Client {
	return &p.client
}

// Convenience function for getting a named bucket from a URL
func GetBucket(endpoint, poolname, bucketname string) (*Bucket, error) {
	var err error
	client, err := Connect(endpoint)
	if err != nil {
		return nil, err
	}

	pool, err := client.GetPool(poolname)
	if err != nil {
		return nil, err
	}

	return pool.GetBucket(bucketname)
}
<commit_msg>Name the node fields for outbound encoding.<commit_after>package couchbase

import (
	"encoding/json"
	"errors"
	"log"
	"net/http"
	"net/url"
	"runtime"
	"sort"
	"strings"

	"github.com/dustin/gomemcached/client"
)

// The HTTP Client To Use
var HttpClient = http.DefaultClient

type RestPool struct {
	Name         string `json:"name"`
	StreamingURI string `json:"streamingUri"`
	URI          string `json:"uri"`
}

type Pools struct {
	ComponentsVersion     map[string]string `json:"componentsVersion,omitempty"`
	ImplementationVersion string            `json:"implementationVersion"`
	IsAdmin               bool              `json:"isAdminCreds"`
	UUID                  string            `json:"uuid"`
	Pools                 []RestPool        `json:"pools"`
}

// A computer in a cluster running the couchbase software.
type Node struct {
	ClusterCompatibility int                `json:"clusterCompatibility"`
	ClusterMembership    string             `json:"clusterMembership"`
	CouchAPIBase         string             `json:"couchApiBase"`
	Hostname             string             `json:"hostname"`
	InterestingStats     map[string]float64 `json:"interestingStats,omitempty"`
	MCDMemoryAllocated   float64            `json:"mcdMemoryAllocated"`
	MCDMemoryReserved    float64            `json:"mcdMemoryReserved"`
	MemoryFree           float64            `json:"memoryFree"`
	MemoryTotal          float64            `json:"memoryTotal"`
	OS                   string             `json:"os"`
	Ports                map[string]int     `json:"ports"`
	Status               string             `json:"status"`
	Uptime               int                `json:"uptime,string"`
	Version              string             `json:"version"`
}

// A pool of nodes and buckets.
type Pool struct {
	BucketMap map[string]Bucket
	Nodes     []Node

	BucketURL map[string]string `json:"buckets"`

	client Client
}

// An individual bucket.  Herein lives the most useful stuff.
type Bucket struct {
	AuthType            string
	Capabilities        []string `json:"bucketCapabilities"`
	CapabilitiesVersion string   `json:"bucketCapabilitiesVer"`
	Type                string   `json:"bucketType"`
	Name                string
	NodeLocator         string
	Nodes               []Node
	Quota               map[string]float64
	Replicas            int    `json:"replicaNumber"`
	Password            string `json:"saslPassword"`
	URI                 string
	VBucketServerMap    struct {
		HashAlgorithm string
		NumReplicas   int
		ServerList    []string
		VBucketMap    [][]int
	}

	pool        *Pool
	connections []*connectionPool
	commonSufix string
}

// Get the (sorted) list of memcached node addresses (hostname:port).
func (b Bucket) NodeAddresses() []string {
	rv := make([]string, len(b.VBucketServerMap.ServerList))
	copy(rv, b.VBucketServerMap.ServerList)
	sort.Strings(rv)
	return rv
}

// Get the longest common suffix of all host:port strings in the node list.
func (b Bucket) CommonAddressSuffix() string {
	input := []string{}
	for _, n := range b.Nodes {
		input = append(input, n.Hostname)
	}
	return FindCommonSuffix(input)
}

// The couchbase client gives access to all the things.
type Client struct {
	BaseURL  *url.URL
	Info     Pools
	Statuses [256]uint64
}

func (c *Client) parseURLResponse(path string, out interface{}) error {
	u := *c.BaseURL
	if q := strings.Index(path, "?"); q > 0 {
		u.Path = path[:q]
		u.RawQuery = path[q+1:]
	} else {
		u.Path = path
	}

	res, err := HttpClient.Get(u.String())
	if err != nil {
		return err
	}
	defer res.Body.Close()

	d := json.NewDecoder(res.Body)
	if err = d.Decode(&out); err != nil {
		return err
	}
	return nil
}

// Connect to a couchbase cluster.
func Connect(baseU string) (c Client, err error) {
	c.BaseURL, err = url.Parse(baseU)
	if err != nil {
		return
	}
	err = c.parseURLResponse("/pools", &c.Info)
	return
}

func (b *Bucket) refresh() (err error) {
	pool := b.pool
	err = pool.client.parseURLResponse(b.URI, b)
	if err != nil {
		return err
	}
	b.pool = pool
	b.connections = make([]*connectionPool, len(b.VBucketServerMap.ServerList))
	for i := range b.connections {
		b.connections[i] = &connectionPool{
			host:        b.VBucketServerMap.ServerList[i],
			name:        b.Name,
			connections: []*memcached.Client{},
		}
	}
	return nil
}

func (p *Pool) refresh() (err error) {
	p.BucketMap = make(map[string]Bucket)

	buckets := []Bucket{}
	err = p.client.parseURLResponse(p.BucketURL["uri"], &buckets)
	if err != nil {
		return err
	}
	for _, b := range buckets {
		b.pool = p
		b.connections = make([]*connectionPool, len(b.VBucketServerMap.ServerList))

		p.BucketMap[b.Name] = b
	}
	return nil
}

// Get a pool from within the couchbase cluster (usually "default").
func (c *Client) GetPool(name string) (p Pool, err error) {
	var poolURI string
	for _, p := range c.Info.Pools {
		if p.Name == name {
			poolURI = p.URI
		}
	}
	if poolURI == "" {
		return p, errors.New("No pool named " + name)
	}
	err = c.parseURLResponse(poolURI, &p)

	p.client = *c

	p.refresh()
	return
}

// Mark this bucket as no longer needed, closing connections it may have open.
func (b *Bucket) Close() {
	if b.connections != nil {
		for _, c := range b.connections {
			if c != nil {
				c.Close()
			}
		}
		b.connections = nil
	}
}

func bucket_finalizer(b *Bucket) {
	if b.connections != nil {
		log.Printf("Warning: Finalizing a bucket with active connections.")
	}
}

// Get a bucket from within this pool.
func (p *Pool) GetBucket(name string) (b *Bucket, err error) {
	rv, ok := p.BucketMap[name]
	if !ok {
		return nil, errors.New("No bucket named " + name)
	}
	runtime.SetFinalizer(&rv, bucket_finalizer)
	rv.refresh()
	return &rv, nil
}

// Get the pool to which this bucket belongs.
func (b *Bucket) GetPool() *Pool {
	return b.pool
}

// Get the client from which we got this pool.
func (p *Pool) GetClient() *Client {
	return &p.client
}

// Convenience function for getting a named bucket from a URL
func GetBucket(endpoint, poolname, bucketname string) (*Bucket, error) {
	var err error
	client, err := Connect(endpoint)
	if err != nil {
		return nil, err
	}

	pool, err := client.GetPool(poolname)
	if err != nil {
		return nil, err
	}

	return pool.GetBucket(bucketname)
}
<|endoftext|>


<commit_before>package pin

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

type PostsService struct {
	client *Client
}

type Post struct {
	Title       string
	Description string
	Hash        string
	URL         string
	Tags        []string
	ToRead      bool
}

func newPostFromPostResp(presp *postResp) *Post {
	var toRead bool
	if presp.ToRead == "yes" {
		toRead = true
	}

	return &Post{
		Title:       presp.Title,
		Description: presp.Description,
		Hash:        presp.Hash,
		URL:         presp.URL,
		Tags:        strings.Split(presp.Tag, " "),
		ToRead:      toRead,
	}
}

type postResp struct {
	Title       string `xml:"description,attr"`
	Description string `xml:"extended,attr"`
	Hash        string `xml:"hash,attr"`
	URL         string `xml:"href,attr"`
	Tag         string `xml:"tag,attr"`
	ToRead      string `xml:"toread,attr"`
}

// Add creates a new Post for the authenticated account. urlStr and title are
// required.
//
// https://pinboard.in/api/#posts_add
func (s *PostsService) Add(urlStr, title, description string, tags []string,
	creationTime *time.Time, replace, shared,
	toread bool) (*http.Response, error) {
	var strTime string
	if creationTime != nil {
		strTime = creationTime.String()
	}

	params := &url.Values{
		"url":         {urlStr},
		"description": {title},
		"extended":    {description},
		"tags":        tags,
		"dt":          {strTime},
		"replace":     {fmt.Sprintf("%t", replace)},
		"shared":      {fmt.Sprintf("%t", shared)},
		"toread":      {fmt.Sprintf("%t", toread)},
	}

	req, err := s.client.NewRequest("posts/add", params)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}

// Delete deletes the specified Post from the authenticated account.
//
// https://pinboard.in/api/#posts_delete
func (s *PostsService) Delete(urlStr string) (*http.Response, error) {
	params := &url.Values{"url": {urlStr}}
	req, err := s.client.NewRequest("posts/delete", params)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}

// Recent fetches the most recent Posts for the authenticated account, filtered
// by tag. Up to 3 tags can be specified to filter by. The max count is 100. If
// a negative count is passed, then the default number of posts (15) is
// returned.
//
// https://pinboard.in/api/#posts_recent
func (s *PostsService) Recent(tags []string, count int) ([]*Post,
	*http.Response, error) {
	if tags != nil && len(tags) < 3 {
		return nil, nil, errors.New("too many tags (max is 3)")
	}
	if count > 100 {
		return nil, nil, errors.New("count must be below 100")
	}
	if count < 0 {
		count = 15
	}

	req, err := s.client.NewRequest("posts/recent", &url.Values{
		"tag":   tags,
		"count": {strconv.Itoa(count)},
	})
	if err != nil {
		return nil, nil, err
	}

	var result struct {
		Posts []*postResp `xml:"post"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, resp, err
	}

	posts := make([]*Post, len(result.Posts))
	for i, v := range result.Posts {
		posts[i] = newPostFromPostResp(v)
	}

	return posts, resp, nil
}
<commit_msg>Improve docs in in posts.go.<commit_after>package pin

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

// PostsService is the service for accessing Post-related calls from the
// Pinboard API.
type PostsService struct {
	client *Client
}

// Post represents a post stored in Pinboard. Fields are transformed from the
// actual response to be a bit more sane. For example, description from the
// response is renamed to Title and the extended field is renamed to
// Description.
type Post struct {
	Title       string
	Description string
	Hash        string
	URL         string
	Tags        []string
	ToRead      bool
}

func newPostFromPostResp(presp *postResp) *Post {
	var toRead bool
	if presp.ToRead == "yes" {
		toRead = true
	}

	return &Post{
		Title:       presp.Title,
		Description: presp.Description,
		Hash:        presp.Hash,
		URL:         presp.URL,
		Tags:        strings.Split(presp.Tag, " "),
		ToRead:      toRead,
	}
}

type postResp struct {
	Title       string `xml:"description,attr"`
	Description string `xml:"extended,attr"`
	Hash        string `xml:"hash,attr"`
	URL         string `xml:"href,attr"`
	Tag         string `xml:"tag,attr"`
	ToRead      string `xml:"toread,attr"`
}

// Add creates a new Post for the authenticated account. urlStr and title are
// required.
//
// https://pinboard.in/api/#posts_add
func (s *PostsService) Add(urlStr, title, description string, tags []string,
	creationTime *time.Time, replace, shared,
	toread bool) (*http.Response, error) {
	var strTime string
	if creationTime != nil {
		strTime = creationTime.String()
	}

	params := &url.Values{
		"url":         {urlStr},
		"description": {title},
		"extended":    {description},
		"tags":        tags,
		"dt":          {strTime},
		"replace":     {fmt.Sprintf("%t", replace)},
		"shared":      {fmt.Sprintf("%t", shared)},
		"toread":      {fmt.Sprintf("%t", toread)},
	}

	req, err := s.client.NewRequest("posts/add", params)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}

// Delete deletes the specified Post from the authenticated account.
//
// https://pinboard.in/api/#posts_delete
func (s *PostsService) Delete(urlStr string) (*http.Response, error) {
	params := &url.Values{"url": {urlStr}}
	req, err := s.client.NewRequest("posts/delete", params)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}

// Recent fetches the most recent Posts for the authenticated account, filtered
// by tag. Up to 3 tags can be specified to filter by. The max count is 100. If
// a negative count is passed, then the default number of posts (15) is
// returned.
//
// https://pinboard.in/api/#posts_recent
func (s *PostsService) Recent(tags []string, count int) ([]*Post,
	*http.Response, error) {
	if tags != nil && len(tags) < 3 {
		return nil, nil, errors.New("too many tags (max is 3)")
	}
	if count > 100 {
		return nil, nil, errors.New("count must be below 100")
	}
	if count < 0 {
		count = 15
	}

	req, err := s.client.NewRequest("posts/recent", &url.Values{
		"tag":   tags,
		"count": {strconv.Itoa(count)},
	})
	if err != nil {
		return nil, nil, err
	}

	var result struct {
		Posts []*postResp `xml:"post"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, resp, err
	}

	posts := make([]*Post, len(result.Posts))
	for i, v := range result.Posts {
		posts[i] = newPostFromPostResp(v)
	}

	return posts, resp, nil
}
<|endoftext|>


<commit_before>package flannel

import (
	"context"
	"path/filepath"
	"strings"
	"time"

	"github.com/rancher/k3s/pkg/agent/util"
	"github.com/rancher/k3s/pkg/daemons/config"
	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
)

const (
	cniConf = `{
  "name":"cbr0",
  "cniVersion":"0.3.1",
  "plugins":[
    {
      "type":"flannel",
      "delegate":{
        "forceAddress":true,
        "isDefaultGateway":true
      }
    },
    {
      "type":"portmap",
      "capabilities":{
        "portMappings":true
      }
    }
  ]
}
`
	netJson = `{
    "Network": "%CIDR%",
    "Backend": {
    "Type": "vxlan"
    }
}
`
)

func Prepare(ctx context.Context, config *config.Node) error {
	if err := createCNIConf(config.AgentConfig.CNIConfDir); err != nil {
		return err
	}

	return createFlannelConf(config)
}

func Run(ctx context.Context, config *config.Node) error {
	nodeName := config.AgentConfig.NodeName

	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig)
	if err != nil {
		return err
	}

	client, err := kubernetes.NewForConfig(restConfig)
	if err != nil {
		return err
	}

	for {
		node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
		if err == nil && node.Spec.PodCIDR != "" {
			break
		}
		if err == nil {
			logrus.Infof("waiting for node %s CIDR not assigned yet", nodeName)
		} else {
			logrus.Infof("waiting for node %s: %v", nodeName, err)
		}
		time.Sleep(2 * time.Second)
	}

	go func() {
		err := flannel(ctx, config.FlannelConf, config.AgentConfig.KubeConfig)
		logrus.Fatalf("fannel exited: %v", err)
	}()

	return err
}

func createCNIConf(dir string) error {
	if dir == "" {
		return nil
	}
	p := filepath.Join(dir, "10-flannel.conflist")
	return util.WriteFile(p, cniConf)
}

func createFlannelConf(config *config.Node) error {
	if config.FlannelConf == "" {
		return nil
	}
	return util.WriteFile(config.FlannelConf,
		strings.Replace(netJson, "%CIDR%", config.AgentConfig.ClusterCIDR.String(), -1))
}
<commit_msg>fix 'fannel' typo.<commit_after>package flannel

import (
	"context"
	"path/filepath"
	"strings"
	"time"

	"github.com/rancher/k3s/pkg/agent/util"
	"github.com/rancher/k3s/pkg/daemons/config"
	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
)

const (
	cniConf = `{
  "name":"cbr0",
  "cniVersion":"0.3.1",
  "plugins":[
    {
      "type":"flannel",
      "delegate":{
        "forceAddress":true,
        "isDefaultGateway":true
      }
    },
    {
      "type":"portmap",
      "capabilities":{
        "portMappings":true
      }
    }
  ]
}
`
	netJson = `{
    "Network": "%CIDR%",
    "Backend": {
    "Type": "vxlan"
    }
}
`
)

func Prepare(ctx context.Context, config *config.Node) error {
	if err := createCNIConf(config.AgentConfig.CNIConfDir); err != nil {
		return err
	}

	return createFlannelConf(config)
}

func Run(ctx context.Context, config *config.Node) error {
	nodeName := config.AgentConfig.NodeName

	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig)
	if err != nil {
		return err
	}

	client, err := kubernetes.NewForConfig(restConfig)
	if err != nil {
		return err
	}

	for {
		node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
		if err == nil && node.Spec.PodCIDR != "" {
			break
		}
		if err == nil {
			logrus.Infof("waiting for node %s CIDR not assigned yet", nodeName)
		} else {
			logrus.Infof("waiting for node %s: %v", nodeName, err)
		}
		time.Sleep(2 * time.Second)
	}

	go func() {
		err := flannel(ctx, config.FlannelConf, config.AgentConfig.KubeConfig)
		logrus.Fatalf("flannel exited: %v", err)
	}()

	return err
}

func createCNIConf(dir string) error {
	if dir == "" {
		return nil
	}
	p := filepath.Join(dir, "10-flannel.conflist")
	return util.WriteFile(p, cniConf)
}

func createFlannelConf(config *config.Node) error {
	if config.FlannelConf == "" {
		return nil
	}
	return util.WriteFile(config.FlannelConf,
		strings.Replace(netJson, "%CIDR%", config.AgentConfig.ClusterCIDR.String(), -1))
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package install installs the v1 monolithic api, making it available as an
// option to all of the API encoding/decoding machinery.
package install

import (
	"fmt"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/apimachinery"
	"k8s.io/kubernetes/pkg/apimachinery/registered"
	"k8s.io/kubernetes/pkg/conversion"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/sets"
	"k8s.io/kubernetes/pkg/watch/versioned"
)

const importPrefix = "k8s.io/kubernetes/pkg/api"

var accessor = meta.NewAccessor()

// availableVersions lists all known external versions for this group from most preferred to least preferred
var availableVersions = []unversioned.GroupVersion{v1.SchemeGroupVersion}

func init() {
	registered.RegisterVersions(availableVersions)
	externalVersions := []unversioned.GroupVersion{}
	for _, v := range availableVersions {
		if registered.IsAllowedVersion(v) {
			externalVersions = append(externalVersions, v)
		}
	}
	if len(externalVersions) == 0 {
		glog.V(4).Infof("No version is registered for group %v", api.GroupName)
		return
	}

	if err := registered.EnableVersions(externalVersions...); err != nil {
		glog.V(4).Infof("%v", err)
		return
	}
	if err := enableVersions(externalVersions); err != nil {
		glog.V(4).Infof("%v", err)
		return
	}
}

// TODO: enableVersions should be centralized rather than spread in each API
// group.
// We can combine registered.RegisterVersions, registered.EnableVersions and
// registered.RegisterGroup once we have moved enableVersions there.
func enableVersions(externalVersions []unversioned.GroupVersion) error {
	addVersionsToScheme(externalVersions...)
	preferredExternalVersion := externalVersions[0]

	groupMeta := apimachinery.GroupMeta{
		GroupVersion:  preferredExternalVersion,
		GroupVersions: externalVersions,
		RESTMapper:    newRESTMapper(externalVersions),
		SelfLinker:    runtime.SelfLinker(accessor),
		InterfacesFor: interfacesFor,
	}

	if err := registered.RegisterGroup(groupMeta); err != nil {
		return err
	}
	api.RegisterRESTMapper(groupMeta.RESTMapper)
	return nil
}

func newRESTMapper(externalVersions []unversioned.GroupVersion) meta.RESTMapper {
	// the list of kinds that are scoped at the root of the api hierarchy
	// if a kind is not enumerated here, it is assumed to have a namespace scope
	rootScoped := sets.NewString(
		"Node",
		"Namespace",
		"PersistentVolume",
		"ComponentStatus",
	)

	// these kinds should be excluded from the list of resources
	ignoredKinds := sets.NewString(
		"ListOptions",
		"DeleteOptions",
		"Status",
		"PodLogOptions",
		"PodExecOptions",
		"PodAttachOptions",
		"PodProxyOptions",
		"NodeProxyOptions",
		"ServiceProxyOptions",
		"ThirdPartyResource",
		"ThirdPartyResourceData",
		"ThirdPartyResourceList")

	return api.NewDefaultRESTMapper(externalVersions, interfacesFor, importPrefix, ignoredKinds, rootScoped)
}

// InterfacesFor returns the default Codec and ResourceVersioner for a given version
// string, or an error if the version is not known.
func interfacesFor(version unversioned.GroupVersion) (*meta.VersionInterfaces, error) {
	switch version {
	case v1.SchemeGroupVersion:
		return &meta.VersionInterfaces{
			ObjectConvertor:  api.Scheme,
			MetadataAccessor: accessor,
		}, nil
	default:
		g, _ := registered.Group(api.GroupName)
		return nil, fmt.Errorf("unsupported storage version: %s (valid: %v)", version, g.GroupVersions)
	}
}

func addVersionsToScheme(externalVersions ...unversioned.GroupVersion) {
	// add the internal version to Scheme
	api.AddToScheme(api.Scheme)
	// add the enabled external versions to Scheme
	for _, v := range externalVersions {
		if !registered.IsEnabledVersion(v) {
			glog.Errorf("Version %s is not enabled, so it will not be added to the Scheme.", v)
			continue
		}
		switch v {
		case v1.SchemeGroupVersion:
			v1.AddToScheme(api.Scheme)
		}
	}

	// This is a "fast-path" that avoids reflection for common types. It focuses on the objects that are
	// converted the most in the cluster.
	// TODO: generate one of these for every external API group - this is to prove the impact
	api.Scheme.AddGenericConversionFunc(func(objA, objB interface{}, s conversion.Scope) (bool, error) {
		switch a := objA.(type) {
		case *v1.Pod:
			switch b := objB.(type) {
			case *api.Pod:
				return true, v1.Convert_v1_Pod_To_api_Pod(a, b, s)
			}
		case *api.Pod:
			switch b := objB.(type) {
			case *v1.Pod:
				return true, v1.Convert_api_Pod_To_v1_Pod(a, b, s)
			}

		case *v1.Event:
			switch b := objB.(type) {
			case *api.Event:
				return true, v1.Convert_v1_Event_To_api_Event(a, b, s)
			}
		case *api.Event:
			switch b := objB.(type) {
			case *v1.Event:
				return true, v1.Convert_api_Event_To_v1_Event(a, b, s)
			}

		case *v1.ReplicationController:
			switch b := objB.(type) {
			case *api.ReplicationController:
				return true, v1.Convert_v1_ReplicationController_To_api_ReplicationController(a, b, s)
			}
		case *api.ReplicationController:
			switch b := objB.(type) {
			case *v1.ReplicationController:
				return true, v1.Convert_api_ReplicationController_To_v1_ReplicationController(a, b, s)
			}

		case *v1.Node:
			switch b := objB.(type) {
			case *api.Node:
				return true, v1.Convert_v1_Node_To_api_Node(a, b, s)
			}
		case *api.Node:
			switch b := objB.(type) {
			case *v1.Node:
				return true, v1.Convert_api_Node_To_v1_Node(a, b, s)
			}

		case *v1.Namespace:
			switch b := objB.(type) {
			case *api.Namespace:
				return true, v1.Convert_v1_Namespace_To_api_Namespace(a, b, s)
			}
		case *api.Namespace:
			switch b := objB.(type) {
			case *v1.Namespace:
				return true, v1.Convert_api_Namespace_To_v1_Namespace(a, b, s)
			}

		case *v1.Service:
			switch b := objB.(type) {
			case *api.Service:
				return true, v1.Convert_v1_Service_To_api_Service(a, b, s)
			}
		case *api.Service:
			switch b := objB.(type) {
			case *v1.Service:
				return true, v1.Convert_api_Service_To_v1_Service(a, b, s)
			}

		case *v1.Endpoints:
			switch b := objB.(type) {
			case *api.Endpoints:
				return true, v1.Convert_v1_Endpoints_To_api_Endpoints(a, b, s)
			}
		case *api.Endpoints:
			switch b := objB.(type) {
			case *v1.Endpoints:
				return true, v1.Convert_api_Endpoints_To_v1_Endpoints(a, b, s)
			}

		case *versioned.Event:
			switch b := objB.(type) {
			case *versioned.InternalEvent:
				return true, versioned.Convert_versioned_Event_to_versioned_InternalEvent(a, b, s)
			}
		case *versioned.InternalEvent:
			switch b := objB.(type) {
			case *versioned.Event:
				return true, versioned.Convert_versioned_InternalEvent_to_versioned_Event(a, b, s)
			}
		}
		return false, nil
	})
}
<commit_msg>Revert "Extend all to more resources"<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package install installs the v1 monolithic api, making it available as an
// option to all of the API encoding/decoding machinery.
package install

import (
	"fmt"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/apimachinery"
	"k8s.io/kubernetes/pkg/apimachinery/registered"
	"k8s.io/kubernetes/pkg/conversion"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/sets"
	"k8s.io/kubernetes/pkg/watch/versioned"
)

const importPrefix = "k8s.io/kubernetes/pkg/api"

var accessor = meta.NewAccessor()

// availableVersions lists all known external versions for this group from most preferred to least preferred
var availableVersions = []unversioned.GroupVersion{v1.SchemeGroupVersion}

func init() {
	registered.RegisterVersions(availableVersions)
	externalVersions := []unversioned.GroupVersion{}
	for _, v := range availableVersions {
		if registered.IsAllowedVersion(v) {
			externalVersions = append(externalVersions, v)
		}
	}
	if len(externalVersions) == 0 {
		glog.V(4).Infof("No version is registered for group %v", api.GroupName)
		return
	}

	if err := registered.EnableVersions(externalVersions...); err != nil {
		glog.V(4).Infof("%v", err)
		return
	}
	if err := enableVersions(externalVersions); err != nil {
		glog.V(4).Infof("%v", err)
		return
	}
}

// TODO: enableVersions should be centralized rather than spread in each API
// group.
// We can combine registered.RegisterVersions, registered.EnableVersions and
// registered.RegisterGroup once we have moved enableVersions there.
func enableVersions(externalVersions []unversioned.GroupVersion) error {
	addVersionsToScheme(externalVersions...)
	preferredExternalVersion := externalVersions[0]

	groupMeta := apimachinery.GroupMeta{
		GroupVersion:  preferredExternalVersion,
		GroupVersions: externalVersions,
		RESTMapper:    newRESTMapper(externalVersions),
		SelfLinker:    runtime.SelfLinker(accessor),
		InterfacesFor: interfacesFor,
	}

	if err := registered.RegisterGroup(groupMeta); err != nil {
		return err
	}
	api.RegisterRESTMapper(groupMeta.RESTMapper)
	return nil
}

// userResources is a group of resources mostly used by a kubectl user
var userResources = []string{"rc", "svc", "pods", "pvc"}

func newRESTMapper(externalVersions []unversioned.GroupVersion) meta.RESTMapper {
	// the list of kinds that are scoped at the root of the api hierarchy
	// if a kind is not enumerated here, it is assumed to have a namespace scope
	rootScoped := sets.NewString(
		"Node",
		"Namespace",
		"PersistentVolume",
		"ComponentStatus",
	)

	// these kinds should be excluded from the list of resources
	ignoredKinds := sets.NewString(
		"ListOptions",
		"DeleteOptions",
		"Status",
		"PodLogOptions",
		"PodExecOptions",
		"PodAttachOptions",
		"PodProxyOptions",
		"NodeProxyOptions",
		"ServiceProxyOptions",
		"ThirdPartyResource",
		"ThirdPartyResourceData",
		"ThirdPartyResourceList")

	mapper := api.NewDefaultRESTMapper(externalVersions, interfacesFor, importPrefix, ignoredKinds, rootScoped)
	// setup aliases for groups of resources
	mapper.AddResourceAlias("all", userResources...)

	return mapper
}

// InterfacesFor returns the default Codec and ResourceVersioner for a given version
// string, or an error if the version is not known.
func interfacesFor(version unversioned.GroupVersion) (*meta.VersionInterfaces, error) {
	switch version {
	case v1.SchemeGroupVersion:
		return &meta.VersionInterfaces{
			ObjectConvertor:  api.Scheme,
			MetadataAccessor: accessor,
		}, nil
	default:
		g, _ := registered.Group(api.GroupName)
		return nil, fmt.Errorf("unsupported storage version: %s (valid: %v)", version, g.GroupVersions)
	}
}

func addVersionsToScheme(externalVersions ...unversioned.GroupVersion) {
	// add the internal version to Scheme
	api.AddToScheme(api.Scheme)
	// add the enabled external versions to Scheme
	for _, v := range externalVersions {
		if !registered.IsEnabledVersion(v) {
			glog.Errorf("Version %s is not enabled, so it will not be added to the Scheme.", v)
			continue
		}
		switch v {
		case v1.SchemeGroupVersion:
			v1.AddToScheme(api.Scheme)
		}
	}

	// This is a "fast-path" that avoids reflection for common types. It focuses on the objects that are
	// converted the most in the cluster.
	// TODO: generate one of these for every external API group - this is to prove the impact
	api.Scheme.AddGenericConversionFunc(func(objA, objB interface{}, s conversion.Scope) (bool, error) {
		switch a := objA.(type) {
		case *v1.Pod:
			switch b := objB.(type) {
			case *api.Pod:
				return true, v1.Convert_v1_Pod_To_api_Pod(a, b, s)
			}
		case *api.Pod:
			switch b := objB.(type) {
			case *v1.Pod:
				return true, v1.Convert_api_Pod_To_v1_Pod(a, b, s)
			}

		case *v1.Event:
			switch b := objB.(type) {
			case *api.Event:
				return true, v1.Convert_v1_Event_To_api_Event(a, b, s)
			}
		case *api.Event:
			switch b := objB.(type) {
			case *v1.Event:
				return true, v1.Convert_api_Event_To_v1_Event(a, b, s)
			}

		case *v1.ReplicationController:
			switch b := objB.(type) {
			case *api.ReplicationController:
				return true, v1.Convert_v1_ReplicationController_To_api_ReplicationController(a, b, s)
			}
		case *api.ReplicationController:
			switch b := objB.(type) {
			case *v1.ReplicationController:
				return true, v1.Convert_api_ReplicationController_To_v1_ReplicationController(a, b, s)
			}

		case *v1.Node:
			switch b := objB.(type) {
			case *api.Node:
				return true, v1.Convert_v1_Node_To_api_Node(a, b, s)
			}
		case *api.Node:
			switch b := objB.(type) {
			case *v1.Node:
				return true, v1.Convert_api_Node_To_v1_Node(a, b, s)
			}

		case *v1.Namespace:
			switch b := objB.(type) {
			case *api.Namespace:
				return true, v1.Convert_v1_Namespace_To_api_Namespace(a, b, s)
			}
		case *api.Namespace:
			switch b := objB.(type) {
			case *v1.Namespace:
				return true, v1.Convert_api_Namespace_To_v1_Namespace(a, b, s)
			}

		case *v1.Service:
			switch b := objB.(type) {
			case *api.Service:
				return true, v1.Convert_v1_Service_To_api_Service(a, b, s)
			}
		case *api.Service:
			switch b := objB.(type) {
			case *v1.Service:
				return true, v1.Convert_api_Service_To_v1_Service(a, b, s)
			}

		case *v1.Endpoints:
			switch b := objB.(type) {
			case *api.Endpoints:
				return true, v1.Convert_v1_Endpoints_To_api_Endpoints(a, b, s)
			}
		case *api.Endpoints:
			switch b := objB.(type) {
			case *v1.Endpoints:
				return true, v1.Convert_api_Endpoints_To_v1_Endpoints(a, b, s)
			}

		case *versioned.Event:
			switch b := objB.(type) {
			case *versioned.InternalEvent:
				return true, versioned.Convert_versioned_Event_to_versioned_InternalEvent(a, b, s)
			}
		case *versioned.InternalEvent:
			switch b := objB.(type) {
			case *versioned.Event:
				return true, versioned.Convert_versioned_InternalEvent_to_versioned_Event(a, b, s)
			}
		}
		return false, nil
	})
}
<|endoftext|>


<commit_before>package stage

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/werf/logboek"

	"github.com/werf/werf/pkg/config"
	"github.com/werf/werf/pkg/container_runtime"
	"github.com/werf/werf/pkg/docker"
	imagePkg "github.com/werf/werf/pkg/image"
	"github.com/werf/werf/pkg/stapel"
	"github.com/werf/werf/pkg/storage"
	"github.com/werf/werf/pkg/util"
)

type getImportsOptions struct {
	Before StageName
	After  StageName
}

func getImports(imageBaseConfig *config.StapelImageBase, options *getImportsOptions) []*config.Import {
	var imports []*config.Import
	for _, elm := range imageBaseConfig.Import {
		if options.Before != "" && elm.Before != "" && elm.Before == string(options.Before) {
			imports = append(imports, elm)
		} else if options.After != "" && elm.After != "" && elm.After == string(options.After) {
			imports = append(imports, elm)
		}
	}

	return imports
}

func newImportsStage(imports []*config.Import, name StageName, baseStageOptions *NewBaseStageOptions) *ImportsStage {
	s := &ImportsStage{}
	s.imports = imports
	s.BaseStage = newBaseStage(name, baseStageOptions)
	return s
}

type ImportsStage struct {
	*BaseStage

	imports []*config.Import
}

func (s *ImportsStage) GetDependencies(ctx context.Context, c Conveyor, _, _ container_runtime.ImageInterface) (string, error) {
	var args []string

	for ind, elm := range s.imports {
		var sourceChecksum string
		var err error
		if err := logboek.Context(ctx).Info().LogProcess("Getting import %d source checksum ...", ind).DoError(func() error {
			sourceChecksum, err = s.getImportSourceChecksum(ctx, c, elm)
			return err
		}); err != nil {
			return "", fmt.Errorf("unable to get import %d source checksum: %s", ind, err)
		}

		args = append(args, sourceChecksum)
		args = append(args, elm.To)
		args = append(args, elm.Group, elm.Owner)
	}

	return util.Sha256Hash(args...), nil
}

func (s *ImportsStage) PrepareImage(ctx context.Context, c Conveyor, _, image container_runtime.ImageInterface) error {
	for _, elm := range s.imports {
		sourceImageName := getSourceImageName(elm)
		srv, err := c.GetImportServer(ctx, sourceImageName, elm.Stage)
		if err != nil {
			return fmt.Errorf("unable to get import server for image %q: %s", sourceImageName, err)
		}

		command := srv.GetCopyCommand(ctx, elm)
		image.Container().AddServiceRunCommands(command)

		imageServiceCommitChangeOptions := image.Container().ServiceCommitChangeOptions()

		labelKey := imagePkg.WerfImportChecksumLabelPrefix + getImportID(elm)

		importSourceID := getImportSourceID(c, elm)
		importMetadata, err := c.GetImportMetadata(ctx, s.projectName, importSourceID)
		if err != nil {
			return fmt.Errorf("unable to get import source checksum: %s", err)
		} else if importMetadata == nil {
			panic(fmt.Sprintf("import metadata %s not found", importSourceID))
		}
		labelValue := importMetadata.Checksum

		imageServiceCommitChangeOptions.AddLabel(map[string]string{labelKey: labelValue})
	}

	return nil
}

func (s *ImportsStage) getImportSourceChecksum(ctx context.Context, c Conveyor, importElm *config.Import) (string, error) {
	importSourceID := getImportSourceID(c, importElm)
	importMetadata, err := c.GetImportMetadata(ctx, s.projectName, importSourceID)
	if err != nil {
		return "", fmt.Errorf("unable to get import metadata: %s", err)
	}

	if importMetadata == nil {
		checksum, err := s.generateImportChecksum(ctx, c, importElm)
		if err != nil {
			return "", fmt.Errorf("unable to generate import source checksum: %s", err)
		}

		sourceImageID := getSourceImageID(c, importElm)
		importMetadata = &storage.ImportMetadata{
			ImportSourceID: importSourceID,
			SourceImageID:  sourceImageID,
			Checksum:       checksum,
		}

		if err := c.PutImportMetadata(ctx, s.projectName, importMetadata); err != nil {
			return "", fmt.Errorf("unable to put import metadata: %s", err)
		}
	}

	return importMetadata.Checksum, nil
}

func (s *ImportsStage) generateImportChecksum(ctx context.Context, c Conveyor, importElm *config.Import) (string, error) {
	sourceImageDockerImageName := getSourceImageDockerImageName(c, importElm)
	importSourceID := getImportSourceID(c, importElm)

	stapelContainerName, err := stapel.GetOrCreateContainer(ctx)
	if err != nil {
		return "", err
	}

	importHostTmpDir := filepath.Join(s.imageTmpDir, string(s.Name()), "imports", importSourceID)
	importContainerDir := s.containerWerfDir

	importScriptHostTmpPath := filepath.Join(importHostTmpDir, "script.sh")
	resultChecksumHostTmpPath := filepath.Join(importHostTmpDir, "checksum")
	importScriptContainerPath := path.Join(importContainerDir, "script.sh")
	resultChecksumContainerPath := path.Join(importContainerDir, "checksum")

	command := generateChecksumCommand(importElm.Add, importElm.IncludePaths, importElm.ExcludePaths, resultChecksumContainerPath)
	if err := stapel.CreateScript(importScriptHostTmpPath, []string{command}); err != nil {
		return "", fmt.Errorf("unable to create script: %s", err)
	}

	runArgs := []string{
		"--rm",
		"--user=0:0",
		"--workdir=/",
		fmt.Sprintf("--volumes-from=%s", stapelContainerName),
		fmt.Sprintf("--volume=%s:%s", importHostTmpDir, importContainerDir),
		fmt.Sprintf("--entrypoint=%s", stapel.BashBinPath()),
		sourceImageDockerImageName,
		importScriptContainerPath,
	}

	if debugImportSourceChecksum() {
		fmt.Println(runArgs)
	}

	if output, err := docker.CliRun_RecordedOutput(ctx, runArgs...); err != nil {
		logboek.Context(ctx).Error().LogF("%s", output)
		return "", err
	}

	data, err := ioutil.ReadFile(resultChecksumHostTmpPath)
	if err != nil {
		return "", fmt.Errorf("unable to read file with import source checksum: %s", err)
	}

	checksum := strings.TrimSpace(string(data))
	return checksum, nil
}

func generateChecksumCommand(from string, includePaths, excludePaths []string, resultChecksumPath string) string {
	findCommandParts := append([]string{}, stapel.FindBinPath(), from, "-type", "f")

	var nameIncludeArgs []string
	for _, includePath := range includePaths {
		nameIncludeArgs = append(nameIncludeArgs, fmt.Sprintf("-wholename \"%s\"", path.Join(from, includePath)))
	}

	if len(nameIncludeArgs) != 0 {
		findCommandParts = append(findCommandParts, fmt.Sprintf("\\( %s \\)", strings.Join(nameIncludeArgs, " -or ")))
	}

	var nameExcludeArgs []string
	for _, excludePath := range excludePaths {
		nameExcludeArgs = append(nameExcludeArgs, fmt.Sprintf("! -wholename \"%s\"", path.Join(from, excludePath)))
	}

	if len(nameExcludeArgs) != 0 {
		if len(nameIncludeArgs) != 0 {
			findCommandParts = append(findCommandParts, fmt.Sprintf("-and"))
		}

		findCommandParts = append(findCommandParts, fmt.Sprintf("\\( %s \\)", strings.Join(nameExcludeArgs, " -and ")))
	}

	findCommand := strings.Join(findCommandParts, " ")

	sortCommandParts := append([]string{}, stapel.SortBinPath(), "-n")
	sortCommand := strings.Join(sortCommandParts, " ")

	xargsCommandParts := append([]string{}, stapel.XargsBinPath(), stapel.Md5sumBinPath())
	xargsCommand := strings.Join(xargsCommandParts, " ")

	md5SumCommand := stapel.Md5sumBinPath()

	cutCommandParts := append([]string{}, stapel.CutBinPath(), "-d", "' '", "-f", "1")
	cutCommand := strings.Join(cutCommandParts, " ")

	commands := append([]string{}, findCommand, sortCommand, xargsCommand, md5SumCommand, cutCommand)
	command := fmt.Sprintf("%s > %s", strings.Join(commands, " | "), resultChecksumPath)

	return command
}

func getImportID(importElm *config.Import) string {
	return util.Sha256Hash(
		"ImageName", importElm.ImageName,
		"ArtifactName", importElm.ArtifactName,
		"Stage", importElm.Stage,
		"After", importElm.After,
		"Before", importElm.Before,
		"Add", importElm.Add,
		"To", importElm.To,
		"Group", importElm.Group,
		"Owner", importElm.Owner,
		"IncludePaths", strings.Join(importElm.IncludePaths, "///"),
		"ExcludePaths", strings.Join(importElm.ExcludePaths, "///"),
	)
}

func getImportSourceID(c Conveyor, importElm *config.Import) string {
	return util.Sha256Hash(
		"SourceImageContentDigest", getSourceImageContentDigest(c, importElm),
		"Add", importElm.Add,
		"IncludePaths", strings.Join(importElm.IncludePaths, "///"),
		"ExcludePaths", strings.Join(importElm.ExcludePaths, "///"),
	)
}

func getSourceImageDockerImageName(c Conveyor, importElm *config.Import) string {
	sourceImageName := getSourceImageName(importElm)

	var sourceImageDockerImageName string
	if importElm.Stage == "" {
		sourceImageDockerImageName = c.GetImageNameForLastImageStage(sourceImageName)
	} else {
		sourceImageDockerImageName = c.GetImageNameForImageStage(sourceImageName, importElm.Stage)
	}

	return sourceImageDockerImageName
}

func getSourceImageID(c Conveyor, importElm *config.Import) string {
	sourceImageName := getSourceImageName(importElm)

	var sourceImageID string
	if importElm.Stage == "" {
		sourceImageID = c.GetImageIDForLastImageStage(sourceImageName)
	} else {
		sourceImageID = c.GetImageIDForImageStage(sourceImageName, importElm.Stage)
	}

	return sourceImageID
}

func getSourceImageContentDigest(c Conveyor, importElm *config.Import) string {
	sourceImageName := getSourceImageName(importElm)

	var sourceImageContentDigest string
	if importElm.Stage == "" {
		sourceImageContentDigest = c.GetImageContentDigest(sourceImageName)
	} else {
		sourceImageContentDigest = c.GetImageStageContentDigest(sourceImageName, importElm.Stage)
	}

	return sourceImageContentDigest
}

func getSourceImageName(importElm *config.Import) string {
	var sourceImageName string
	if importElm.ImageName != "" {
		sourceImageName = importElm.ImageName
	} else {
		sourceImageName = importElm.ArtifactName
	}

	return sourceImageName
}

func debugImportSourceChecksum() bool {
	return os.Getenv("WERF_DEBUG_IMPORT_SOURCE_CHECKSUM") == "1"
}
<commit_msg>fix(stapel): changes in directories of import.include/excludePaths not triggered import<commit_after>package stage

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/werf/logboek"

	"github.com/werf/werf/pkg/config"
	"github.com/werf/werf/pkg/container_runtime"
	"github.com/werf/werf/pkg/docker"
	imagePkg "github.com/werf/werf/pkg/image"
	"github.com/werf/werf/pkg/stapel"
	"github.com/werf/werf/pkg/storage"
	"github.com/werf/werf/pkg/util"
)

type getImportsOptions struct {
	Before StageName
	After  StageName
}

func getImports(imageBaseConfig *config.StapelImageBase, options *getImportsOptions) []*config.Import {
	var imports []*config.Import
	for _, elm := range imageBaseConfig.Import {
		if options.Before != "" && elm.Before != "" && elm.Before == string(options.Before) {
			imports = append(imports, elm)
		} else if options.After != "" && elm.After != "" && elm.After == string(options.After) {
			imports = append(imports, elm)
		}
	}

	return imports
}

func newImportsStage(imports []*config.Import, name StageName, baseStageOptions *NewBaseStageOptions) *ImportsStage {
	s := &ImportsStage{}
	s.imports = imports
	s.BaseStage = newBaseStage(name, baseStageOptions)
	return s
}

type ImportsStage struct {
	*BaseStage

	imports []*config.Import
}

func (s *ImportsStage) GetDependencies(ctx context.Context, c Conveyor, _, _ container_runtime.ImageInterface) (string, error) {
	var args []string

	for ind, elm := range s.imports {
		var sourceChecksum string
		var err error
		if err := logboek.Context(ctx).Info().LogProcess("Getting import %d source checksum ...", ind).DoError(func() error {
			sourceChecksum, err = s.getImportSourceChecksum(ctx, c, elm)
			return err
		}); err != nil {
			return "", fmt.Errorf("unable to get import %d source checksum: %s", ind, err)
		}

		args = append(args, sourceChecksum)
		args = append(args, elm.To)
		args = append(args, elm.Group, elm.Owner)
	}

	return util.Sha256Hash(args...), nil
}

func (s *ImportsStage) PrepareImage(ctx context.Context, c Conveyor, _, image container_runtime.ImageInterface) error {
	for _, elm := range s.imports {
		sourceImageName := getSourceImageName(elm)
		srv, err := c.GetImportServer(ctx, sourceImageName, elm.Stage)
		if err != nil {
			return fmt.Errorf("unable to get import server for image %q: %s", sourceImageName, err)
		}

		command := srv.GetCopyCommand(ctx, elm)
		image.Container().AddServiceRunCommands(command)

		imageServiceCommitChangeOptions := image.Container().ServiceCommitChangeOptions()

		labelKey := imagePkg.WerfImportChecksumLabelPrefix + getImportID(elm)

		importSourceID := getImportSourceID(c, elm)
		importMetadata, err := c.GetImportMetadata(ctx, s.projectName, importSourceID)
		if err != nil {
			return fmt.Errorf("unable to get import source checksum: %s", err)
		} else if importMetadata == nil {
			panic(fmt.Sprintf("import metadata %s not found", importSourceID))
		}
		labelValue := importMetadata.Checksum

		imageServiceCommitChangeOptions.AddLabel(map[string]string{labelKey: labelValue})
	}

	return nil
}

func (s *ImportsStage) getImportSourceChecksum(ctx context.Context, c Conveyor, importElm *config.Import) (string, error) {
	importSourceID := getImportSourceID(c, importElm)
	importMetadata, err := c.GetImportMetadata(ctx, s.projectName, importSourceID)
	if err != nil {
		return "", fmt.Errorf("unable to get import metadata: %s", err)
	}

	if importMetadata == nil {
		checksum, err := s.generateImportChecksum(ctx, c, importElm)
		if err != nil {
			return "", fmt.Errorf("unable to generate import source checksum: %s", err)
		}

		sourceImageID := getSourceImageID(c, importElm)
		importMetadata = &storage.ImportMetadata{
			ImportSourceID: importSourceID,
			SourceImageID:  sourceImageID,
			Checksum:       checksum,
		}

		if err := c.PutImportMetadata(ctx, s.projectName, importMetadata); err != nil {
			return "", fmt.Errorf("unable to put import metadata: %s", err)
		}
	}

	return importMetadata.Checksum, nil
}

func (s *ImportsStage) generateImportChecksum(ctx context.Context, c Conveyor, importElm *config.Import) (string, error) {
	sourceImageDockerImageName := getSourceImageDockerImageName(c, importElm)
	importSourceID := getImportSourceID(c, importElm)

	stapelContainerName, err := stapel.GetOrCreateContainer(ctx)
	if err != nil {
		return "", err
	}

	importHostTmpDir := filepath.Join(s.imageTmpDir, string(s.Name()), "imports", importSourceID)
	importContainerDir := s.containerWerfDir

	importScriptHostTmpPath := filepath.Join(importHostTmpDir, "script.sh")
	resultChecksumHostTmpPath := filepath.Join(importHostTmpDir, "checksum")
	importScriptContainerPath := path.Join(importContainerDir, "script.sh")
	resultChecksumContainerPath := path.Join(importContainerDir, "checksum")

	command := generateChecksumCommand(importElm.Add, importElm.IncludePaths, importElm.ExcludePaths, resultChecksumContainerPath)
	if err := stapel.CreateScript(importScriptHostTmpPath, []string{command}); err != nil {
		return "", fmt.Errorf("unable to create script: %s", err)
	}

	runArgs := []string{
		"--rm",
		"--user=0:0",
		"--workdir=/",
		fmt.Sprintf("--volumes-from=%s", stapelContainerName),
		fmt.Sprintf("--volume=%s:%s", importHostTmpDir, importContainerDir),
		fmt.Sprintf("--entrypoint=%s", stapel.BashBinPath()),
		sourceImageDockerImageName,
		importScriptContainerPath,
	}

	if debugImportSourceChecksum() {
		fmt.Println(runArgs)
	}

	if output, err := docker.CliRun_RecordedOutput(ctx, runArgs...); err != nil {
		logboek.Context(ctx).Error().LogF("%s", output)
		return "", err
	}

	data, err := ioutil.ReadFile(resultChecksumHostTmpPath)
	if err != nil {
		return "", fmt.Errorf("unable to read file with import source checksum: %s", err)
	}

	checksum := strings.TrimSpace(string(data))
	return checksum, nil
}

func generateChecksumCommand(from string, includePaths, excludePaths []string, resultChecksumPath string) string {
	findCommandParts := append([]string{}, stapel.FindBinPath(), from, "-type", "f")

	var nameIncludeArgs []string
	for _, includePath := range includePaths {
		formattedPath := formatIncludeAndExcludePath(includePath)
		nameIncludeArgs = append(
			nameIncludeArgs,
			fmt.Sprintf("-wholename \"%s\"", path.Join(from, formattedPath)),
			fmt.Sprintf("-wholename \"%s\"", path.Join(from, formattedPath, "**")),
		)
	}

	if len(nameIncludeArgs) != 0 {
		findCommandParts = append(findCommandParts, fmt.Sprintf("\\( %s \\)", strings.Join(nameIncludeArgs, " -or ")))
	}

	var nameExcludeArgs []string
	for _, excludePath := range excludePaths {
		formattedPath := formatIncludeAndExcludePath(excludePath)
		nameExcludeArgs = append(
			nameExcludeArgs,
			fmt.Sprintf("! -wholename \"%s\"", path.Join(from, formattedPath)),
			fmt.Sprintf("! -wholename \"%s\"", path.Join(from, formattedPath, "**")),
		)
	}

	if len(nameExcludeArgs) != 0 {
		if len(nameIncludeArgs) != 0 {
			findCommandParts = append(findCommandParts, fmt.Sprintf("-and"))
		}

		findCommandParts = append(findCommandParts, fmt.Sprintf("\\( %s \\)", strings.Join(nameExcludeArgs, " -and ")))
	}

	findCommand := strings.Join(findCommandParts, " ")

	sortCommandParts := append([]string{}, stapel.SortBinPath(), "-n")
	sortCommand := strings.Join(sortCommandParts, " ")

	xargsCommandParts := append([]string{}, stapel.XargsBinPath(), stapel.Md5sumBinPath())
	xargsCommand := strings.Join(xargsCommandParts, " ")

	md5SumCommand := stapel.Md5sumBinPath()

	cutCommandParts := append([]string{}, stapel.CutBinPath(), "-d", "' '", "-f", "1")
	cutCommand := strings.Join(cutCommandParts, " ")

	commands := append([]string{}, findCommand, sortCommand, xargsCommand, md5SumCommand, cutCommand)
	command := fmt.Sprintf("%s > %s", strings.Join(commands, " | "), resultChecksumPath)

	return command
}

func formatIncludeAndExcludePath(path string) string {
	return strings.TrimRight(path, "*/")
}

func getImportID(importElm *config.Import) string {
	return util.Sha256Hash(
		"ImageName", importElm.ImageName,
		"ArtifactName", importElm.ArtifactName,
		"Stage", importElm.Stage,
		"After", importElm.After,
		"Before", importElm.Before,
		"Add", importElm.Add,
		"To", importElm.To,
		"Group", importElm.Group,
		"Owner", importElm.Owner,
		"IncludePaths", strings.Join(importElm.IncludePaths, "///"),
		"ExcludePaths", strings.Join(importElm.ExcludePaths, "///"),
	)
}

func getImportSourceID(c Conveyor, importElm *config.Import) string {
	return util.Sha256Hash(
		"SourceImageContentDigest", getSourceImageContentDigest(c, importElm),
		"Add", importElm.Add,
		"IncludePaths", strings.Join(importElm.IncludePaths, "///"),
		"ExcludePaths", strings.Join(importElm.ExcludePaths, "///"),
	)
}

func getSourceImageDockerImageName(c Conveyor, importElm *config.Import) string {
	sourceImageName := getSourceImageName(importElm)

	var sourceImageDockerImageName string
	if importElm.Stage == "" {
		sourceImageDockerImageName = c.GetImageNameForLastImageStage(sourceImageName)
	} else {
		sourceImageDockerImageName = c.GetImageNameForImageStage(sourceImageName, importElm.Stage)
	}

	return sourceImageDockerImageName
}

func getSourceImageID(c Conveyor, importElm *config.Import) string {
	sourceImageName := getSourceImageName(importElm)

	var sourceImageID string
	if importElm.Stage == "" {
		sourceImageID = c.GetImageIDForLastImageStage(sourceImageName)
	} else {
		sourceImageID = c.GetImageIDForImageStage(sourceImageName, importElm.Stage)
	}

	return sourceImageID
}

func getSourceImageContentDigest(c Conveyor, importElm *config.Import) string {
	sourceImageName := getSourceImageName(importElm)

	var sourceImageContentDigest string
	if importElm.Stage == "" {
		sourceImageContentDigest = c.GetImageContentDigest(sourceImageName)
	} else {
		sourceImageContentDigest = c.GetImageStageContentDigest(sourceImageName, importElm.Stage)
	}

	return sourceImageContentDigest
}

func getSourceImageName(importElm *config.Import) string {
	var sourceImageName string
	if importElm.ImageName != "" {
		sourceImageName = importElm.ImageName
	} else {
		sourceImageName = importElm.ArtifactName
	}

	return sourceImageName
}

func debugImportSourceChecksum() bool {
	return os.Getenv("WERF_DEBUG_IMPORT_SOURCE_CHECKSUM") == "1"
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cluster

import (
	"fmt"
	"math"
	"time"

	"github.com/coreos/etcd-operator/pkg/util/etcdutil"
	"github.com/coreos/etcd-operator/pkg/util/k8sutil"
	"github.com/coreos/etcd-operator/pkg/util/retryutil"

	"github.com/pborman/uuid"
)

func (c *Cluster) addOneSelfHostedMember() error {
	c.status.AppendScalingUpCondition(c.members.Size(), c.cluster.Spec.Size)

	newMember := c.newMember(c.memberCounter)
	c.memberCounter++
	peerURL := newMember.PeerURL()
	initialCluster := append(c.members.PeerURLPairs(), newMember.Name+"="+peerURL)

	ns := c.cluster.Metadata.Namespace
	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, c.members.ClientURLs(), c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())

	_, err := c.config.KubeCli.CoreV1().Pods(ns).Create(pod)
	if err != nil {
		return err
	}
	// wait for the new pod to start and add itself into the etcd cluster.
	oldN := c.members.Size()
	err = retryutil.Retry(10*time.Second, 6, func() (bool, error) {
		err = c.updateMembers(c.members)
		if err != nil {
			c.logger.Warningf("unable to update members: %v", err)
			return false, nil
		}
		if c.members.Size() > oldN {
			return true, nil
		}
		c.logger.Infof("still waiting for the new self hosted member (%s) to start...", newMember.Name)
		return false, nil
	})
	if err != nil {
		c.logger.Warningf("failed to add member (%s) due to scheduling failure. Removing its pod (%s)", newMember.Name, pod.Name)
		// our reconcile loop assumes that no new member is added without control.
		retryutil.Retry(10*time.Second, math.MaxInt64, func() (bool, error) {
			err := c.removePod(newMember.Name)
			if err != nil {
				c.logger.Errorf("failed to delete pod (%s), retry later: %v", newMember.Name, err)
				return false, nil
			}
			return true, nil
		})
		c.logger.Infof("pod (%s) has been removed", pod.Name)
		return nil
	}

	c.logger.Infof("added a self-hosted member (%s)", newMember.Name)
	return nil
}

func (c *Cluster) newSelfHostedSeedMember() error {
	newMember := c.newMember(c.memberCounter)
	c.memberCounter++
	initialCluster := []string{newMember.Name + "=" + newMember.PeerURL()}

	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, nil, c.cluster.Metadata.Name, "new", uuid.New(), c.cluster.Spec, c.cluster.AsOwner())
	_, err := k8sutil.CreateAndWaitPod(c.config.KubeCli, c.cluster.Metadata.Namespace, pod, 30*time.Second)
	if err != nil {
		return err
	}

	c.logger.Infof("self-hosted cluster created with seed member (%s)", newMember.Name)
	return nil
}

func (c *Cluster) migrateBootMember() error {
	endpoint := c.cluster.Spec.SelfHosted.BootMemberClientEndpoint

	c.logger.Infof("migrating boot member (%s)", endpoint)

	resp, err := etcdutil.ListMembers([]string{endpoint}, c.tlsConfig)
	if err != nil {
		return fmt.Errorf("failed to list members from boot member (%v)", err)
	}
	if len(resp.Members) != 1 {
		return fmt.Errorf("boot cluster contains more than one member")
	}
	bootMember := resp.Members[0]

	initialCluster := make([]string, 0)
	for _, purl := range bootMember.PeerURLs {
		initialCluster = append(initialCluster, fmt.Sprintf("%s=%s", bootMember.Name, purl))
	}

	// create the member inside Kubernetes for migration
	newMember := c.newMember(c.memberCounter)
	c.memberCounter++

	peerURL := newMember.PeerURL()
	initialCluster = append(initialCluster, newMember.Name+"="+peerURL)

	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, []string{endpoint}, c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())
	ns := c.cluster.Metadata.Namespace
	_, err = k8sutil.CreateAndWaitPod(c.config.KubeCli, ns, pod, 30*time.Second)
	if err != nil {
		return err
	}

	go func() {
		// TODO: a shorter timeout?
		// Waiting here for cluster to get stable:
		// - etcd data are replicated;
		// - cluster TPR state has switched to "Running"
		delay := 60 * time.Second
		c.logger.Infof("wait %v before removing the boot member", delay)
		time.Sleep(delay)

		err = etcdutil.RemoveMember([]string{newMember.ClientAddr()}, c.tlsConfig, bootMember.ID)
		if err != nil {
			c.logger.Errorf("boot member migration: failed to remove the boot member (%v)", err)
		}
	}()

	c.logger.Infof("self-hosted cluster created with boot member (%s)", endpoint)

	return nil
}
<commit_msg>selfHosted: print warning if not enough nodes to scale<commit_after>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cluster

import (
	"fmt"
	"math"
	"time"

	"github.com/coreos/etcd-operator/pkg/util/etcdutil"
	"github.com/coreos/etcd-operator/pkg/util/k8sutil"
	"github.com/coreos/etcd-operator/pkg/util/retryutil"

	"github.com/pborman/uuid"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
)

func (c *Cluster) selectMasterNodes() ([]string, error) {
	var selector string
	if c.cluster.Spec.Pod != nil && len(c.cluster.Spec.Pod.NodeSelector) != 0 {
		selector = labels.SelectorFromSet(c.cluster.Spec.Pod.NodeSelector).String()
	}
	nodes, err := c.config.KubeCli.CoreV1().Nodes().List(metav1.ListOptions{
		LabelSelector: selector,
	})
	if err != nil {
		return nil, err
	}
	res := make([]string, len(nodes.Items))
	for i := range nodes.Items {
		res[i] = nodes.Items[i].Name
	}
	return res, nil
}

func (c *Cluster) addOneSelfHostedMember() error {
	selectedNodes, err := c.selectMasterNodes()
	if err != nil {
		return err
	}
	if nodeNum := len(selectedNodes); nodeNum < c.cluster.Spec.Size {
		c.logger.Warningf("cannot scale to size (%d), only have %d nodes (%v)", c.cluster.Spec.Size, nodeNum, selectedNodes)
		return nil
	}

	c.status.AppendScalingUpCondition(c.members.Size(), c.cluster.Spec.Size)

	newMember := c.newMember(c.memberCounter)
	c.memberCounter++
	peerURL := newMember.PeerURL()
	initialCluster := append(c.members.PeerURLPairs(), newMember.Name+"="+peerURL)

	ns := c.cluster.Metadata.Namespace
	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, c.members.ClientURLs(), c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())

	_, err = c.config.KubeCli.CoreV1().Pods(ns).Create(pod)
	if err != nil {
		return err
	}
	// wait for the new pod to start and add itself into the etcd cluster.
	oldN := c.members.Size()
	err = retryutil.Retry(10*time.Second, 6, func() (bool, error) {
		err = c.updateMembers(c.members)
		if err != nil {
			c.logger.Warningf("unable to update members: %v", err)
			return false, nil
		}
		if c.members.Size() > oldN {
			return true, nil
		}
		c.logger.Infof("still waiting for the new self hosted member (%s) to start...", newMember.Name)
		return false, nil
	})
	if err != nil {
		c.logger.Warningf("failed to add member (%s) due to scheduling failure. Removing its pod (%s)", newMember.Name, pod.Name)
		// our reconcile loop assumes that no new member is added without control.
		retryutil.Retry(10*time.Second, math.MaxInt64, func() (bool, error) {
			err := c.removePod(newMember.Name)
			if err != nil {
				c.logger.Errorf("failed to delete pod (%s), retry later: %v", newMember.Name, err)
				return false, nil
			}
			return true, nil
		})
		c.logger.Infof("pod (%s) has been removed", pod.Name)
		return nil
	}

	c.logger.Infof("added a self-hosted member (%s)", newMember.Name)
	return nil
}

func (c *Cluster) newSelfHostedSeedMember() error {
	newMember := c.newMember(c.memberCounter)
	c.memberCounter++
	initialCluster := []string{newMember.Name + "=" + newMember.PeerURL()}

	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, nil, c.cluster.Metadata.Name, "new", uuid.New(), c.cluster.Spec, c.cluster.AsOwner())
	_, err := k8sutil.CreateAndWaitPod(c.config.KubeCli, c.cluster.Metadata.Namespace, pod, 30*time.Second)
	if err != nil {
		return err
	}

	c.logger.Infof("self-hosted cluster created with seed member (%s)", newMember.Name)
	return nil
}

func (c *Cluster) migrateBootMember() error {
	endpoint := c.cluster.Spec.SelfHosted.BootMemberClientEndpoint

	c.logger.Infof("migrating boot member (%s)", endpoint)

	resp, err := etcdutil.ListMembers([]string{endpoint}, c.tlsConfig)
	if err != nil {
		return fmt.Errorf("failed to list members from boot member (%v)", err)
	}
	if len(resp.Members) != 1 {
		return fmt.Errorf("boot cluster contains more than one member")
	}
	bootMember := resp.Members[0]

	initialCluster := make([]string, 0)
	for _, purl := range bootMember.PeerURLs {
		initialCluster = append(initialCluster, fmt.Sprintf("%s=%s", bootMember.Name, purl))
	}

	// create the member inside Kubernetes for migration
	newMember := c.newMember(c.memberCounter)
	c.memberCounter++

	peerURL := newMember.PeerURL()
	initialCluster = append(initialCluster, newMember.Name+"="+peerURL)

	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, []string{endpoint}, c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())
	ns := c.cluster.Metadata.Namespace
	_, err = k8sutil.CreateAndWaitPod(c.config.KubeCli, ns, pod, 30*time.Second)
	if err != nil {
		return err
	}

	go func() {
		// TODO: a shorter timeout?
		// Waiting here for cluster to get stable:
		// - etcd data are replicated;
		// - cluster TPR state has switched to "Running"
		delay := 60 * time.Second
		c.logger.Infof("wait %v before removing the boot member", delay)
		time.Sleep(delay)

		err = etcdutil.RemoveMember([]string{newMember.ClientAddr()}, c.tlsConfig, bootMember.ID)
		if err != nil {
			c.logger.Errorf("boot member migration: failed to remove the boot member (%v)", err)
		}
	}()

	c.logger.Infof("self-hosted cluster created with boot member (%s)", endpoint)

	return nil
}
<|endoftext|>


<commit_before>package component

import (
	"fmt"

	"rsprd.com/spread/pkg/deploy"

	"k8s.io/kubernetes/pkg/api"
)

// A Component is an entity (potentially containing sub-components) that can be deployed to Kubernetes.
type Component interface {
	deploy.Deployable
	Type() Type
	Objects() []deploy.KubeObject
	Source() string
	DefaultMeta() api.ObjectMeta
}

// Base provides fields that are shared between all Components.
type Base struct {
	componentType Type
	objects       deploy.Deployment
	source        string
	defaults      api.ObjectMeta
}

func newBase(t Type, defaults api.ObjectMeta, source string, objects []deploy.KubeObject) (base Base, err error) {
	base.defaults = defaults

	deployment := deploy.Deployment{}
	for _, obj := range objects {
		setMetaDefaults(obj, defaults)
		err = deployment.Add(obj)
		if err != nil {
			err = fmt.Errorf("error adding '%s': %v", source, err)
			return
		}
	}

	base.source = source
	base.componentType = t
	base.objects = deployment
	return
}

// Objects returns slice of objects attached to Component
func (base Base) Objects() []deploy.KubeObject {
	return base.objects.Objects()
}

// Source returns an import source specific identifier
func (base Base) Source() string {
	return base.source
}

func (base Base) DefaultMeta() api.ObjectMeta {
	return base.defaults
}

// Type returns itself for trivial implementation of Component
func (base Base) Type() Type {
	return base.componentType
}

// Type identifies the component's type.
type Type int

const (
	ComponentApplication           Type = iota // Application (top of tree)
	ComponentReplicationController             // Wrapper for api.ReplicationController
	ComponentPod                               // Wrapper for api.Pod
	ComponentContainer                         // Wrapper for api.Container
	ComponentImage                             // Represented by api.Container's image field
)

// metaDefaults applies a set of defaults on a KubeObject. Non-empty fields on object override defaults.
func setMetaDefaults(obj deploy.KubeObject, defaults api.ObjectMeta) {
	meta := obj.GetObjectMeta()

	// if namespace is not set, use default
	namespace := api.NamespaceDefault
	if len(defaults.Namespace) > 0 {
		namespace = defaults.Namespace
	}

	if len(meta.GetNamespace()) == 0 {
		meta.SetNamespace(namespace)
	}

	// if name and generateName are not set use default generateName
	if len(defaults.GenerateName) > 0 && len(meta.GetName()) == 0 && len(meta.GetGenerateName()) == 0 {
		meta.SetGenerateName(defaults.GenerateName)
	}

	// set default labels
	labels := defaults.Labels
	for k, v := range meta.GetLabels() {
		labels[k] = v
	}
	meta.SetLabels(labels)

	// set default annotations
	annotations := defaults.Annotations
	for k, v := range meta.GetAnnotations() {
		annotations[k] = v
	}
	meta.SetAnnotations(annotations)
}
<commit_msg>added Attach method to component<commit_after>package component

import (
	"fmt"

	"rsprd.com/spread/pkg/deploy"

	"k8s.io/kubernetes/pkg/api"
)

// A Component is an entity (potentially containing sub-components) that can be deployed to Kubernetes.
type Component interface {
	deploy.Deployable
	Type() Type
	Objects() []deploy.KubeObject
	Source() string
	Attach(Component) error
	DefaultMeta() api.ObjectMeta
}

// Base provides fields that are shared between all Components.
type Base struct {
	componentType Type
	objects       deploy.Deployment
	source        string
	defaults      api.ObjectMeta
}

func newBase(t Type, defaults api.ObjectMeta, source string, objects []deploy.KubeObject) (base Base, err error) {
	base.defaults = defaults

	deployment := deploy.Deployment{}
	for _, obj := range objects {
		setMetaDefaults(obj, defaults)
		err = deployment.Add(obj)
		if err != nil {
			err = fmt.Errorf("error adding '%s': %v", source, err)
			return
		}
	}

	base.source = source
	base.componentType = t
	base.objects = deployment
	return
}

// Objects returns slice of objects attached to Component
func (base Base) Objects() []deploy.KubeObject {
	return base.objects.Objects()
}

// Source returns an import source specific identifier
func (base Base) Source() string {
	return base.source
}

// DefaultMeta returns the ObjectMeta that the Component was created with
func (base Base) DefaultMeta() api.ObjectMeta {
	return base.defaults
}

// Type returns itself for trivial implementation of Component
func (base Base) Type() Type {
	return base.componentType
}

// Type identifies the component's type.
type Type int

const (
	ComponentApplication           Type = iota // Application (top of tree)
	ComponentReplicationController             // Wrapper for api.ReplicationController
	ComponentPod                               // Wrapper for api.Pod
	ComponentContainer                         // Wrapper for api.Container
	ComponentImage                             // Represented by api.Container's image field
)

// metaDefaults applies a set of defaults on a KubeObject. Non-empty fields on object override defaults.
func setMetaDefaults(obj deploy.KubeObject, defaults api.ObjectMeta) {
	meta := obj.GetObjectMeta()

	// if namespace is not set, use default
	namespace := api.NamespaceDefault
	if len(defaults.Namespace) > 0 {
		namespace = defaults.Namespace
	}

	if len(meta.GetNamespace()) == 0 {
		meta.SetNamespace(namespace)
	}

	// if name and generateName are not set use default generateName
	if len(defaults.GenerateName) > 0 && len(meta.GetName()) == 0 && len(meta.GetGenerateName()) == 0 {
		meta.SetGenerateName(defaults.GenerateName)
	}

	// set default labels
	labels := defaults.Labels
	for k, v := range meta.GetLabels() {
		labels[k] = v
	}
	meta.SetLabels(labels)

	// set default annotations
	annotations := defaults.Annotations
	for k, v := range meta.GetAnnotations() {
		annotations[k] = v
	}
	meta.SetAnnotations(annotations)
}
<|endoftext|>