<commit_before>// Copyright (c) 2017, Janoš Guljaš <janos@resenje.org>
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package templates

import (
	"bytes"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"path/filepath"
)

// Error is a common error type that holds
// information about error message and template name.
type Error struct {
	Err      error
	Template string
}

func (e *Error) Error() string {
	if e.Template == "" {
		return e.Err.Error()
	}
	return fmt.Sprintf("%s: %s", e.Err.Error(), e.Template)
}

// ErrUnknownTemplate will be returned by Render function if
// the template does not exist.
var ErrUnknownTemplate = fmt.Errorf("unknown template")

// Options holds parameters for creating Templates.
type Options struct {
	baseDir     string
	contentType string
	files       map[string][]string
	strings     map[string][]string
	functions   template.FuncMap
	delimOpen   string
	delimClose  string
	logf        func(format string, a ...interface{})
}

// Option sets parameters used in New function.
type Option func(*Options)

// WithContentType sets the content type HTTP header that
// will be written on Render and Response functions.
func WithContentType(contentType string) Option {
	return func(o *Options) { o.contentType = contentType }
}

// WithBaseDir sets the directory in which template files
// are stored.
func WithBaseDir(dir string) Option {
	return func(o *Options) { o.baseDir = dir }
}

// WithTemplateFromFiles adds a template parsed from files.
func WithTemplateFromFiles(name string, files ...string) Option {
	return func(o *Options) { o.files[name] = files }
}

// WithTemplatesFromFiles adds a map of templates parsed from files.
func WithTemplatesFromFiles(ts map[string][]string) Option {
	return func(o *Options) {
		for name, files := range ts {
			o.files[name] = files
		}
	}
}

// WithTemplateFromStrings adds a template parsed from string.
func WithTemplateFromStrings(name string, strings ...string) Option {
	return func(o *Options) { o.strings[name] = strings }
}

// WithTemplatesFromStrings adds a map of templates parsed from strings.
func WithTemplatesFromStrings(ts map[string][]string) Option {
	return func(o *Options) {
		for name, strings := range ts {
			o.strings[name] = strings
		}
	}
}

// WithFunction adds a function to templates.
func WithFunction(name string, fn interface{}) Option {
	return func(o *Options) { o.functions[name] = fn }
}

// WithFunctions adds function map to templates.
func WithFunctions(fns template.FuncMap) Option {
	return func(o *Options) {
		for name, fn := range fns {
			o.functions[name] = fn
		}
	}
}

// WithDelims sets the delimiters used in templates.
func WithDelims(open, close string) Option {
	return func(o *Options) {
		o.delimOpen = open
		o.delimClose = close
	}
}

// WithLogFunc sets the function that will perform message logging.
// Default is log.Printf.
func WithLogFunc(logf func(format string, a ...interface{})) Option {
	return func(o *Options) { o.logf = logf }
}

// Templates structure holds parsed templates.
type Templates struct {
	templates   map[string]*template.Template
	defaultName string
	contentType string
	logf        func(format string, a ...interface{})
}

// New creates a new instance of Templates and parses
// provided files and strings.
func New(opts ...Option) (t *Templates, err error) {
	functions := template.FuncMap{}
	for name, fn := range defaultFunctions {
		functions[name] = fn
	}
	o := &Options{
		files:      map[string][]string{},
		functions:  functions,
		delimOpen:  "{{",
		delimClose: "}}",
		logf:       log.Printf,
	}
	for _, opt := range opts {
		opt(o)
	}

	t = &Templates{
		templates:   map[string]*template.Template{},
		contentType: o.contentType,
		logf:        o.logf,
	}
	for name, strings := range o.strings {
		tpl, err := parseStrings(template.New("").Funcs(o.functions).Delims(o.delimOpen, o.delimClose), strings...)
		if err != nil {
			return nil, err
		}
		t.templates[name] = tpl
	}
	for name, files := range o.files {
		fs := []string{}
		for _, f := range files {
			fs = append(fs, filepath.Join(o.baseDir, f))
		}
		tpl, err := parseFiles(template.New("").Funcs(o.functions).Delims(o.delimOpen, o.delimClose), fs...)
		if err != nil {
			return nil, err
		}
		t.templates[name] = tpl
	}
	return
}

// RespondTemplateWithStatus executes a named template with provided data into buffer,
// then writes the the status and body to the response writer.
// A panic will be raised if the template does not exist or fails to execute.
func (t Templates) RespondTemplateWithStatus(w http.ResponseWriter, name, templateName string, data interface{}, status int) {
	buf := bytes.Buffer{}
	tpl, ok := t.templates[name]
	if !ok {
		panic(&Error{Err: ErrUnknownTemplate, Template: name})
	}
	if err := tpl.ExecuteTemplate(&buf, templateName, data); err != nil {
		panic(err)
	}
	if t.contentType != "" {
		w.Header().Set("Content-Type", t.contentType)
	}
	if status > 0 {
		w.WriteHeader(status)
	}
	if _, err := buf.WriteTo(w); err != nil {
		t.logf("respond %q template %q: %v", name, templateName, err)
	}
}

// RespondWithStatus executes a template with provided data into buffer,
// then writes the the status and body to the response writer.
// A panic will be raised if the template does not exist or fails to execute.
func (t Templates) RespondWithStatus(w http.ResponseWriter, name string, data interface{}, status int) {
	buf := bytes.Buffer{}
	tpl, ok := t.templates[name]
	if !ok {
		panic(&Error{Err: ErrUnknownTemplate, Template: name})
	}
	if err := tpl.Execute(&buf, data); err != nil {
		panic(err)
	}
	if t.contentType != "" {
		w.Header().Set("Content-Type", t.contentType)
	}
	if status > 0 {
		w.WriteHeader(status)
	}
	if _, err := buf.WriteTo(w); err != nil {
		t.logf("respond %q: %v", name, err)
	}
}

// RespondTemplate executes a named template with provided data into buffer,
// then writes the the body to the response writer.
// A panic will be raised if the template does not exist or fails to execute.
func (t Templates) RespondTemplate(w http.ResponseWriter, name, templateName string, data interface{}) {
	t.RespondTemplateWithStatus(w, name, templateName, data, 0)
}

// Respond executes template with provided data into buffer,
// then writes the the body to the response writer.
// A panic will be raised if the template does not exist or fails to execute.
func (t Templates) Respond(w http.ResponseWriter, name string, data interface{}) {
	t.RespondWithStatus(w, name, data, 0)
}

// RenderTemplate executes a named template and returns the string.
func (t Templates) RenderTemplate(name, templateName string, data interface{}) (s string, err error) {
	buf := bytes.Buffer{}
	tpl, ok := t.templates[name]
	if !ok {
		return "", &Error{Err: ErrUnknownTemplate, Template: name}
	}
	if err := tpl.ExecuteTemplate(&buf, templateName, data); err != nil {
		return "", err
	}
	return buf.String(), nil
}

// Render executes a template and returns the string.
func (t Templates) Render(name string, data interface{}) (s string, err error) {
	buf := bytes.Buffer{}
	tpl, ok := t.templates[name]
	if !ok {
		return "", &Error{Err: ErrUnknownTemplate, Template: name}
	}
	if err := tpl.Execute(&buf, data); err != nil {
		return "", err
	}
	return buf.String(), nil
}

func parseFiles(t *template.Template, filenames ...string) (*template.Template, error) {
	for _, filename := range filenames {
		b, err := ioutil.ReadFile(filename)
		if err != nil {
			return nil, err
		}
		_, err = t.Parse(string(b))
		if err != nil {
			return nil, err
		}
	}
	return t, nil
}

func parseStrings(t *template.Template, strings ...string) (*template.Template, error) {
	for _, str := range strings {
		_, err := t.Parse(str)
		if err != nil {
			return nil, err
		}
	}
	return t, nil
}
<commit_msg>Add templates.WithFileFindFunc option<commit_after>// Copyright (c) 2017, Janoš Guljaš <janos@resenje.org>
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package templates

import (
	"bytes"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"path/filepath"
)

// Error is a common error type that holds
// information about error message and template name.
type Error struct {
	Err      error
	Template string
}

func (e *Error) Error() string {
	if e.Template == "" {
		return e.Err.Error()
	}
	return fmt.Sprintf("%s: %s", e.Err.Error(), e.Template)
}

// ErrUnknownTemplate will be returned by Render function if
// the template does not exist.
var ErrUnknownTemplate = fmt.Errorf("unknown template")

// Options holds parameters for creating Templates.
type Options struct {
	fileFindFunc func(filename string) string
	contentType  string
	files        map[string][]string
	strings      map[string][]string
	functions    template.FuncMap
	delimOpen    string
	delimClose   string
	logf         func(format string, a ...interface{})
}

// Option sets parameters used in New function.
type Option func(*Options)

// WithContentType sets the content type HTTP header that
// will be written on Render and Response functions.
func WithContentType(contentType string) Option {
	return func(o *Options) { o.contentType = contentType }
}

// WithBaseDir sets the directory in which template files
// are stored.
func WithBaseDir(dir string) Option {
	return func(o *Options) {
		o.fileFindFunc = func(f string) string {
			return filepath.Join(dir, f)
		}
	}
}

// WithFileFindFunc sets the function that will return the
// file path on disk based on filename provided from files
// defind using WithTemplateFromFile or WithTemplateFromFiles.
func WithFileFindFunc(fn func(filename string) string) Option {
	return func(o *Options) { o.fileFindFunc = fn }
}

// WithTemplateFromFiles adds a template parsed from files.
func WithTemplateFromFiles(name string, files ...string) Option {
	return func(o *Options) { o.files[name] = files }
}

// WithTemplatesFromFiles adds a map of templates parsed from files.
func WithTemplatesFromFiles(ts map[string][]string) Option {
	return func(o *Options) {
		for name, files := range ts {
			o.files[name] = files
		}
	}
}

// WithTemplateFromStrings adds a template parsed from string.
func WithTemplateFromStrings(name string, strings ...string) Option {
	return func(o *Options) { o.strings[name] = strings }
}

// WithTemplatesFromStrings adds a map of templates parsed from strings.
func WithTemplatesFromStrings(ts map[string][]string) Option {
	return func(o *Options) {
		for name, strings := range ts {
			o.strings[name] = strings
		}
	}
}

// WithFunction adds a function to templates.
func WithFunction(name string, fn interface{}) Option {
	return func(o *Options) { o.functions[name] = fn }
}

// WithFunctions adds function map to templates.
func WithFunctions(fns template.FuncMap) Option {
	return func(o *Options) {
		for name, fn := range fns {
			o.functions[name] = fn
		}
	}
}

// WithDelims sets the delimiters used in templates.
func WithDelims(open, close string) Option {
	return func(o *Options) {
		o.delimOpen = open
		o.delimClose = close
	}
}

// WithLogFunc sets the function that will perform message logging.
// Default is log.Printf.
func WithLogFunc(logf func(format string, a ...interface{})) Option {
	return func(o *Options) { o.logf = logf }
}

// Templates structure holds parsed templates.
type Templates struct {
	templates   map[string]*template.Template
	defaultName string
	contentType string
	logf        func(format string, a ...interface{})
}

// New creates a new instance of Templates and parses
// provided files and strings.
func New(opts ...Option) (t *Templates, err error) {
	functions := template.FuncMap{}
	for name, fn := range defaultFunctions {
		functions[name] = fn
	}
	o := &Options{
		files:      map[string][]string{},
		functions:  functions,
		delimOpen:  "{{",
		delimClose: "}}",
		logf:       log.Printf,
	}
	for _, opt := range opts {
		opt(o)
	}

	t = &Templates{
		templates:   map[string]*template.Template{},
		contentType: o.contentType,
		logf:        o.logf,
	}
	for name, strings := range o.strings {
		tpl, err := parseStrings(template.New("").Funcs(o.functions).Delims(o.delimOpen, o.delimClose), strings...)
		if err != nil {
			return nil, err
		}
		t.templates[name] = tpl
	}
	for name, files := range o.files {
		fs := []string{}
		for _, f := range files {
			fs = append(fs, o.fileFindFunc(f))
		}
		tpl, err := parseFiles(template.New("").Funcs(o.functions).Delims(o.delimOpen, o.delimClose), fs...)
		if err != nil {
			return nil, err
		}
		t.templates[name] = tpl
	}
	return
}

// RespondTemplateWithStatus executes a named template with provided data into buffer,
// then writes the the status and body to the response writer.
// A panic will be raised if the template does not exist or fails to execute.
func (t Templates) RespondTemplateWithStatus(w http.ResponseWriter, name, templateName string, data interface{}, status int) {
	buf := bytes.Buffer{}
	tpl, ok := t.templates[name]
	if !ok {
		panic(&Error{Err: ErrUnknownTemplate, Template: name})
	}
	if err := tpl.ExecuteTemplate(&buf, templateName, data); err != nil {
		panic(err)
	}
	if t.contentType != "" {
		w.Header().Set("Content-Type", t.contentType)
	}
	if status > 0 {
		w.WriteHeader(status)
	}
	if _, err := buf.WriteTo(w); err != nil {
		t.logf("respond %q template %q: %v", name, templateName, err)
	}
}

// RespondWithStatus executes a template with provided data into buffer,
// then writes the the status and body to the response writer.
// A panic will be raised if the template does not exist or fails to execute.
func (t Templates) RespondWithStatus(w http.ResponseWriter, name string, data interface{}, status int) {
	buf := bytes.Buffer{}
	tpl, ok := t.templates[name]
	if !ok {
		panic(&Error{Err: ErrUnknownTemplate, Template: name})
	}
	if err := tpl.Execute(&buf, data); err != nil {
		panic(err)
	}
	if t.contentType != "" {
		w.Header().Set("Content-Type", t.contentType)
	}
	if status > 0 {
		w.WriteHeader(status)
	}
	if _, err := buf.WriteTo(w); err != nil {
		t.logf("respond %q: %v", name, err)
	}
}

// RespondTemplate executes a named template with provided data into buffer,
// then writes the the body to the response writer.
// A panic will be raised if the template does not exist or fails to execute.
func (t Templates) RespondTemplate(w http.ResponseWriter, name, templateName string, data interface{}) {
	t.RespondTemplateWithStatus(w, name, templateName, data, 0)
}

// Respond executes template with provided data into buffer,
// then writes the the body to the response writer.
// A panic will be raised if the template does not exist or fails to execute.
func (t Templates) Respond(w http.ResponseWriter, name string, data interface{}) {
	t.RespondWithStatus(w, name, data, 0)
}

// RenderTemplate executes a named template and returns the string.
func (t Templates) RenderTemplate(name, templateName string, data interface{}) (s string, err error) {
	buf := bytes.Buffer{}
	tpl, ok := t.templates[name]
	if !ok {
		return "", &Error{Err: ErrUnknownTemplate, Template: name}
	}
	if err := tpl.ExecuteTemplate(&buf, templateName, data); err != nil {
		return "", err
	}
	return buf.String(), nil
}

// Render executes a template and returns the string.
func (t Templates) Render(name string, data interface{}) (s string, err error) {
	buf := bytes.Buffer{}
	tpl, ok := t.templates[name]
	if !ok {
		return "", &Error{Err: ErrUnknownTemplate, Template: name}
	}
	if err := tpl.Execute(&buf, data); err != nil {
		return "", err
	}
	return buf.String(), nil
}

func parseFiles(t *template.Template, filenames ...string) (*template.Template, error) {
	for _, filename := range filenames {
		b, err := ioutil.ReadFile(filename)
		if err != nil {
			return nil, err
		}
		_, err = t.Parse(string(b))
		if err != nil {
			return nil, err
		}
	}
	return t, nil
}

func parseStrings(t *template.Template, strings ...string) (*template.Template, error) {
	for _, str := range strings {
		_, err := t.Parse(str)
		if err != nil {
			return nil, err
		}
	}
	return t, nil
}
<|endoftext|>


<commit_before>package terraform

import (
	"bytes"
	"crypto/rand"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"

	"github.com/EngineerBetter/concourse-up/resource"
	"github.com/EngineerBetter/concourse-up/terraform/internal/aws"
	"github.com/EngineerBetter/concourse-up/terraform/internal/gcp"
)

const awsConst = "AWS"
const gcpConst = "GCP"

// InputVars exposes ConfigureDirectorManifestCPI
type InputVars interface {
	ConfigureTerraform(string) (string, error)
	Build(map[string]interface{}) error
}

// IAASMetadata holds IAAS specific terraform metadata
type IAASMetadata interface {
	AssertValid() error
	Init(*bytes.Buffer) error
	Get(string) (string, error)
}

//CLIInterface is interface the abstraction of execCmd
type CLIInterface interface {
	IAAS(string) (InputVars, IAASMetadata, error)
	Apply(InputVars, bool) error
	Destroy(InputVars) error
	BuildOutput(InputVars, IAASMetadata) error
}

// CLI struct holds the abstraction of execCmd
type CLI struct {
	execCmd func(string, ...string) *exec.Cmd
	Path    string
	iaas    string
}

// Option defines the arbitary element of Options for New
type Option func(*CLI) error

// Path returns the path of the terraform-cli as an Option
func Path(path string) Option {
	return func(c *CLI) error {
		c.Path = path
		return nil
	}
}

// DownloadTerraform returns the dowloaded CLI path Option
func DownloadTerraform() Option {
	return func(c *CLI) error {
		path, err := resource.TerraformCLIPath()
		c.Path = path
		return err
	}
}

// New provides a new CLI
func New(ops ...Option) (*CLI, error) {
	// @Note: we will have to switch between IAASs at this point
	// for the time being we are using directly AWS
	cli := &CLI{
		execCmd: exec.Command,
		Path:    "terraform",
	}
	for _, op := range ops {
		if err := op(cli); err != nil {
			return nil, err
		}
	}
	return cli, nil
}

// NullInputVars exposes ConfigureDirectorManifestCPI
type NullInputVars struct{}

// ConfigureTerraform is a nullified function
func (n *NullInputVars) ConfigureTerraform(string) (string, error) { return "", nil }

// Build is a nullified function
func (n *NullInputVars) Build(map[string]interface{}) error { return nil }

// NullMetadata holds IAAS specific terraform metadata
type NullMetadata struct{}

// AssertValid is a nullified function
func (n *NullMetadata) AssertValid() error { return nil }

// Init is a nullified function
func (n *NullMetadata) Init(*bytes.Buffer) error { return nil }

// Get is a nullified function
func (n *NullMetadata) Get(string) (string, error) { return "", nil }

// IAAS is returning the IAAS specific metadata and environment
func (c *CLI) IAAS(name string) (InputVars, IAASMetadata, error) {
	switch strings.ToUpper(name) {
	case awsConst: // nolint
		c.iaas = "AWS" // nolint
		return &aws.InputVars{}, &aws.Metadata{}, nil
	case gcpConst: // nolint
		c.iaas = "GCP" // nolint
		return &gcp.InputVars{}, &gcp.Metadata{}, nil
	}
	return &NullInputVars{}, &NullMetadata{}, errors.New("terraform: " + name + " not a valid iaas provider")

}

func (c *CLI) init(config InputVars) (string, error) {
	var (
		tfConfig string
		err      error
	)
	switch c.iaas {
	case awsConst: // nolint
		tfConfig, err = config.ConfigureTerraform(resource.AWSTerraformConfig)
		if err != nil {
			return "", err
		}
	case gcpConst: // nolint
		tfConfig, err = config.ConfigureTerraform(resource.GCPTerraformConfig)
		if err != nil {
			return "", err
		}
	default:
		return "", errors.New("terraform: init cannot choose a provider")
	}

	terraformConfigPath, err := writeTempFile([]byte(tfConfig))
	if err != nil {
		return "", err
	}
	cmd := c.execCmd(c.Path, "init")
	cmd.Dir = terraformConfigPath
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err != nil {
		os.RemoveAll(terraformConfigPath)
		return "", err
	}
	return terraformConfigPath, nil
}

// Apply runs terraform apply for a given config
func (c *CLI) Apply(config InputVars, dryrun bool) error {
	terraformConfigPath, err := c.init(config)
	if err != nil {
		return err
	}

	defer os.RemoveAll(terraformConfigPath)

	action := "apply"
	if dryrun {
		action = "plan"
	}

	cmd := c.execCmd(c.Path, action, "-input=false", "-auto-approve")
	cmd.Dir = terraformConfigPath

	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout

	return cmd.Run()
}

// Destroy destroys terraform resources specified in a config file
func (c *CLI) Destroy(config InputVars) error {
	terraformConfigPath, err := c.init(config)
	if err != nil {
		return err
	}

	defer os.RemoveAll(terraformConfigPath)

	// A temporary fix for terraform destroy issues on gcp
	var cmd *exec.Cmd
	switch c.iaas {
	case awsConst:
		cmd = c.execCmd(c.Path, "destroy", "-auto-approve")
	case gcpConst:
		cmd = c.execCmd(c.Path, "destroy", "-auto-approve", "--parallelism=1")
	}

	cmd.Dir = terraformConfigPath
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

// BuildOutput builds the terraform output/metadata
func (c *CLI) BuildOutput(config InputVars, metadata IAASMetadata) error {
	terraformConfigPath, err := c.init(config)
	if err != nil {
		return err
	}

	defer os.RemoveAll(terraformConfigPath)

	stdoutBuffer := bytes.NewBuffer(nil)
	cmd := c.execCmd(c.Path, "output", "-json")
	cmd.Dir = terraformConfigPath
	cmd.Stderr = os.Stderr
	cmd.Stdout = stdoutBuffer
	if err := cmd.Run(); err != nil {
		return err
	}

	return metadata.Init(stdoutBuffer)
}

func writeTempFile(data []byte) (string, error) {
	mode := int(0740)
	perm := os.FileMode(mode)
	dirName := randomString()
	filePath := path.Join(os.TempDir(), dirName)
	err := os.MkdirAll(filePath, perm)
	if err != nil {
		return "", err
	}
	f, err := ioutil.TempFile(filePath, "*.tf")
	if err != nil {
		return "", err
	}
	_, err = f.Write(data)
	if err1 := f.Close(); err == nil {
		err = err1
	}
	if err != nil {
		os.RemoveAll(filePath)
		return "", err
	}
	return filePath, err
}

func randomString() string {
	b := make([]byte, 8)
	_, err := rand.Read(b)
	if err != nil {
		return ""
	}
	return fmt.Sprintf("%x%x%x%x", b[0:2], b[2:4], b[4:6], b[6:8])
}
<commit_msg>fix typo<commit_after>package terraform

import (
	"bytes"
	"crypto/rand"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"

	"github.com/EngineerBetter/concourse-up/resource"
	"github.com/EngineerBetter/concourse-up/terraform/internal/aws"
	"github.com/EngineerBetter/concourse-up/terraform/internal/gcp"
)

const awsConst = "AWS"
const gcpConst = "GCP"

// InputVars exposes ConfigureDirectorManifestCPI
type InputVars interface {
	ConfigureTerraform(string) (string, error)
	Build(map[string]interface{}) error
}

// IAASMetadata holds IAAS specific terraform metadata
type IAASMetadata interface {
	AssertValid() error
	Init(*bytes.Buffer) error
	Get(string) (string, error)
}

//CLIInterface is interface the abstraction of execCmd
type CLIInterface interface {
	IAAS(string) (InputVars, IAASMetadata, error)
	Apply(InputVars, bool) error
	Destroy(InputVars) error
	BuildOutput(InputVars, IAASMetadata) error
}

// CLI struct holds the abstraction of execCmd
type CLI struct {
	execCmd func(string, ...string) *exec.Cmd
	Path    string
	iaas    string
}

// Option defines the arbitary element of Options for New
type Option func(*CLI) error

// Path returns the path of the terraform-cli as an Option
func Path(path string) Option {
	return func(c *CLI) error {
		c.Path = path
		return nil
	}
}

// DownloadTerraform returns the dowloaded CLI path Option
func DownloadTerraform() Option {
	return func(c *CLI) error {
		path, err := resource.TerraformCLIPath()
		c.Path = path
		return err
	}
}

// New provides a new CLI
func New(ops ...Option) (*CLI, error) {
	// @Note: we will have to switch between IAASs at this point
	// for the time being we are using directly AWS
	cli := &CLI{
		execCmd: exec.Command,
		Path:    "terraform",
	}
	for _, op := range ops {
		if err := op(cli); err != nil {
			return nil, err
		}
	}
	return cli, nil
}

// NullInputVars exposes ConfigureDirectorManifestCPI
type NullInputVars struct{}

// ConfigureTerraform is a nullified function
func (n *NullInputVars) ConfigureTerraform(string) (string, error) { return "", nil }

// Build is a nullified function
func (n *NullInputVars) Build(map[string]interface{}) error { return nil }

// NullMetadata holds IAAS specific terraform metadata
type NullMetadata struct{}

// AssertValid is a nullified function
func (n *NullMetadata) AssertValid() error { return nil }

// Init is a nullified function
func (n *NullMetadata) Init(*bytes.Buffer) error { return nil }

// Get is a nullified function
func (n *NullMetadata) Get(string) (string, error) { return "", nil }

// IAAS is returning the IAAS specific metadata and environment
func (c *CLI) IAAS(name string) (InputVars, IAASMetadata, error) {
	switch strings.ToUpper(name) {
	case awsConst: // nolint
		c.iaas = "AWS" // nolint
		return &aws.InputVars{}, &aws.Metadata{}, nil
	case gcpConst: // nolint
		c.iaas = "GCP" // nolint
		return &gcp.InputVars{}, &gcp.Metadata{}, nil
	}
	return &NullInputVars{}, &NullMetadata{}, errors.New("terraform: " + name + " not a valid iaas provider")

}

func (c *CLI) init(config InputVars) (string, error) {
	var (
		tfConfig string
		err      error
	)
	switch c.iaas {
	case awsConst: // nolint
		tfConfig, err = config.ConfigureTerraform(resource.AWSTerraformConfig)
		if err != nil {
			return "", err
		}
	case gcpConst: // nolint
		tfConfig, err = config.ConfigureTerraform(resource.GCPTerraformConfig)
		if err != nil {
			return "", err
		}
	default:
		return "", errors.New("terraform: init cannot choose a provider")
	}

	terraformConfigPath, err := writeTempFile([]byte(tfConfig))
	if err != nil {
		return "", err
	}
	cmd := c.execCmd(c.Path, "init")
	cmd.Dir = terraformConfigPath
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err != nil {
		os.RemoveAll(terraformConfigPath)
		return "", err
	}
	return terraformConfigPath, nil
}

// Apply runs terraform apply for a given config
func (c *CLI) Apply(config InputVars, dryrun bool) error {
	terraformConfigPath, err := c.init(config)
	if err != nil {
		return err
	}

	defer os.RemoveAll(terraformConfigPath)

	action := "apply"
	if dryrun {
		action = "plan"
	}

	cmd := c.execCmd(c.Path, action, "-input=false", "-auto-approve")
	cmd.Dir = terraformConfigPath

	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout

	return cmd.Run()
}

// Destroy destroys terraform resources specified in a config file
func (c *CLI) Destroy(config InputVars) error {
	terraformConfigPath, err := c.init(config)
	if err != nil {
		return err
	}

	defer os.RemoveAll(terraformConfigPath)

	// A temporary fix for terraform destroy issues on gcp
	var cmd *exec.Cmd
	switch c.iaas {
	case awsConst:
		cmd = c.execCmd(c.Path, "destroy", "-auto-approve")
	case gcpConst:
		cmd = c.execCmd(c.Path, "destroy", "-auto-approve", "-parallelism=1")
	}

	cmd.Dir = terraformConfigPath
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

// BuildOutput builds the terraform output/metadata
func (c *CLI) BuildOutput(config InputVars, metadata IAASMetadata) error {
	terraformConfigPath, err := c.init(config)
	if err != nil {
		return err
	}

	defer os.RemoveAll(terraformConfigPath)

	stdoutBuffer := bytes.NewBuffer(nil)
	cmd := c.execCmd(c.Path, "output", "-json")
	cmd.Dir = terraformConfigPath
	cmd.Stderr = os.Stderr
	cmd.Stdout = stdoutBuffer
	if err := cmd.Run(); err != nil {
		return err
	}

	return metadata.Init(stdoutBuffer)
}

func writeTempFile(data []byte) (string, error) {
	mode := int(0740)
	perm := os.FileMode(mode)
	dirName := randomString()
	filePath := path.Join(os.TempDir(), dirName)
	err := os.MkdirAll(filePath, perm)
	if err != nil {
		return "", err
	}
	f, err := ioutil.TempFile(filePath, "*.tf")
	if err != nil {
		return "", err
	}
	_, err = f.Write(data)
	if err1 := f.Close(); err == nil {
		err = err1
	}
	if err != nil {
		os.RemoveAll(filePath)
		return "", err
	}
	return filePath, err
}

func randomString() string {
	b := make([]byte, 8)
	_, err := rand.Read(b)
	if err != nil {
		return ""
	}
	return fmt.Sprintf("%x%x%x%x", b[0:2], b[2:4], b[4:6], b[6:8])
}
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"
	"log"
	"sync"

	"github.com/hashicorp/terraform/config"
	"github.com/hashicorp/terraform/depgraph"
)

// Terraform is the primary structure that is used to interact with
// Terraform from code, and can perform operations such as returning
// all resources, a resource tree, a specific resource, etc.
type Terraform struct {
	providers map[string]ResourceProviderFactory
}

// This is a function type used to implement a walker for the resource
// tree internally on the Terraform structure.
type genericWalkFunc func(*Resource) (map[string]string, error)

// Config is the configuration that must be given to instantiate
// a Terraform structure.
type Config struct {
	Providers map[string]ResourceProviderFactory
}

// New creates a new Terraform structure, initializes resource providers
// for the given configuration, etc.
//
// Semantic checks of the entire configuration structure are done at this
// time, as well as richer checks such as verifying that the resource providers
// can be properly initialized, can be configured, etc.
func New(c *Config) (*Terraform, error) {
	return &Terraform{
		providers: c.Providers,
	}, nil
}

func (t *Terraform) Apply(p *Plan) (*State, error) {
	g, err := t.Graph(p.Config, p.State)
	if err != nil {
		return nil, err
	}

	return t.apply(g, p)
}

// Graph returns the dependency graph for the given configuration and
// state file.
//
// The resulting graph may have more resources than the configuration, because
// it can contain resources in the state file that need to be modified.
func (t *Terraform) Graph(c *config.Config, s *State) (*depgraph.Graph, error) {
	// Get the basic graph with the raw metadata
	g := Graph(c, s)
	if err := g.Validate(); err != nil {
		return nil, err
	}

	// Fill the graph with the providers
	if err := GraphFull(g, t.providers); err != nil {
		return nil, err
	}

	// Validate the graph so that it can setup a root and such
	if err := g.Validate(); err != nil {
		return nil, err
	}

	return g, nil
}

func (t *Terraform) Plan(
	c *config.Config, s *State, vs map[string]string) (*Plan, error) {
	g, err := t.Graph(c, s)
	if err != nil {
		return nil, err
	}

	return t.plan(g, c, s, vs)
}

// Refresh goes through all the resources in the state and refreshes them
// to their latest status.
func (t *Terraform) Refresh(c *config.Config, s *State) (*State, error) {
	g, err := t.Graph(c, s)
	if err != nil {
		return s, err
	}

	return t.refresh(g)
}

func (t *Terraform) apply(
	g *depgraph.Graph,
	p *Plan) (*State, error) {
	s := new(State)
	err := g.Walk(t.applyWalkFn(s, p.Vars))
	return s, err
}

func (t *Terraform) plan(
	g *depgraph.Graph,
	c *config.Config,
	s *State,
	vs map[string]string) (*Plan, error) {
	p := &Plan{
		Config: c,
		Vars:   vs,
		State:  s,
	}
	err := g.Walk(t.planWalkFn(p, vs))
	return p, err
}

func (t *Terraform) refresh(g *depgraph.Graph) (*State, error) {
	s := new(State)
	err := g.Walk(t.refreshWalkFn(s))
	return s, err
}

func (t *Terraform) refreshWalkFn(result *State) depgraph.WalkFunc {
	var l sync.Mutex

	// Initialize the result so we don't have to nil check everywhere
	result.init()

	cb := func(r *Resource) (map[string]string, error) {
		rs, err := r.Provider.Refresh(r.State)
		if err != nil {
			return nil, err
		}

		// Fix the type to be the type we have
		rs.Type = r.State.Type

		l.Lock()
		result.Resources[r.Id] = rs
		l.Unlock()

		return nil, nil
	}

	return t.genericWalkFn(nil, cb)
}

func (t *Terraform) applyWalkFn(
	result *State,
	vs map[string]string) depgraph.WalkFunc {
	var l sync.Mutex

	// Initialize the result
	result.init()

	cb := func(r *Resource) (map[string]string, error) {
		// Get the latest diff since there are no computed values anymore
		diff, err := r.Provider.Diff(r.State, r.Config)
		if err != nil {
			return nil, err
		}

		// TODO(mitchellh): we need to verify the diff doesn't change
		// anything and that the diff has no computed values (pre-computed)

		// With the completed diff, apply!
		rs, err := r.Provider.Apply(r.State, diff)
		if err != nil {
			return nil, err
		}

		// If no state was returned, then no variables were updated so
		// just return.
		if rs == nil {
			return nil, nil
		}

		var errs []error
		for ak, av := range rs.Attributes {
			// If the value is the unknown variable value, then it is an error.
			// In this case we record the error and remove it from the state
			if av == config.UnknownVariableValue {
				errs = append(errs, fmt.Errorf(
					"Attribute with unknown value: %s", ak))
				delete(rs.Attributes, ak)
			}
		}

		// Update the resulting diff
		l.Lock()
		result.Resources[r.Id] = rs
		l.Unlock()

		// Determine the new state and update variables
		vars := make(map[string]string)
		for ak, av := range rs.Attributes {
			vars[fmt.Sprintf("%s.%s", r.Id, ak)] = av
		}

		err = nil
		if len(errs) > 0 {
			err = &MultiError{Errors: errs}
		}

		return vars, err
	}

	return t.genericWalkFn(vs, cb)
}

func (t *Terraform) planWalkFn(
	result *Plan, vs map[string]string) depgraph.WalkFunc {
	var l sync.Mutex

	// Initialize the result
	result.init()

	cb := func(r *Resource) (map[string]string, error) {
		// Get a diff from the newest state
		diff, err := r.Provider.Diff(r.State, r.Config)
		if err != nil {
			return nil, err
		}

		l.Lock()
		if !diff.Empty() {
			result.Diff.Resources[r.Id] = diff
		}
		l.Unlock()

		// Determine the new state and update variables
		vars := make(map[string]string)
		if !diff.Empty() {
			r.State = r.State.MergeDiff(diff)
		}
		if r.State != nil {
			for ak, av := range r.State.Attributes {
				vars[fmt.Sprintf("%s.%s", r.Id, ak)] = av
			}
		}

		return vars, nil
	}

	return t.genericWalkFn(vs, cb)
}

func (t *Terraform) genericWalkFn(
	invars map[string]string,
	cb genericWalkFunc) depgraph.WalkFunc {
	var l sync.Mutex

	// Initialize the variables for application
	vars := make(map[string]string)
	for k, v := range invars {
		vars[fmt.Sprintf("var.%s", k)] = v
	}

	return func(n *depgraph.Noun) error {
		// If it is the root node, ignore
		if n.Name == GraphRootNode {
			return nil
		}

		switch m := n.Meta.(type) {
		case *GraphNodeResource:
		case *GraphNodeResourceProvider:
			var rc *ResourceConfig
			if m.Config != nil {
				if err := m.Config.RawConfig.Interpolate(vars); err != nil {
					panic(err)
				}
				rc = NewResourceConfig(m.Config.RawConfig)
			}

			for k, p := range m.Providers {
				log.Printf("Configuring provider: %s", k)
				err := p.Configure(rc)
				if err != nil {
					return err
				}
			}

			return nil
		}

		rn := n.Meta.(*GraphNodeResource)
		if len(vars) > 0 && rn.Config != nil {
			if err := rn.Config.RawConfig.Interpolate(vars); err != nil {
				panic(fmt.Sprintf("Interpolate error: %s", err))
			}

			// Force the config to be set later
			rn.Resource.Config = nil
		}

		// Make sure that at least some resource configuration is set
		if rn.Resource.Config == nil {
			if rn.Config == nil {
				rn.Resource.Config = new(ResourceConfig)
			} else {
				rn.Resource.Config = NewResourceConfig(rn.Config.RawConfig)
			}
		}

		// Call the callack
		newVars, err := cb(rn.Resource)
		if err != nil {
			return err
		}

		if len(newVars) > 0 {
			// Acquire a lock since this function is called in parallel
			l.Lock()
			defer l.Unlock()

			// Update variables
			for k, v := range newVars {
				vars[k] = v
			}
		}

		return nil
	}
}
<commit_msg>terraform: set type on apply<commit_after>package terraform

import (
	"fmt"
	"log"
	"sync"

	"github.com/hashicorp/terraform/config"
	"github.com/hashicorp/terraform/depgraph"
)

// Terraform is the primary structure that is used to interact with
// Terraform from code, and can perform operations such as returning
// all resources, a resource tree, a specific resource, etc.
type Terraform struct {
	providers map[string]ResourceProviderFactory
}

// This is a function type used to implement a walker for the resource
// tree internally on the Terraform structure.
type genericWalkFunc func(*Resource) (map[string]string, error)

// Config is the configuration that must be given to instantiate
// a Terraform structure.
type Config struct {
	Providers map[string]ResourceProviderFactory
}

// New creates a new Terraform structure, initializes resource providers
// for the given configuration, etc.
//
// Semantic checks of the entire configuration structure are done at this
// time, as well as richer checks such as verifying that the resource providers
// can be properly initialized, can be configured, etc.
func New(c *Config) (*Terraform, error) {
	return &Terraform{
		providers: c.Providers,
	}, nil
}

func (t *Terraform) Apply(p *Plan) (*State, error) {
	g, err := t.Graph(p.Config, p.State)
	if err != nil {
		return nil, err
	}

	return t.apply(g, p)
}

// Graph returns the dependency graph for the given configuration and
// state file.
//
// The resulting graph may have more resources than the configuration, because
// it can contain resources in the state file that need to be modified.
func (t *Terraform) Graph(c *config.Config, s *State) (*depgraph.Graph, error) {
	// Get the basic graph with the raw metadata
	g := Graph(c, s)
	if err := g.Validate(); err != nil {
		return nil, err
	}

	// Fill the graph with the providers
	if err := GraphFull(g, t.providers); err != nil {
		return nil, err
	}

	// Validate the graph so that it can setup a root and such
	if err := g.Validate(); err != nil {
		return nil, err
	}

	return g, nil
}

func (t *Terraform) Plan(
	c *config.Config, s *State, vs map[string]string) (*Plan, error) {
	g, err := t.Graph(c, s)
	if err != nil {
		return nil, err
	}

	return t.plan(g, c, s, vs)
}

// Refresh goes through all the resources in the state and refreshes them
// to their latest status.
func (t *Terraform) Refresh(c *config.Config, s *State) (*State, error) {
	g, err := t.Graph(c, s)
	if err != nil {
		return s, err
	}

	return t.refresh(g)
}

func (t *Terraform) apply(
	g *depgraph.Graph,
	p *Plan) (*State, error) {
	s := new(State)
	err := g.Walk(t.applyWalkFn(s, p.Vars))
	return s, err
}

func (t *Terraform) plan(
	g *depgraph.Graph,
	c *config.Config,
	s *State,
	vs map[string]string) (*Plan, error) {
	p := &Plan{
		Config: c,
		Vars:   vs,
		State:  s,
	}
	err := g.Walk(t.planWalkFn(p, vs))
	return p, err
}

func (t *Terraform) refresh(g *depgraph.Graph) (*State, error) {
	s := new(State)
	err := g.Walk(t.refreshWalkFn(s))
	return s, err
}

func (t *Terraform) refreshWalkFn(result *State) depgraph.WalkFunc {
	var l sync.Mutex

	// Initialize the result so we don't have to nil check everywhere
	result.init()

	cb := func(r *Resource) (map[string]string, error) {
		rs, err := r.Provider.Refresh(r.State)
		if err != nil {
			return nil, err
		}

		// Fix the type to be the type we have
		rs.Type = r.State.Type

		l.Lock()
		result.Resources[r.Id] = rs
		l.Unlock()

		return nil, nil
	}

	return t.genericWalkFn(nil, cb)
}

func (t *Terraform) applyWalkFn(
	result *State,
	vs map[string]string) depgraph.WalkFunc {
	var l sync.Mutex

	// Initialize the result
	result.init()

	cb := func(r *Resource) (map[string]string, error) {
		// Get the latest diff since there are no computed values anymore
		diff, err := r.Provider.Diff(r.State, r.Config)
		if err != nil {
			return nil, err
		}

		// TODO(mitchellh): we need to verify the diff doesn't change
		// anything and that the diff has no computed values (pre-computed)

		// With the completed diff, apply!
		rs, err := r.Provider.Apply(r.State, diff)
		if err != nil {
			return nil, err
		}

		// Force the resource state type to be our type
		rs.Type = r.State.Type

		// If no state was returned, then no variables were updated so
		// just return.
		if rs == nil {
			return nil, nil
		}

		var errs []error
		for ak, av := range rs.Attributes {
			// If the value is the unknown variable value, then it is an error.
			// In this case we record the error and remove it from the state
			if av == config.UnknownVariableValue {
				errs = append(errs, fmt.Errorf(
					"Attribute with unknown value: %s", ak))
				delete(rs.Attributes, ak)
			}
		}

		// Update the resulting diff
		l.Lock()
		result.Resources[r.Id] = rs
		l.Unlock()

		// Determine the new state and update variables
		vars := make(map[string]string)
		for ak, av := range rs.Attributes {
			vars[fmt.Sprintf("%s.%s", r.Id, ak)] = av
		}

		err = nil
		if len(errs) > 0 {
			err = &MultiError{Errors: errs}
		}

		return vars, err
	}

	return t.genericWalkFn(vs, cb)
}

func (t *Terraform) planWalkFn(
	result *Plan, vs map[string]string) depgraph.WalkFunc {
	var l sync.Mutex

	// Initialize the result
	result.init()

	cb := func(r *Resource) (map[string]string, error) {
		// Get a diff from the newest state
		diff, err := r.Provider.Diff(r.State, r.Config)
		if err != nil {
			return nil, err
		}

		l.Lock()
		if !diff.Empty() {
			result.Diff.Resources[r.Id] = diff
		}
		l.Unlock()

		// Determine the new state and update variables
		vars := make(map[string]string)
		if !diff.Empty() {
			r.State = r.State.MergeDiff(diff)
		}
		if r.State != nil {
			for ak, av := range r.State.Attributes {
				vars[fmt.Sprintf("%s.%s", r.Id, ak)] = av
			}
		}

		return vars, nil
	}

	return t.genericWalkFn(vs, cb)
}

func (t *Terraform) genericWalkFn(
	invars map[string]string,
	cb genericWalkFunc) depgraph.WalkFunc {
	var l sync.Mutex

	// Initialize the variables for application
	vars := make(map[string]string)
	for k, v := range invars {
		vars[fmt.Sprintf("var.%s", k)] = v
	}

	return func(n *depgraph.Noun) error {
		// If it is the root node, ignore
		if n.Name == GraphRootNode {
			return nil
		}

		switch m := n.Meta.(type) {
		case *GraphNodeResource:
		case *GraphNodeResourceProvider:
			var rc *ResourceConfig
			if m.Config != nil {
				if err := m.Config.RawConfig.Interpolate(vars); err != nil {
					panic(err)
				}
				rc = NewResourceConfig(m.Config.RawConfig)
			}

			for k, p := range m.Providers {
				log.Printf("Configuring provider: %s", k)
				err := p.Configure(rc)
				if err != nil {
					return err
				}
			}

			return nil
		}

		rn := n.Meta.(*GraphNodeResource)
		if len(vars) > 0 && rn.Config != nil {
			if err := rn.Config.RawConfig.Interpolate(vars); err != nil {
				panic(fmt.Sprintf("Interpolate error: %s", err))
			}

			// Force the config to be set later
			rn.Resource.Config = nil
		}

		// Make sure that at least some resource configuration is set
		if rn.Resource.Config == nil {
			if rn.Config == nil {
				rn.Resource.Config = new(ResourceConfig)
			} else {
				rn.Resource.Config = NewResourceConfig(rn.Config.RawConfig)
			}
		}

		// Call the callack
		newVars, err := cb(rn.Resource)
		if err != nil {
			return err
		}

		if len(newVars) > 0 {
			// Acquire a lock since this function is called in parallel
			l.Lock()
			defer l.Unlock()

			// Update variables
			for k, v := range newVars {
				vars[k] = v
			}
		}

		return nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"gopkg.in/flosch/pongo2.v3"
	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd/shared"
)

type TemplateEntry struct {
	When       []string
	Template   string
	Properties map[string]string
}

func templateApply(c *lxdContainer, trigger string) error {
	fname := shared.VarPath("lxc", c.name, "metadata.yaml")

	if !shared.PathExists(fname) {
		return nil
	}

	content, err := ioutil.ReadFile(fname)
	if err != nil {
		return err
	}

	metadata := new(imageMetadata)
	err = yaml.Unmarshal(content, &metadata)

	if err != nil {
		return fmt.Errorf("Could not parse %s: %v", fname, err)
	}

	for filepath, template := range metadata.Templates {
		var w *os.File

		found := false
		for _, tplTrigger := range template.When {
			if tplTrigger == trigger {
				found = true
				break
			}
		}

		if !found {
			continue
		}

		fullpath := shared.VarPath("lxc", c.name, "rootfs", strings.TrimLeft(filepath, "/"))

		if shared.PathExists(fullpath) {
			w, err = os.Create(fullpath)
			if err != nil {
				return err
			}
		} else {
			var uid, gid int
			if ! c.isPrivileged() {
				uid, gid := c.idmapset.ShiftIntoNs(0, 0)
				shared.MkdirAllOwner(path.Dir(fullpath), 0755, uid, gid)
			}

			w, err = os.Create(fullpath)
			if err != nil {
				return err
			}

			if ! c.isPrivileged() {
				w.Chown(uid, gid)
			}
			w.Chmod(0644)
		}

		tplString, err := ioutil.ReadFile(shared.VarPath("lxc", c.name, "templates", template.Template))
		if err != nil {
			return err
		}

		tpl, err := pongo2.FromString("{% autoescape off %}" + string(tplString) + "{% endautoescape %}")
		if err != nil {
			return err
		}

		containerMeta := make(map[string]string)
		containerMeta["name"] = c.name
		containerMeta["architecture"], _ = shared.ArchitectureName(c.architecture)

		if c.ephemeral {
			containerMeta["ephemeral"] = "true"
		} else {
			containerMeta["ephemeral"] = "false"
		}

		if c.isPrivileged() {
			containerMeta["privileged"] = "true"
		} else {
			containerMeta["privileged"] = "false"
		}

		configGet := func(confKey, confDefault *pongo2.Value) *pongo2.Value {
			val, ok := c.config[confKey.String()]
			if !ok {
				return confDefault
			}

			return pongo2.AsValue(strings.TrimRight(val, "\r\n"))
		}

		tpl.ExecuteWriter(pongo2.Context{"trigger": trigger,
			"path":       filepath,
			"container":  containerMeta,
			"config":     c.config,
			"devices":    c.devices,
			"properties": template.Properties,
			"config_get": configGet}, w)
	}

	return nil
}
<commit_msg>go fmt<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"gopkg.in/flosch/pongo2.v3"
	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd/shared"
)

type TemplateEntry struct {
	When       []string
	Template   string
	Properties map[string]string
}

func templateApply(c *lxdContainer, trigger string) error {
	fname := shared.VarPath("lxc", c.name, "metadata.yaml")

	if !shared.PathExists(fname) {
		return nil
	}

	content, err := ioutil.ReadFile(fname)
	if err != nil {
		return err
	}

	metadata := new(imageMetadata)
	err = yaml.Unmarshal(content, &metadata)

	if err != nil {
		return fmt.Errorf("Could not parse %s: %v", fname, err)
	}

	for filepath, template := range metadata.Templates {
		var w *os.File

		found := false
		for _, tplTrigger := range template.When {
			if tplTrigger == trigger {
				found = true
				break
			}
		}

		if !found {
			continue
		}

		fullpath := shared.VarPath("lxc", c.name, "rootfs", strings.TrimLeft(filepath, "/"))

		if shared.PathExists(fullpath) {
			w, err = os.Create(fullpath)
			if err != nil {
				return err
			}
		} else {
			var uid, gid int
			if !c.isPrivileged() {
				uid, gid := c.idmapset.ShiftIntoNs(0, 0)
				shared.MkdirAllOwner(path.Dir(fullpath), 0755, uid, gid)
			}

			w, err = os.Create(fullpath)
			if err != nil {
				return err
			}

			if !c.isPrivileged() {
				w.Chown(uid, gid)
			}
			w.Chmod(0644)
		}

		tplString, err := ioutil.ReadFile(shared.VarPath("lxc", c.name, "templates", template.Template))
		if err != nil {
			return err
		}

		tpl, err := pongo2.FromString("{% autoescape off %}" + string(tplString) + "{% endautoescape %}")
		if err != nil {
			return err
		}

		containerMeta := make(map[string]string)
		containerMeta["name"] = c.name
		containerMeta["architecture"], _ = shared.ArchitectureName(c.architecture)

		if c.ephemeral {
			containerMeta["ephemeral"] = "true"
		} else {
			containerMeta["ephemeral"] = "false"
		}

		if c.isPrivileged() {
			containerMeta["privileged"] = "true"
		} else {
			containerMeta["privileged"] = "false"
		}

		configGet := func(confKey, confDefault *pongo2.Value) *pongo2.Value {
			val, ok := c.config[confKey.String()]
			if !ok {
				return confDefault
			}

			return pongo2.AsValue(strings.TrimRight(val, "\r\n"))
		}

		tpl.ExecuteWriter(pongo2.Context{"trigger": trigger,
			"path":       filepath,
			"container":  containerMeta,
			"config":     c.config,
			"devices":    c.devices,
			"properties": template.Properties,
			"config_get": configGet}, w)
	}

	return nil
}
<|endoftext|>


<commit_before><commit_msg>Squelch most noise on first pass<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
)

func main() {
	var s string = "Hello, world!"
	var i int = 42
	var f float32 = 42.24
	fmt.Println("%v", s)
	fmt.Println("%s", s)
	fmt.Println("%v%v", s, i)
	fmt.Println("%v", i)
	fmt.Println("%v is a string", s)
	fmt.Println("%g", f)
	fmt.Println("%v", f)

	fmt.Println("%s%v", s, i)

	fmt.Println("%s%v%d", s, s, i)
}
<commit_msg>Fix errors based on review flow<commit_after>//Package main is an example package
package main

import (
	"fmt"
)

func main() {
	var s = "Hello, world!"
	var i = 42
	var f float32 = 42.24
	fmt.Println("%v", s)
	fmt.Println("%s", s)
	fmt.Println("%v%v", s, i)
	fmt.Println("%v", i)
	fmt.Println("%v is a string", s)
	fmt.Println("%g", f)
	fmt.Println("%v", f)

	fmt.Println("%s%v", s, i)

	fmt.Println("%s%v%d", s, s, i)
}
<|endoftext|>


<commit_before><commit_msg>simplify skip logic<commit_after><|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Marshalling and unmarshalling of
// JSON data into Go structs using reflection.

package json

import (
	"reflect";
	"strings";
)

type structBuilder struct {
	val	reflect.Value;

	// if map_ != nil, write val to map_[key] on each change
	map_	*reflect.MapValue;
	key	reflect.Value;
}

var nobuilder *structBuilder

func isfloat(v reflect.Value) bool {
	switch v.(type) {
	case *reflect.FloatValue, *reflect.Float32Value, *reflect.Float64Value:
		return true
	}
	return false;
}

func setfloat(v reflect.Value, f float64) {
	switch v := v.(type) {
	case *reflect.FloatValue:
		v.Set(float(f))
	case *reflect.Float32Value:
		v.Set(float32(f))
	case *reflect.Float64Value:
		v.Set(float64(f))
	}
}

func setint(v reflect.Value, i int64) {
	switch v := v.(type) {
	case *reflect.IntValue:
		v.Set(int(i))
	case *reflect.Int8Value:
		v.Set(int8(i))
	case *reflect.Int16Value:
		v.Set(int16(i))
	case *reflect.Int32Value:
		v.Set(int32(i))
	case *reflect.Int64Value:
		v.Set(int64(i))
	case *reflect.UintValue:
		v.Set(uint(i))
	case *reflect.Uint8Value:
		v.Set(uint8(i))
	case *reflect.Uint16Value:
		v.Set(uint16(i))
	case *reflect.Uint32Value:
		v.Set(uint32(i))
	case *reflect.Uint64Value:
		v.Set(uint64(i))
	}
}

// If updating b.val is not enough to update the original,
// copy a changed b.val out to the original.
func (b *structBuilder) Flush() {
	if b == nil {
		return
	}
	if b.map_ != nil {
		b.map_.SetElem(b.key, b.val)
	}
}

func (b *structBuilder) Int64(i int64) {
	if b == nil {
		return
	}
	v := b.val;
	if isfloat(v) {
		setfloat(v, float64(i))
	} else {
		setint(v, i)
	}
}

func (b *structBuilder) Uint64(i uint64) {
	if b == nil {
		return
	}
	v := b.val;
	if isfloat(v) {
		setfloat(v, float64(i))
	} else {
		setint(v, int64(i))
	}
}

func (b *structBuilder) Float64(f float64) {
	if b == nil {
		return
	}
	v := b.val;
	if isfloat(v) {
		setfloat(v, f)
	} else {
		setint(v, int64(f))
	}
}

func (b *structBuilder) Null()	{}

func (b *structBuilder) String(s string) {
	if b == nil {
		return
	}
	if v, ok := b.val.(*reflect.StringValue); ok {
		v.Set(s)
	}
}

func (b *structBuilder) Bool(tf bool) {
	if b == nil {
		return
	}
	if v, ok := b.val.(*reflect.BoolValue); ok {
		v.Set(tf)
	}
}

func (b *structBuilder) Array() {
	if b == nil {
		return
	}
	if v, ok := b.val.(*reflect.SliceValue); ok {
		if v.IsNil() {
			v.Set(reflect.MakeSlice(v.Type().(*reflect.SliceType), 0, 8))
		}
	}
}

func (b *structBuilder) Elem(i int) Builder {
	if b == nil || i < 0 {
		return nobuilder
	}
	switch v := b.val.(type) {
	case *reflect.ArrayValue:
		if i < v.Len() {
			return &structBuilder{val: v.Elem(i)}
		}
	case *reflect.SliceValue:
		if i >= v.Cap() {
			n := v.Cap();
			if n < 8 {
				n = 8
			}
			for n <= i {
				n *= 2
			}
			nv := reflect.MakeSlice(v.Type().(*reflect.SliceType), v.Len(), n);
			reflect.ArrayCopy(nv, v);
			v.Set(nv);
		}
		if v.Len() <= i && i < v.Cap() {
			v.SetLen(i + 1)
		}
		if i < v.Len() {
			return &structBuilder{val: v.Elem(i)}
		}
	}
	return nobuilder;
}

func (b *structBuilder) Map() {
	if b == nil {
		return
	}
	if v, ok := b.val.(*reflect.PtrValue); ok && v.IsNil() {
		if v.IsNil() {
			v.PointTo(reflect.MakeZero(v.Type().(*reflect.PtrType).Elem()));
			b.Flush();
		}
		b.map_ = nil;
		b.val = v.Elem();
	}
	if v, ok := b.val.(*reflect.MapValue); ok && v.IsNil() {
		v.Set(reflect.MakeMap(v.Type().(*reflect.MapType)))
	}
}

func (b *structBuilder) Key(k string) Builder {
	if b == nil {
		return nobuilder
	}
	switch v := reflect.Indirect(b.val).(type) {
	case *reflect.StructValue:
		t := v.Type().(*reflect.StructType);
		// Case-insensitive field lookup.
		k = strings.ToLower(k);
		for i := 0; i < t.NumField(); i++ {
			if strings.ToLower(t.Field(i).Name) == k {
				return &structBuilder{val: v.Field(i)}
			}
		}
	case *reflect.MapValue:
		t := v.Type().(*reflect.MapType);
		if t.Key() != reflect.Typeof(k) {
			break
		}
		key := reflect.NewValue(k);
		elem := v.Elem(key);
		if elem == nil {
			v.SetElem(key, reflect.MakeZero(t.Elem()));
			elem = v.Elem(key);
		}
		return &structBuilder{val: elem, map_: v, key: key};
	}
	return nobuilder;
}

// Unmarshal parses the JSON syntax string s and fills in
// an arbitrary struct or slice pointed at by val.
// It uses the reflect package to assign to fields
// and arrays embedded in val.  Well-formed data that does not fit
// into the struct is discarded.
//
// For example, given these definitions:
//
//	type Email struct {
//		Where string;
//		Addr string;
//	}
//
//	type Result struct {
//		Name string;
//		Phone string;
//		Email []Email
//	}
//
//	var r = Result{ "name", "phone", nil }
//
// unmarshalling the JSON syntax string
//
//	{
//	  "email": [
//	    {
//	      "where": "home",
//	      "addr": "gre@example.com"
//	    },
//	    {
//	      "where": "work",
//	      "addr": "gre@work.com"
//	    }
//	  ],
//	  "name": "Grace R. Emlin",
//	  "address": "123 Main Street"
//	}
//
// via Unmarshal(s, &r) is equivalent to assigning
//
//	r = Result{
//		"Grace R. Emlin",	// name
//		"phone",		// no phone given
//		[]Email{
//			Email{ "home", "gre@example.com" },
//			Email{ "work", "gre@work.com" }
//		}
//	}
//
// Note that the field r.Phone has not been modified and
// that the JSON field "address" was discarded.
//
// Because Unmarshal uses the reflect package, it can only
// assign to upper case fields.  Unmarshal uses a case-insensitive
// comparison to match JSON field names to struct field names.
//
// To unmarshal a top-level JSON array, pass in a pointer to an empty
// slice of the correct type.
//
// On success, Unmarshal returns with ok set to true.
// On a syntax error, it returns with ok set to false and errtok
// set to the offending token.
func Unmarshal(s string, val interface{}) (ok bool, errtok string) {
	v := reflect.NewValue(val);
	var b *structBuilder;

	// If val is a pointer to a slice, we mutate the pointee.
	if ptr, ok := v.(*reflect.PtrValue); ok {
		if slice, ok := ptr.Elem().(*reflect.SliceValue); ok {
			b = &structBuilder{val: slice}
		}
	}

	if b == nil {
		b = &structBuilder{val: v}
	}

	ok, _, errtok = Parse(s, b);
	if !ok {
		return false, errtok
	}
	return true, "";
}
<commit_msg>json: minor comment fix.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Marshalling and unmarshalling of
// JSON data into Go structs using reflection.

package json

import (
	"reflect";
	"strings";
)

type structBuilder struct {
	val	reflect.Value;

	// if map_ != nil, write val to map_[key] on each change
	map_	*reflect.MapValue;
	key	reflect.Value;
}

var nobuilder *structBuilder

func isfloat(v reflect.Value) bool {
	switch v.(type) {
	case *reflect.FloatValue, *reflect.Float32Value, *reflect.Float64Value:
		return true
	}
	return false;
}

func setfloat(v reflect.Value, f float64) {
	switch v := v.(type) {
	case *reflect.FloatValue:
		v.Set(float(f))
	case *reflect.Float32Value:
		v.Set(float32(f))
	case *reflect.Float64Value:
		v.Set(float64(f))
	}
}

func setint(v reflect.Value, i int64) {
	switch v := v.(type) {
	case *reflect.IntValue:
		v.Set(int(i))
	case *reflect.Int8Value:
		v.Set(int8(i))
	case *reflect.Int16Value:
		v.Set(int16(i))
	case *reflect.Int32Value:
		v.Set(int32(i))
	case *reflect.Int64Value:
		v.Set(int64(i))
	case *reflect.UintValue:
		v.Set(uint(i))
	case *reflect.Uint8Value:
		v.Set(uint8(i))
	case *reflect.Uint16Value:
		v.Set(uint16(i))
	case *reflect.Uint32Value:
		v.Set(uint32(i))
	case *reflect.Uint64Value:
		v.Set(uint64(i))
	}
}

// If updating b.val is not enough to update the original,
// copy a changed b.val out to the original.
func (b *structBuilder) Flush() {
	if b == nil {
		return
	}
	if b.map_ != nil {
		b.map_.SetElem(b.key, b.val)
	}
}

func (b *structBuilder) Int64(i int64) {
	if b == nil {
		return
	}
	v := b.val;
	if isfloat(v) {
		setfloat(v, float64(i))
	} else {
		setint(v, i)
	}
}

func (b *structBuilder) Uint64(i uint64) {
	if b == nil {
		return
	}
	v := b.val;
	if isfloat(v) {
		setfloat(v, float64(i))
	} else {
		setint(v, int64(i))
	}
}

func (b *structBuilder) Float64(f float64) {
	if b == nil {
		return
	}
	v := b.val;
	if isfloat(v) {
		setfloat(v, f)
	} else {
		setint(v, int64(f))
	}
}

func (b *structBuilder) Null()	{}

func (b *structBuilder) String(s string) {
	if b == nil {
		return
	}
	if v, ok := b.val.(*reflect.StringValue); ok {
		v.Set(s)
	}
}

func (b *structBuilder) Bool(tf bool) {
	if b == nil {
		return
	}
	if v, ok := b.val.(*reflect.BoolValue); ok {
		v.Set(tf)
	}
}

func (b *structBuilder) Array() {
	if b == nil {
		return
	}
	if v, ok := b.val.(*reflect.SliceValue); ok {
		if v.IsNil() {
			v.Set(reflect.MakeSlice(v.Type().(*reflect.SliceType), 0, 8))
		}
	}
}

func (b *structBuilder) Elem(i int) Builder {
	if b == nil || i < 0 {
		return nobuilder
	}
	switch v := b.val.(type) {
	case *reflect.ArrayValue:
		if i < v.Len() {
			return &structBuilder{val: v.Elem(i)}
		}
	case *reflect.SliceValue:
		if i >= v.Cap() {
			n := v.Cap();
			if n < 8 {
				n = 8
			}
			for n <= i {
				n *= 2
			}
			nv := reflect.MakeSlice(v.Type().(*reflect.SliceType), v.Len(), n);
			reflect.ArrayCopy(nv, v);
			v.Set(nv);
		}
		if v.Len() <= i && i < v.Cap() {
			v.SetLen(i + 1)
		}
		if i < v.Len() {
			return &structBuilder{val: v.Elem(i)}
		}
	}
	return nobuilder;
}

func (b *structBuilder) Map() {
	if b == nil {
		return
	}
	if v, ok := b.val.(*reflect.PtrValue); ok && v.IsNil() {
		if v.IsNil() {
			v.PointTo(reflect.MakeZero(v.Type().(*reflect.PtrType).Elem()));
			b.Flush();
		}
		b.map_ = nil;
		b.val = v.Elem();
	}
	if v, ok := b.val.(*reflect.MapValue); ok && v.IsNil() {
		v.Set(reflect.MakeMap(v.Type().(*reflect.MapType)))
	}
}

func (b *structBuilder) Key(k string) Builder {
	if b == nil {
		return nobuilder
	}
	switch v := reflect.Indirect(b.val).(type) {
	case *reflect.StructValue:
		t := v.Type().(*reflect.StructType);
		// Case-insensitive field lookup.
		k = strings.ToLower(k);
		for i := 0; i < t.NumField(); i++ {
			if strings.ToLower(t.Field(i).Name) == k {
				return &structBuilder{val: v.Field(i)}
			}
		}
	case *reflect.MapValue:
		t := v.Type().(*reflect.MapType);
		if t.Key() != reflect.Typeof(k) {
			break
		}
		key := reflect.NewValue(k);
		elem := v.Elem(key);
		if elem == nil {
			v.SetElem(key, reflect.MakeZero(t.Elem()));
			elem = v.Elem(key);
		}
		return &structBuilder{val: elem, map_: v, key: key};
	}
	return nobuilder;
}

// Unmarshal parses the JSON syntax string s and fills in
// an arbitrary struct or slice pointed at by val.
// It uses the reflect package to assign to fields
// and arrays embedded in val.  Well-formed data that does not fit
// into the struct is discarded.
//
// For example, given these definitions:
//
//	type Email struct {
//		Where string;
//		Addr string;
//	}
//
//	type Result struct {
//		Name string;
//		Phone string;
//		Email []Email
//	}
//
//	var r = Result{ "name", "phone", nil }
//
// unmarshalling the JSON syntax string
//
//	{
//	  "email": [
//	    {
//	      "where": "home",
//	      "addr": "gre@example.com"
//	    },
//	    {
//	      "where": "work",
//	      "addr": "gre@work.com"
//	    }
//	  ],
//	  "name": "Grace R. Emlin",
//	  "address": "123 Main Street"
//	}
//
// via Unmarshal(s, &r) is equivalent to assigning
//
//	r = Result{
//		"Grace R. Emlin",	// name
//		"phone",		// no phone given
//		[]Email{
//			Email{ "home", "gre@example.com" },
//			Email{ "work", "gre@work.com" }
//		}
//	}
//
// Note that the field r.Phone has not been modified and
// that the JSON field "address" was discarded.
//
// Because Unmarshal uses the reflect package, it can only
// assign to upper case fields.  Unmarshal uses a case-insensitive
// comparison to match JSON field names to struct field names.
//
// To unmarshal a top-level JSON array, pass in a pointer to an empty
// slice of the correct type.
//
// On success, Unmarshal returns with ok set to true.
// On a syntax error, it returns with ok set to false and errtok
// set to the offending token.
func Unmarshal(s string, val interface{}) (ok bool, errtok string) {
	v := reflect.NewValue(val);
	var b *structBuilder;

	// If val is a pointer to a slice, we append to the slice.
	if ptr, ok := v.(*reflect.PtrValue); ok {
		if slice, ok := ptr.Elem().(*reflect.SliceValue); ok {
			b = &structBuilder{val: slice}
		}
	}

	if b == nil {
		b = &structBuilder{val: v}
	}

	ok, _, errtok = Parse(s, b);
	if !ok {
		return false, errtok
	}
	return true, "";
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"testing"
)

func isEqual(a, b IP) bool {
	if a == nil && b == nil {
		return true
	}
	if a == nil || b == nil || len(a) != len(b) {
		return false
	}
	for i := 0; i < len(a); i++ {
		if a[i] != b[i] {
			return false
		}
	}
	return true
}

type parseIPTest struct {
	in  string
	out IP
}

var parseiptests = []parseIPTest{
	parseIPTest{"127.0.1.2", IPv4(127, 0, 1, 2)},
	parseIPTest{"127.0.0.1", IPv4(127, 0, 0, 1)},
	parseIPTest{"127.0.0.256", nil},
	parseIPTest{"abc", nil},
	parseIPTest{"::ffff:127.0.0.1", IPv4(127, 0, 0, 1)},
	parseIPTest{"2001:4860:0:2001::68",
		IP{0x20, 0x01, 0x48, 0x60, 0, 0, 0x20, 0x01,
			0, 0, 0, 0, 0, 0, 0x00, 0x68,
		},
	},
	parseIPTest{"::ffff:4a7d:1363", IPv4(74, 125, 19, 99)},
}

func TestParseIP(t *testing.T) {
	for i := 0; i < len(parseiptests); i++ {
		tt := parseiptests[i]
		if out := ParseIP(tt.in); !isEqual(out, tt.out) {
			t.Errorf("ParseIP(%#q) = %v, want %v", tt.in, out, tt.out)
		}
	}
}

type ipStringTest struct {
	in  IP
	out string
}

var ipstringtests = []ipStringTest{
	// cf. draft-ietf-6man-text-addr-representation-07
	// (A Recommendation for IPv6 Address Text Representation)
	ipStringTest{IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0,
		0, 0, 0x1, 0x23, 0, 0x12, 0, 0x1},
		"2001:db8::123:12:1"},
	ipStringTest{IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0x1},
		"2001:db8::1"},
	ipStringTest{IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0x1,
		0, 0, 0, 0x1, 0, 0, 0, 0x1},
		"2001:db8:0:1:0:1:0:1"},
	ipStringTest{IP{0x20, 0x1, 0xd, 0xb8, 0, 0x1, 0, 0,
		0, 0x1, 0, 0, 0, 0x1, 0, 0},
		"2001:db8:1:0:1:0:1:0"},
	ipStringTest{IP{0x20, 0x1, 0, 0, 0, 0, 0, 0,
		0, 0x1, 0, 0, 0, 0, 0, 0x1},
		"2001::1:0:0:1"},
	ipStringTest{IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0,
		0, 0x1, 0, 0, 0, 0, 0, 0},
		"2001:db8:0:0:1::"},
	ipStringTest{IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0,
		0, 0x1, 0, 0, 0, 0, 0, 0x1},
		"2001:db8::1:0:0:1"},
	ipStringTest{IP{0x20, 0x1, 0xD, 0xB8, 0, 0, 0, 0,
		0, 0xA, 0, 0xB, 0, 0xC, 0, 0xD},
		"2001:db8::a:b:c:d"},
}

func TestIPString(t *testing.T) {
	for i := 0; i < len(ipstringtests); i++ {
		tt := ipstringtests[i]
		if out := tt.in.String(); out != tt.out {
			t.Errorf("IP.String(%v) = %#q, want %#q", tt.in, out, tt.out)
		}
	}
}
<commit_msg>net: fix comment<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"testing"
)

func isEqual(a, b IP) bool {
	if a == nil && b == nil {
		return true
	}
	if a == nil || b == nil || len(a) != len(b) {
		return false
	}
	for i := 0; i < len(a); i++ {
		if a[i] != b[i] {
			return false
		}
	}
	return true
}

type parseIPTest struct {
	in  string
	out IP
}

var parseiptests = []parseIPTest{
	parseIPTest{"127.0.1.2", IPv4(127, 0, 1, 2)},
	parseIPTest{"127.0.0.1", IPv4(127, 0, 0, 1)},
	parseIPTest{"127.0.0.256", nil},
	parseIPTest{"abc", nil},
	parseIPTest{"::ffff:127.0.0.1", IPv4(127, 0, 0, 1)},
	parseIPTest{"2001:4860:0:2001::68",
		IP{0x20, 0x01, 0x48, 0x60, 0, 0, 0x20, 0x01,
			0, 0, 0, 0, 0, 0, 0x00, 0x68,
		},
	},
	parseIPTest{"::ffff:4a7d:1363", IPv4(74, 125, 19, 99)},
}

func TestParseIP(t *testing.T) {
	for i := 0; i < len(parseiptests); i++ {
		tt := parseiptests[i]
		if out := ParseIP(tt.in); !isEqual(out, tt.out) {
			t.Errorf("ParseIP(%#q) = %v, want %v", tt.in, out, tt.out)
		}
	}
}

type ipStringTest struct {
	in  IP
	out string
}

var ipstringtests = []ipStringTest{
	// cf. RFC 5952 (A Recommendation for IPv6 Address Text Representation)
	ipStringTest{IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0,
		0, 0, 0x1, 0x23, 0, 0x12, 0, 0x1},
		"2001:db8::123:12:1"},
	ipStringTest{IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0x1},
		"2001:db8::1"},
	ipStringTest{IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0x1,
		0, 0, 0, 0x1, 0, 0, 0, 0x1},
		"2001:db8:0:1:0:1:0:1"},
	ipStringTest{IP{0x20, 0x1, 0xd, 0xb8, 0, 0x1, 0, 0,
		0, 0x1, 0, 0, 0, 0x1, 0, 0},
		"2001:db8:1:0:1:0:1:0"},
	ipStringTest{IP{0x20, 0x1, 0, 0, 0, 0, 0, 0,
		0, 0x1, 0, 0, 0, 0, 0, 0x1},
		"2001::1:0:0:1"},
	ipStringTest{IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0,
		0, 0x1, 0, 0, 0, 0, 0, 0},
		"2001:db8:0:0:1::"},
	ipStringTest{IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0,
		0, 0x1, 0, 0, 0, 0, 0, 0x1},
		"2001:db8::1:0:0:1"},
	ipStringTest{IP{0x20, 0x1, 0xD, 0xB8, 0, 0, 0, 0,
		0, 0xA, 0, 0xB, 0, 0xC, 0, 0xD},
		"2001:db8::a:b:c:d"},
}

func TestIPString(t *testing.T) {
	for i := 0; i < len(ipstringtests); i++ {
		tt := ipstringtests[i]
		if out := tt.in.String(); out != tt.out {
			t.Errorf("IP.String(%v) = %#q, want %#q", tt.in, out, tt.out)
		}
	}
}
<|endoftext|>


<commit_before>package migration

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path"
	"strconv"
	"strings"

	helm_2to3 "github.com/helm/helm-2to3/pkg/v3"
	"helm.sh/helm/v3/pkg/action"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/helm/pkg/helm"

	v1 "github.com/sapcc/kubernikus/pkg/apis/kubernikus/v1"
	"github.com/sapcc/kubernikus/pkg/controller/config"
	"github.com/sapcc/kubernikus/pkg/util"
	helm_util "github.com/sapcc/kubernikus/pkg/util/helm"
	"github.com/sapcc/kubernikus/pkg/version"
)

func Helm2to3(rawKluster []byte, current *v1.Kluster, clients config.Clients, factories config.Factories) error {
	return migrateHelmReleases(current, clients)
}

func migrateHelmReleases(kluster *v1.Kluster, clients config.Clients) error {
	klusterSecret, err := util.KlusterSecret(clients.Kubernetes, kluster)
	if err != nil {
		return err
	}
	accessMode, err := util.PVAccessMode(clients.Kubernetes, nil)
	if err != nil {
		return err
	}
	// fetching the pullRegion from the kluster secret causes foo in qa-de-1 as
	// the option not provided via cli defaults to eu-de-1
	pullRegion := klusterSecret.Region
	if strings.HasPrefix(pullRegion, "qa-de") {
		pullRegion = "eu-de-1"
	}
	chartsPath := path.Join("charts", "images.yaml")
	if _, err := os.Stat(chartsPath); errors.Is(err, os.ErrNotExist) {
		chartsPath = "/etc/kubernikus/charts/images.yaml"
	}
	imageRegistry, err := version.NewImageRegistry(chartsPath, pullRegion)
	if err != nil {
		return err
	}
	// Implements `helm2to3 convert` roughly
	// https://github.com/helm/helm-2to3/blob/927e49f49fb04a562a3e14d9ada073ca61d21e7c/cmd/convert.go#L106
	versions2, err := getHelm2ReleaseVersions(kluster.Name, clients)
	if err != nil {
		return err
	}
	client2 := clients.Helm
	client3 := clients.Helm3
	for _, version2 := range versions2 {
		rsp, err := client2.ReleaseContent(kluster.Name, helm.ContentReleaseVersion(int32(version2)))
		if err != nil {
			return err
		}
		release3, err := helm_2to3.CreateRelease(rsp.Release)
		if err != nil {
			return err
		}
		err = client3.Releases.Create(release3)
		if err != nil {
			return err
		}
		values, err := helm_util.KlusterToHelmValues(kluster, klusterSecret, kluster.Spec.Version, imageRegistry, accessMode)
		if err != nil {
			return err
		}
		upgrade := action.NewUpgrade(client3)
		_, err = upgrade.Run(release3.Name, release3.Chart, values)
		if err != nil {
			return err
		}
	}
	return nil
}

func getHelm2ReleaseVersions(releaseName string, clients config.Clients) ([]int, error) {
	configMaps, err := clients.Kubernetes.CoreV1().ConfigMaps("kube-system").List(context.TODO(), meta_v1.ListOptions{
		LabelSelector: fmt.Sprintf("OWNER=TILLER,NAME=%s", releaseName),
	})
	if err != nil {
		return nil, err
	}
	versions := make([]int, 0)
	for _, configMap := range configMaps.Items {
		versionStr := configMap.Labels["VERSION"]
		version, err := strconv.Atoi(versionStr)
		if err != nil {
			return nil, err
		}
		versions = append(versions, version)
	}
	return versions, err
}
<commit_msg>Don't create releases for all old version (#664)<commit_after>package migration

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"

	helm_2to3 "github.com/helm/helm-2to3/pkg/v3"
	"helm.sh/helm/v3/pkg/action"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/helm/pkg/helm"

	v1 "github.com/sapcc/kubernikus/pkg/apis/kubernikus/v1"
	"github.com/sapcc/kubernikus/pkg/controller/config"
	"github.com/sapcc/kubernikus/pkg/util"
	helm_util "github.com/sapcc/kubernikus/pkg/util/helm"
	"github.com/sapcc/kubernikus/pkg/version"
)

func Helm2to3(rawKluster []byte, current *v1.Kluster, clients config.Clients, factories config.Factories) error {
	return migrateHelmReleases(current, clients)
}

func migrateHelmReleases(kluster *v1.Kluster, clients config.Clients) error {
	klusterSecret, err := util.KlusterSecret(clients.Kubernetes, kluster)
	if err != nil {
		return err
	}
	accessMode, err := util.PVAccessMode(clients.Kubernetes, nil)
	if err != nil {
		return err
	}
	// fetching the pullRegion from the kluster secret causes foo in qa-de-1 as
	// the option not provided via cli defaults to eu-de-1
	pullRegion := klusterSecret.Region
	if strings.HasPrefix(pullRegion, "qa-de") {
		pullRegion = "eu-de-1"
	}
	chartsPath := path.Join("charts", "images.yaml")
	if _, err := os.Stat(chartsPath); errors.Is(err, os.ErrNotExist) {
		chartsPath = "/etc/kubernikus/charts/images.yaml"
	}
	imageRegistry, err := version.NewImageRegistry(chartsPath, pullRegion)
	if err != nil {
		return err
	}
	// Implements `helm2to3 convert` roughly
	// https://github.com/helm/helm-2to3/blob/927e49f49fb04a562a3e14d9ada073ca61d21e7c/cmd/convert.go#L106
	versions2, err := getHelm2ReleaseVersions(kluster.Name, clients)
	if err != nil {
		return err
	}
	sort.Ints(versions2)
	client2 := clients.Helm
	client3 := clients.Helm3
	latestVersion2 := versions2[len(versions2)-1]
	rsp, err := client2.ReleaseContent(kluster.Name, helm.ContentReleaseVersion(int32(latestVersion2)))
	if err != nil {
		return err
	}
	release3, err := helm_2to3.CreateRelease(rsp.Release)
	if err != nil {
		return err
	}
	err = client3.Releases.Create(release3)
	if err != nil {
		return err
	}
	values, err := helm_util.KlusterToHelmValues(kluster, klusterSecret, kluster.Spec.Version, imageRegistry, accessMode)
	if err != nil {
		return err
	}
	upgrade := action.NewUpgrade(client3)
	_, err = upgrade.Run(release3.Name, release3.Chart, values)
	if err != nil {
		return err
	}
	return nil
}

func getHelm2ReleaseVersions(releaseName string, clients config.Clients) ([]int, error) {
	configMaps, err := clients.Kubernetes.CoreV1().ConfigMaps("kube-system").List(context.TODO(), meta_v1.ListOptions{
		LabelSelector: fmt.Sprintf("OWNER=TILLER,NAME=%s", releaseName),
	})
	if err != nil {
		return nil, err
	}
	versions := make([]int, 0)
	for _, configMap := range configMaps.Items {
		versionStr := configMap.Labels["VERSION"]
		version, err := strconv.Atoi(versionStr)
		if err != nil {
			return nil, err
		}
		versions = append(versions, version)
	}
	return versions, err
}
<|endoftext|>


<commit_before>// Copyright 2018 The OpenPitrix Authors. All rights reserved.
// Use of this source code is governed by a Apache license
// that can be found in the LICENSE file.

package app

import (
	"context"
	"fmt"
	"sort"
	"time"

	"openpitrix.io/openpitrix/pkg/constants"
	"openpitrix.io/openpitrix/pkg/db"
	"openpitrix.io/openpitrix/pkg/logger"
	"openpitrix.io/openpitrix/pkg/models"
	"openpitrix.io/openpitrix/pkg/pb"
	"openpitrix.io/openpitrix/pkg/pi"
	"openpitrix.io/openpitrix/pkg/repoiface"
	"openpitrix.io/openpitrix/pkg/repoiface/wrapper"
	"openpitrix.io/openpitrix/pkg/sender"
	"openpitrix.io/openpitrix/pkg/service/category/categoryutil"
	"openpitrix.io/openpitrix/pkg/util/stringutil"
)

type repoProxy struct {
	repo *pb.Repo
}

func newRepoProxy(repo *pb.Repo) *repoProxy {
	vp := &repoProxy{}
	vp.repo = repo
	return vp
}

func (rp *repoProxy) deleteAppVersions(ctx context.Context) error {
	repoId := rp.repo.RepoId.GetValue()
	err := clearApps(ctx, repoId, []string{})
	if err != nil {
		return err
	}
	return clearRepoAppVersions(ctx, repoId, []string{})
}

func (rp *repoProxy) SyncRepo(ctx context.Context) error {
	repo := rp.repo
	if repo.Status.GetValue() == constants.StatusDeleted {
		return rp.deleteAppVersions(ctx)
	}
	reader, err := repoiface.NewReader(ctx, repo)
	if err != nil {
		return err
	}
	indexFile, err := reader.GetIndex(ctx)
	if err != nil {
		return err
	}
	var appIds []string
	for appName, appVersions := range indexFile.GetEntries() {
		var appId string
		logger.Debug(ctx, "Start index app [%s]", appName)
		logger.Debug(ctx, "App [%s] has [%d] versions", appName, appVersions.Len())
		if len(appVersions) == 0 {
			return fmt.Errorf("failed to sync app [%s], no versions", appName)
		}
		sort.Sort(appVersions)
		appId, err := rp.syncAppInfo(ctx, appVersions[0])
		if err != nil {
			logger.Error(ctx, "Failed to sync app [%s] to app info", appName)
			return err
		}
		logger.Info(ctx, "Sync [%s] to app [%s] success", appName, appId)
		var versionIds []string
		for _, appVersion := range appVersions {
			var versionId string
			versionId, err = rp.syncAppVersionInfo(ctx, appId, appVersion)
			if err != nil {
				logger.Error(ctx, "Failed to sync app version [%s] to app version", appVersion.GetAppVersion())
				return err
			}
			logger.Debug(ctx, "App version [%s] sync to app version [%s]", appVersion.GetVersion(), versionId)
			versionIds = append(versionIds, versionId)
		}
		err = clearAppVersions(ctx, appId, versionIds)
		if err != nil {
			return err
		}
		err = resortAppVersions(ctx, appId)
		if err != nil {
			return err
		}
		err = syncAppStatus(ctx, appId)
		if err != nil {
			return err
		}
		appIds = append(appIds, appId)
	}
	err = clearApps(ctx, rp.repo.RepoId.GetValue(), appIds)
	if err != nil {
		return err
	}
	return nil
}

func (rp *repoProxy) syncAppInfo(ctx context.Context, appIface wrapper.VersionInterface) (string, error) {
	chartName := appIface.GetName()
	repoId := rp.repo.GetRepoId().GetValue()

	var enabledCategoryIds []string
	var disabledCategoryIds []string

	for _, c := range rp.repo.GetCategorySet() {
		switch c.Status.GetValue() {
		case constants.StatusEnabled:
			enabledCategoryIds = append(enabledCategoryIds, c.CategoryId.GetValue())
		case constants.StatusDisabled:
			disabledCategoryIds = append(disabledCategoryIds, c.CategoryId.GetValue())
		}
	}
	if len(enabledCategoryIds) == 0 {
		enabledCategoryIds = append(enabledCategoryIds, models.UncategorizedId)
	}

	var appId string
	var app = &models.App{}
	err := pi.Global().DB(ctx).
		Select(models.AppColumns...).
		From(constants.TableApp).
		Where(db.Eq(constants.ColumnActive, false)).
		Where(db.Eq(constants.ColumnRepoId, repoId)).
		Where(db.Eq(constants.ColumnChartName, chartName)).
		LoadOne(&app)
	if err != nil {
		// insert new
		if err != db.ErrNotFound {
			return appId, err
		}
		app = models.NewApp(
			chartName,
			sender.OwnerPath(rp.repo.GetOwnerPath().GetValue()),
		)
		app.RepoId = repoId
		app.ChartName = chartName
		app.Description = appIface.GetDescription()
		app.Home = appIface.GetHome()
		app.Icon = appIface.GetIcon()
		app.Screenshots = appIface.GetScreenshots()
		app.Maintainers = appIface.GetMaintainers()
		app.Keywords = appIface.GetKeywords()
		app.Sources = appIface.GetSources()

		_, err = pi.Global().DB(ctx).
			InsertInto(constants.TableApp).
			Record(app).
			Exec()
		if err != nil {
			return appId, err
		}

		err = categoryutil.SyncResourceCategories(
			ctx,
			pi.Global().DB(ctx),
			app.AppId,
			enabledCategoryIds,
		)
		if err != nil {
			return appId, err
		}

		return app.AppId, err
	}

	appId = app.AppId
	var updateAttr = make(map[string]interface{})
	if app.Description != appIface.GetDescription() {
		updateAttr[constants.ColumnDescription] = appIface.GetDescription()
	}
	if app.Home != appIface.GetHome() {
		updateAttr[constants.ColumnHome] = appIface.GetHome()
	}
	if app.Icon != appIface.GetIcon() {
		updateAttr[constants.ColumnIcon] = appIface.GetIcon()
	}
	if app.Screenshots != appIface.GetScreenshots() {
		updateAttr[constants.ColumnScreenshots] = appIface.GetScreenshots()
	}
	if app.Maintainers != appIface.GetMaintainers() {
		updateAttr[constants.ColumnMaintainers] = appIface.GetMaintainers()
	}
	if app.Keywords != appIface.GetKeywords() {
		updateAttr[constants.ColumnKeywords] = appIface.GetKeywords()
	}
	if app.Sources != appIface.GetSources() {
		updateAttr[constants.ColumnSources] = appIface.GetSources()
	}
	if len(updateAttr) > 0 {
		_, err = pi.Global().DB(ctx).
			Update(constants.TableApp).
			SetMap(updateAttr).
			Set(constants.ColumnUpdateTime, time.Now()).
			Where(db.Eq(constants.ColumnAppId, appId)).
			Where(db.Eq(constants.ColumnActive, false)).
			Exec()
		if err != nil {
			return appId, err
		}
	}

	// update exists, only need sync categories
	appCategories, err := getAppCategories(ctx, appId)
	if err != nil {
		return appId, err
	}
	var categoryMap = make(map[string]bool)
	for _, c := range appCategories {
		categoryId := c.GetCategoryId().GetValue()
		// app follow repo's categories:
		// if repo *disable* some categories, app MUST *disable* it
		// if repo *enable*  some categories, app MUST *enable*  it
		if c.GetStatus().GetValue() == constants.StatusEnabled {
			if !stringutil.StringIn(categoryId, disabledCategoryIds) {
				categoryMap[categoryId] = true
			}
		}
	}
	for _, c := range enabledCategoryIds {
		categoryMap[c] = true
	}
	var categoryIds []string
	for c := range categoryMap {
		if c == models.UncategorizedId && len(categoryMap) > 1 {
			continue
		}
		categoryIds = append(categoryIds, c)
	}
	err = categoryutil.SyncResourceCategories(
		ctx,
		pi.Global().DB(ctx),
		appId,
		categoryIds,
	)
	return app.AppId, err
}

func (rp *repoProxy) syncAppVersionInfo(ctx context.Context, appId string, versionInterface wrapper.VersionInterface) (string, error) {
	versionName := versionInterface.GetVersionName()
	var appVersion = &models.AppVersion{}
	var versionId = ""
	var status = getAppDefaultStatus(rp.repo)
	if status == constants.StatusActive {
		defer func() {
			if versionId != "" {
				err := syncAppVersion(ctx, versionId)
				if err != nil {
					logger.Error(ctx, "Active app version [%s] failed: %+v", versionId, err)
				}
			}
		}()
	}
	err := pi.Global().DB(ctx).
		Select(models.AppVersionColumns...).
		From(constants.TableAppVersion).
		Where(db.Eq(constants.ColumnAppId, appId)).
		Where(db.Eq(constants.ColumnName, versionName)).
		Where(db.Eq(constants.ColumnActive, false)).
		LoadOne(&appVersion)
	if err != nil {
		if err != db.ErrNotFound {
			return versionId, err
		}
		// not found version, create new
		appVersion = models.NewAppVersion(
			appId,
			versionName,
			versionInterface.GetDescription(),
			sender.OwnerPath(rp.repo.GetOwnerPath().GetValue()),
		)

		appVersion.PackageName = versionInterface.GetPackageName()
		appVersion.Status = status

		_, err = pi.Global().DB(ctx).
			InsertInto(constants.TableAppVersion).
			Record(appVersion).
			Exec()
		if err != nil {
			return versionId, err
		}
	}
	// update exists
	versionId = appVersion.VersionId
	var updateAttr = make(map[string]interface{})

	if appVersion.Status != rp.repo.GetAppDefaultStatus().GetValue() {
		updateAttr[constants.ColumnStatus] = getAppVersionStatus(
			status,
			appVersion.Status,
		)
	}

	if appVersion.PackageName != versionInterface.GetPackageName() {
		updateAttr[constants.ColumnPackageName] = versionInterface.GetPackageName()
	}
	if len(updateAttr) == 0 {
		return versionId, nil
	}
	_, err = pi.Global().DB(ctx).
		Update(constants.TableAppVersion).
		SetMap(updateAttr).
		Set(constants.ColumnUpdateTime, time.Now()).
		Where(db.Eq(constants.ColumnVersionId, versionId)).
		Where(db.Eq(constants.ColumnActive, false)).
		Exec()
	return versionId, err
}
<commit_msg>Bugfix: fix SyncRepo cannot sync repo type<commit_after>// Copyright 2018 The OpenPitrix Authors. All rights reserved.
// Use of this source code is governed by a Apache license
// that can be found in the LICENSE file.

package app

import (
	"context"
	"fmt"
	"sort"
	"time"

	"openpitrix.io/openpitrix/pkg/constants"
	"openpitrix.io/openpitrix/pkg/db"
	"openpitrix.io/openpitrix/pkg/logger"
	"openpitrix.io/openpitrix/pkg/models"
	"openpitrix.io/openpitrix/pkg/pb"
	"openpitrix.io/openpitrix/pkg/pi"
	"openpitrix.io/openpitrix/pkg/repoiface"
	"openpitrix.io/openpitrix/pkg/repoiface/wrapper"
	"openpitrix.io/openpitrix/pkg/sender"
	"openpitrix.io/openpitrix/pkg/service/category/categoryutil"
	"openpitrix.io/openpitrix/pkg/util/stringutil"
)

type repoProxy struct {
	repo *pb.Repo
}

func newRepoProxy(repo *pb.Repo) *repoProxy {
	vp := &repoProxy{}
	vp.repo = repo
	return vp
}

func (rp *repoProxy) deleteAppVersions(ctx context.Context) error {
	repoId := rp.repo.RepoId.GetValue()
	err := clearApps(ctx, repoId, []string{})
	if err != nil {
		return err
	}
	return clearRepoAppVersions(ctx, repoId, []string{})
}

func (rp *repoProxy) SyncRepo(ctx context.Context) error {
	repo := rp.repo
	if repo.Status.GetValue() == constants.StatusDeleted {
		return rp.deleteAppVersions(ctx)
	}
	reader, err := repoiface.NewReader(ctx, repo)
	if err != nil {
		return err
	}
	indexFile, err := reader.GetIndex(ctx)
	if err != nil {
		return err
	}
	var appIds []string
	for appName, appVersions := range indexFile.GetEntries() {
		var appId string
		logger.Debug(ctx, "Start index app [%s]", appName)
		logger.Debug(ctx, "App [%s] has [%d] versions", appName, appVersions.Len())
		if len(appVersions) == 0 {
			return fmt.Errorf("failed to sync app [%s], no versions", appName)
		}
		sort.Sort(appVersions)
		appId, err := rp.syncAppInfo(ctx, appVersions[0])
		if err != nil {
			logger.Error(ctx, "Failed to sync app [%s] to app info", appName)
			return err
		}
		logger.Info(ctx, "Sync [%s] to app [%s] success", appName, appId)
		var versionIds []string
		for _, appVersion := range appVersions {
			var versionId string
			versionId, err = rp.syncAppVersionInfo(ctx, appId, appVersion)
			if err != nil {
				logger.Error(ctx, "Failed to sync app version [%s] to app version", appVersion.GetAppVersion())
				return err
			}
			logger.Debug(ctx, "App version [%s] sync to app version [%s]", appVersion.GetVersion(), versionId)
			versionIds = append(versionIds, versionId)
		}
		err = clearAppVersions(ctx, appId, versionIds)
		if err != nil {
			return err
		}
		err = resortAppVersions(ctx, appId)
		if err != nil {
			return err
		}
		err = syncAppStatus(ctx, appId)
		if err != nil {
			return err
		}
		appIds = append(appIds, appId)
	}
	err = clearApps(ctx, rp.repo.RepoId.GetValue(), appIds)
	if err != nil {
		return err
	}
	return nil
}

func (rp *repoProxy) syncAppInfo(ctx context.Context, appIface wrapper.VersionInterface) (string, error) {
	chartName := appIface.GetName()
	repoId := rp.repo.GetRepoId().GetValue()

	var enabledCategoryIds []string
	var disabledCategoryIds []string

	for _, c := range rp.repo.GetCategorySet() {
		switch c.Status.GetValue() {
		case constants.StatusEnabled:
			enabledCategoryIds = append(enabledCategoryIds, c.CategoryId.GetValue())
		case constants.StatusDisabled:
			disabledCategoryIds = append(disabledCategoryIds, c.CategoryId.GetValue())
		}
	}
	if len(enabledCategoryIds) == 0 {
		enabledCategoryIds = append(enabledCategoryIds, models.UncategorizedId)
	}

	var appId string
	var app = &models.App{}
	err := pi.Global().DB(ctx).
		Select(models.AppColumns...).
		From(constants.TableApp).
		Where(db.Eq(constants.ColumnActive, false)).
		Where(db.Eq(constants.ColumnRepoId, repoId)).
		Where(db.Eq(constants.ColumnChartName, chartName)).
		LoadOne(&app)
	if err != nil {
		// insert new
		if err != db.ErrNotFound {
			return appId, err
		}
		app = models.NewApp(
			chartName,
			sender.OwnerPath(rp.repo.GetOwnerPath().GetValue()),
		)
		app.RepoId = repoId
		app.ChartName = chartName
		app.Description = appIface.GetDescription()
		app.Home = appIface.GetHome()
		app.Icon = appIface.GetIcon()
		app.Screenshots = appIface.GetScreenshots()
		app.Maintainers = appIface.GetMaintainers()
		app.Keywords = appIface.GetKeywords()
		app.Sources = appIface.GetSources()

		_, err = pi.Global().DB(ctx).
			InsertInto(constants.TableApp).
			Record(app).
			Exec()
		if err != nil {
			return appId, err
		}

		err = categoryutil.SyncResourceCategories(
			ctx,
			pi.Global().DB(ctx),
			app.AppId,
			enabledCategoryIds,
		)
		if err != nil {
			return appId, err
		}

		return app.AppId, err
	}

	appId = app.AppId
	var updateAttr = make(map[string]interface{})
	if app.Description != appIface.GetDescription() {
		updateAttr[constants.ColumnDescription] = appIface.GetDescription()
	}
	if app.Home != appIface.GetHome() {
		updateAttr[constants.ColumnHome] = appIface.GetHome()
	}
	if app.Icon != appIface.GetIcon() {
		updateAttr[constants.ColumnIcon] = appIface.GetIcon()
	}
	if app.Screenshots != appIface.GetScreenshots() {
		updateAttr[constants.ColumnScreenshots] = appIface.GetScreenshots()
	}
	if app.Maintainers != appIface.GetMaintainers() {
		updateAttr[constants.ColumnMaintainers] = appIface.GetMaintainers()
	}
	if app.Keywords != appIface.GetKeywords() {
		updateAttr[constants.ColumnKeywords] = appIface.GetKeywords()
	}
	if app.Sources != appIface.GetSources() {
		updateAttr[constants.ColumnSources] = appIface.GetSources()
	}
	if len(updateAttr) > 0 {
		_, err = pi.Global().DB(ctx).
			Update(constants.TableApp).
			SetMap(updateAttr).
			Set(constants.ColumnUpdateTime, time.Now()).
			Where(db.Eq(constants.ColumnAppId, appId)).
			Where(db.Eq(constants.ColumnActive, false)).
			Exec()
		if err != nil {
			return appId, err
		}
	}

	// update exists, only need sync categories
	appCategories, err := getAppCategories(ctx, appId)
	if err != nil {
		return appId, err
	}
	var categoryMap = make(map[string]bool)
	for _, c := range appCategories {
		categoryId := c.GetCategoryId().GetValue()
		// app follow repo's categories:
		// if repo *disable* some categories, app MUST *disable* it
		// if repo *enable*  some categories, app MUST *enable*  it
		if c.GetStatus().GetValue() == constants.StatusEnabled {
			if !stringutil.StringIn(categoryId, disabledCategoryIds) {
				categoryMap[categoryId] = true
			}
		}
	}
	for _, c := range enabledCategoryIds {
		categoryMap[c] = true
	}
	var categoryIds []string
	for c := range categoryMap {
		if c == models.UncategorizedId && len(categoryMap) > 1 {
			continue
		}
		categoryIds = append(categoryIds, c)
	}
	err = categoryutil.SyncResourceCategories(
		ctx,
		pi.Global().DB(ctx),
		appId,
		categoryIds,
	)
	return app.AppId, err
}

func (rp *repoProxy) syncAppVersionInfo(ctx context.Context, appId string, versionInterface wrapper.VersionInterface) (string, error) {
	versionName := versionInterface.GetVersionName()
	var appVersion = &models.AppVersion{}
	var versionId = ""
	var status = getAppDefaultStatus(rp.repo)
	if status == constants.StatusActive {
		defer func() {
			if versionId != "" {
				err := syncAppVersion(ctx, versionId)
				if err != nil {
					logger.Error(ctx, "Active app version [%s] failed: %+v", versionId, err)
				}
			}
		}()
	}
	err := pi.Global().DB(ctx).
		Select(models.AppVersionColumns...).
		From(constants.TableAppVersion).
		Where(db.Eq(constants.ColumnAppId, appId)).
		Where(db.Eq(constants.ColumnName, versionName)).
		Where(db.Eq(constants.ColumnActive, false)).
		LoadOne(&appVersion)
	if err != nil {
		if err != db.ErrNotFound {
			return versionId, err
		}
		// not found version, create new
		appVersion = models.NewAppVersion(
			appId,
			versionName,
			versionInterface.GetDescription(),
			sender.OwnerPath(rp.repo.GetOwnerPath().GetValue()),
		)

		appVersion.PackageName = versionInterface.GetPackageName()
		appVersion.Status = status
		appVersion.Type = rp.repo.Type.GetValue()

		_, err = pi.Global().DB(ctx).
			InsertInto(constants.TableAppVersion).
			Record(appVersion).
			Exec()
		if err != nil {
			return versionId, err
		}
	}
	// update exists
	versionId = appVersion.VersionId
	var updateAttr = make(map[string]interface{})

	if appVersion.Status != rp.repo.GetAppDefaultStatus().GetValue() {
		updateAttr[constants.ColumnStatus] = getAppVersionStatus(
			status,
			appVersion.Status,
		)
	}

	if appVersion.PackageName != versionInterface.GetPackageName() {
		updateAttr[constants.ColumnPackageName] = versionInterface.GetPackageName()
	}
	if len(updateAttr) == 0 {
		return versionId, nil
	}
	_, err = pi.Global().DB(ctx).
		Update(constants.TableAppVersion).
		SetMap(updateAttr).
		Set(constants.ColumnUpdateTime, time.Now()).
		Where(db.Eq(constants.ColumnVersionId, versionId)).
		Where(db.Eq(constants.ColumnActive, false)).
		Exec()
	return versionId, err
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kaniko

import (
	"context"
	"fmt"
	"io"

	cstorage "cloud.google.com/go/storage"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v1alpha2"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func RunKanikoBuild(ctx context.Context, out io.Writer, artifact *v1alpha2.Artifact, cfg *v1alpha2.KanikoBuild) (string, error) {
	dockerfilePath := artifact.DockerArtifact.DockerfilePath

	initialTag := util.RandomID()
	tarName := fmt.Sprintf("context-%s.tar.gz", initialTag)
	if err := docker.UploadContextToGCS(ctx, artifact.Workspace, dockerfilePath, cfg.GCSBucket, tarName); err != nil {
		return "", errors.Wrap(err, "uploading tar to gcs")
	}
	defer gcsDelete(ctx, cfg.GCSBucket, tarName)

	client, err := kubernetes.GetClientset()
	if err != nil {
		return "", errors.Wrap(err, "")
	}

	imageList := kubernetes.NewImageList()
	imageList.Add(constants.DefaultKanikoImage)

	logger := kubernetes.NewLogAggregator(out, imageList, kubernetes.NewColorPicker([]*v1alpha2.Artifact{artifact}))
	if err := logger.Start(ctx); err != nil {
		return "", errors.Wrap(err, "starting log streamer")
	}

	imageDst := fmt.Sprintf("%s:%s", artifact.ImageName, initialTag)
	p, err := client.CoreV1().Pods(cfg.Namespace).Create(&v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "kaniko",
			Labels:       map[string]string{"skaffold-kaniko": "skaffold-kaniko"},
			Namespace:    cfg.Namespace,
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:            "kaniko",
					Image:           constants.DefaultKanikoImage,
					ImagePullPolicy: v1.PullIfNotPresent,
					Args: []string{
						fmt.Sprintf("--dockerfile=%s", dockerfilePath),
						fmt.Sprintf("--context=gs://%s/%s", cfg.GCSBucket, tarName),
						fmt.Sprintf("--destination=%s", imageDst),
						fmt.Sprintf("-v=%s", logrus.GetLevel().String()),
					},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      constants.DefaultKanikoSecretName,
							MountPath: "/secret",
						},
					},
					Env: []v1.EnvVar{
						{
							Name:  "GOOGLE_APPLICATION_CREDENTIALS",
							Value: "/secret/kaniko-secret",
						},
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: constants.DefaultKanikoSecretName,
					VolumeSource: v1.VolumeSource{
						Secret: &v1.SecretVolumeSource{
							SecretName: cfg.PullSecretName,
						},
					},
				},
			},
			RestartPolicy: v1.RestartPolicyNever,
		},
	})
	if err != nil {
		return "", errors.Wrap(err, "creating kaniko pod")
	}

	defer func() {
		imageList.Remove(constants.DefaultKanikoImage)
		if err := client.CoreV1().Pods(cfg.Namespace).Delete(p.Name, &metav1.DeleteOptions{
			GracePeriodSeconds: new(int64),
		}); err != nil {
			logrus.Fatalf("deleting pod: %s", err)
		}
	}()

	if err := kubernetes.WaitForPodComplete(client.CoreV1().Pods(cfg.Namespace), p.Name); err != nil {
		return "", errors.Wrap(err, "waiting for pod to complete")
	}

	return imageDst, nil
}

func gcsDelete(ctx context.Context, bucket, path string) error {
	c, err := cstorage.NewClient(ctx)
	if err != nil {
		return err
	}
	defer c.Close()

	return c.Bucket(bucket).Object(path).Delete(ctx)
}
<commit_msg>Pass buildArgs to Kaniko<commit_after>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kaniko

import (
	"context"
	"fmt"
	"io"

	cstorage "cloud.google.com/go/storage"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v1alpha2"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func RunKanikoBuild(ctx context.Context, out io.Writer, artifact *v1alpha2.Artifact, cfg *v1alpha2.KanikoBuild) (string, error) {
	dockerfilePath := artifact.DockerArtifact.DockerfilePath

	initialTag := util.RandomID()
	tarName := fmt.Sprintf("context-%s.tar.gz", initialTag)
	if err := docker.UploadContextToGCS(ctx, artifact.Workspace, dockerfilePath, cfg.GCSBucket, tarName); err != nil {
		return "", errors.Wrap(err, "uploading tar to gcs")
	}
	defer gcsDelete(ctx, cfg.GCSBucket, tarName)

	client, err := kubernetes.GetClientset()
	if err != nil {
		return "", errors.Wrap(err, "")
	}

	imageList := kubernetes.NewImageList()
	imageList.Add(constants.DefaultKanikoImage)

	logger := kubernetes.NewLogAggregator(out, imageList, kubernetes.NewColorPicker([]*v1alpha2.Artifact{artifact}))
	if err := logger.Start(ctx); err != nil {
		return "", errors.Wrap(err, "starting log streamer")
	}

	imageDst := fmt.Sprintf("%s:%s", artifact.ImageName, initialTag)
	p, err := client.CoreV1().Pods(cfg.Namespace).Create(&v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "kaniko",
			Labels:       map[string]string{"skaffold-kaniko": "skaffold-kaniko"},
			Namespace:    cfg.Namespace,
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:            "kaniko",
					Image:           constants.DefaultKanikoImage,
					ImagePullPolicy: v1.PullIfNotPresent,
					Args: addBuildArgs([]string{
						fmt.Sprintf("--dockerfile=%s", dockerfilePath),
						fmt.Sprintf("--context=gs://%s/%s", cfg.GCSBucket, tarName),
						fmt.Sprintf("--destination=%s", imageDst),
						fmt.Sprintf("-v=%s", logrus.GetLevel().String()),
					}, artifact),
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      constants.DefaultKanikoSecretName,
							MountPath: "/secret",
						},
					},
					Env: []v1.EnvVar{
						{
							Name:  "GOOGLE_APPLICATION_CREDENTIALS",
							Value: "/secret/kaniko-secret",
						},
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: constants.DefaultKanikoSecretName,
					VolumeSource: v1.VolumeSource{
						Secret: &v1.SecretVolumeSource{
							SecretName: cfg.PullSecretName,
						},
					},
				},
			},
			RestartPolicy: v1.RestartPolicyNever,
		},
	})
	if err != nil {
		return "", errors.Wrap(err, "creating kaniko pod")
	}

	defer func() {
		imageList.Remove(constants.DefaultKanikoImage)
		if err := client.CoreV1().Pods(cfg.Namespace).Delete(p.Name, &metav1.DeleteOptions{
			GracePeriodSeconds: new(int64),
		}); err != nil {
			logrus.Fatalf("deleting pod: %s", err)
		}
	}()

	if err := kubernetes.WaitForPodComplete(client.CoreV1().Pods(cfg.Namespace), p.Name); err != nil {
		return "", errors.Wrap(err, "waiting for pod to complete")
	}

	return imageDst, nil
}

func gcsDelete(ctx context.Context, bucket, path string) error {
	c, err := cstorage.NewClient(ctx)
	if err != nil {
		return err
	}
	defer c.Close()

	return c.Bucket(bucket).Object(path).Delete(ctx)
}

func addBuildArgs(args []string, artifact *v1alpha2.Artifact) []string {
	if artifact.DockerArtifact == nil {
		return args
	}

	if artifact.DockerArtifact.BuildArgs == nil || len(artifact.DockerArtifact.BuildArgs) == 0 {
		return args
	}

	withBuildArgs := make([]string, len(args)+len(artifact.DockerArtifact.BuildArgs))
	copy(withBuildArgs, args)

	i := len(args)
	for k, v := range artifact.DockerArtifact.BuildArgs {
		withBuildArgs[i] = fmt.Sprintf("--build-arg=%s=%s", k, *v)
		i = i + 1
	}

	return withBuildArgs
}
<|endoftext|>