<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/google/syzkaller/dashboard/dashapi"
	"github.com/google/syzkaller/pkg/config"
	"github.com/google/syzkaller/pkg/csource"
	"github.com/google/syzkaller/pkg/git"
	"github.com/google/syzkaller/pkg/kernel"
	. "github.com/google/syzkaller/pkg/log"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/pkg/report"
	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/syz-manager/mgrconfig"
	"github.com/google/syzkaller/vm"
)

type JobProcessor struct {
	managers []*Manager
	dash     *dashapi.Dashboard
}

func newJobProcessor(cfg *Config, managers []*Manager) *JobProcessor {
	jp := &JobProcessor{
		managers: managers,
	}
	if cfg.Dashboard_Addr != "" && cfg.Dashboard_Client != "" {
		jp.dash = dashapi.New(cfg.Dashboard_Client, cfg.Dashboard_Addr, cfg.Dashboard_Key)
	}
	return jp
}

func (jp *JobProcessor) loop(stop chan struct{}) {
	if jp.dash == nil {
		return
	}
	ticker := time.NewTicker(time.Minute)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			jp.poll()
		case <-stop:
			Logf(0, "job loop stopped")
			return
		}
	}
}

func (jp *JobProcessor) poll() {
	var names []string
	for _, mgr := range jp.managers {
		names = append(names, mgr.name)
	}
	req, err := jp.dash.JobPoll(names)
	if err != nil {
		Logf(0, "failed to poll jobs: %v", err)
		return
	}
	if req.ID == "" {
		return
	}
	var mgr *Manager
	for _, m := range jp.managers {
		if m.name == req.Manager {
			mgr = m
			break
		}
	}
	if mgr == nil {
		Logf(0, "got job for unknown manager: %v", req.Manager)
		return
	}
	job := &Job{
		req: req,
		mgr: mgr,
	}
	Logf(0, "starting job %v for manager %v on %v/%v",
		req.ID, req.Manager, req.KernelRepo, req.KernelBranch)
	resp := job.process()
	Logf(0, "done job %v: commit %v, crash %q, error: %s",
		resp.ID, resp.Build.KernelCommit, resp.CrashTitle, resp.Error)
	if err := jp.dash.JobDone(resp); err != nil {
		Logf(0, "failed to mark job as done: %v", err)
		return
	}
}

type Job struct {
	req    *dashapi.JobPollResp
	resp   *dashapi.JobDoneReq
	mgr    *Manager
	mgrcfg *mgrconfig.Config
}

func (job *Job) process() *dashapi.JobDoneReq {
	req, mgr := job.req, job.mgr
	build := dashapi.Build{
		Manager:      mgr.name,
		ID:           req.ID,
		OS:           mgr.managercfg.TargetOS,
		Arch:         mgr.managercfg.TargetArch,
		VMArch:       mgr.managercfg.TargetVMArch,
		CompilerID:   mgr.compilerID,
		KernelRepo:   req.KernelRepo,
		KernelBranch: req.KernelBranch,
	}
	job.resp = &dashapi.JobDoneReq{
		ID:    req.ID,
		Build: build,
	}
	required := []struct {
		name string
		ok   bool
	}{
		{"kernel repository", req.KernelRepo != ""},
		{"kernel branch", req.KernelBranch != ""},
		{"kernel config", len(req.KernelConfig) != 0},
		{"syzkaller commit", req.SyzkallerCommit != ""},
		{"test patch", len(req.Patch) != 0},
		{"reproducer options", len(req.ReproOpts) != 0},
		{"reproducer program", len(req.ReproSyz) != 0},
	}
	for _, req := range required {
		if !req.ok {
			job.resp.Error = []byte(req.name + " is empty")
			return job.resp
		}
	}
	// TODO(dvyukov): this will only work for qemu/gce,
	// because e.g. adb requires unique device IDs and we can't use what
	// manager already uses. For qemu/gce this is also bad, because we
	// override resource limits specified in config (e.g. can OOM), but works.
	switch typ := mgr.managercfg.Type; typ {
	case "gce", "qemu":
	default:
		job.resp.Error = []byte(fmt.Sprintf("testing is not yet supported for %v machine type.", typ))
		return job.resp
	}
	if err := job.buildImage(); err != nil {
		job.resp.Error = []byte(err.Error())
		return job.resp
	}
	if err := job.test(); err != nil {
		job.resp.Error = []byte(err.Error())
		return job.resp
	}
	return job.resp
}

func (job *Job) buildImage() error {
	kernelBuildSem <- struct{}{}
	defer func() { <-kernelBuildSem }()
	req, resp, mgr := job.req, job.resp, job.mgr

	// TODO(dvyukov): build syzkaller on req.SyzkallerCommit and use it.
	// Newer syzkaller may not parse an old reproducer program.
	syzkallerCommit, _ := readTag(filepath.FromSlash("syzkaller/current/tag"))
	if syzkallerCommit == "" {
		return fmt.Errorf("failed to read syzkaller build tag")
	}
	resp.Build.SyzkallerCommit = syzkallerCommit

	dir := osutil.Abs(filepath.Join("jobs", mgr.managercfg.TargetOS))
	kernelDir := filepath.Join(dir, "kernel")
	if err := osutil.MkdirAll(kernelDir); err != nil {
		return fmt.Errorf("failed to create temp dir: %v", err)
	}
	imageDir := filepath.Join(dir, "image")
	os.RemoveAll(imageDir)
	if err := osutil.MkdirAll(imageDir); err != nil {
		return fmt.Errorf("failed to create temp dir: %v", err)
	}
	workDir := filepath.Join(dir, "workdir")
	os.RemoveAll(workDir)
	if err := osutil.MkdirAll(workDir); err != nil {
		return fmt.Errorf("failed to create temp dir: %v", err)
	}

	Logf(0, "job: fetching kernel...")
	kernelCommit, err := git.Checkout(kernelDir, req.KernelRepo, req.KernelBranch)
	if err != nil {
		return fmt.Errorf("failed to checkout kernel repo: %v", err)
	}
	resp.Build.KernelCommit = kernelCommit

	if err := git.Patch(kernelDir, req.Patch); err != nil {
		return err
	}

	Logf(0, "job: building kernel...")
	configFile := filepath.Join(dir, "kernel.config")
	if err := osutil.WriteFile(configFile, req.KernelConfig); err != nil {
		return fmt.Errorf("failed to write temp file: %v", err)
	}
	if err := kernel.Build(kernelDir, mgr.mgrcfg.Compiler, configFile); err != nil {
		return fmt.Errorf("kernel build failed: %v", err)
	}
	kernelConfig, err := ioutil.ReadFile(filepath.Join(kernelDir, ".config"))
	if err != nil {
		return fmt.Errorf("failed to read config file: %v", err)
	}
	resp.Build.KernelConfig = kernelConfig

	Logf(0, "job: creating image...")
	image := filepath.Join(imageDir, "image")
	key := filepath.Join(imageDir, "key")
	err = kernel.CreateImage(kernelDir, mgr.mgrcfg.Userspace,
		mgr.mgrcfg.Kernel_Cmdline, mgr.mgrcfg.Kernel_Sysctl, image, key)
	if err != nil {
		return fmt.Errorf("image build failed: %v", err)
	}
	// TODO(dvyukov): test that the image is good (boots and we can ssh into it).

	mgrcfg := new(mgrconfig.Config)
	*mgrcfg = *mgr.managercfg
	mgrcfg.Name += "-job"
	mgrcfg.Workdir = workDir
	mgrcfg.Vmlinux = filepath.Join(kernelDir, "vmlinux")
	mgrcfg.Kernel_Src = kernelDir
	mgrcfg.Syzkaller = filepath.FromSlash("syzkaller/current")
	mgrcfg.Image = image
	mgrcfg.Sshkey = key

	// Reload config to fill derived fields (ugly hack).
	cfgdata, err := config.SaveData(mgrcfg)
	if err != nil {
		return fmt.Errorf("failed to save manager config: %v", err)
	}
	if job.mgrcfg, err = mgrconfig.LoadData(cfgdata); err != nil {
		return fmt.Errorf("failed to reload manager config: %v", err)
	}
	return nil
}

func (job *Job) test() error {
	req, mgrcfg := job.req, job.mgrcfg

	Logf(0, "job: booting VM...")
	vmEnv := mgrconfig.CreateVMEnv(mgrcfg, false)
	vmPool, err := vm.Create(mgrcfg.Type, vmEnv)
	if err != nil {
		return fmt.Errorf("failed to create VM pool: %v", err)
	}
	inst, err := vmPool.Create(0)
	if err != nil {
		return fmt.Errorf("failed to create VM: %v", err)
	}
	defer inst.Close()

	Logf(0, "job: copying binaries...")
	execprogBin, err := inst.Copy(mgrcfg.SyzExecprogBin)
	if err != nil {
		return fmt.Errorf("failed to copy test binary to VM: %v", err)
	}
	executorBin, err := inst.Copy(mgrcfg.SyzExecutorBin)
	if err != nil {
		return fmt.Errorf("failed to copy test binary to VM: %v", err)
	}
	progFile := filepath.Join(mgrcfg.Workdir, "repro.prog")
	if err := osutil.WriteFile(progFile, req.ReproSyz); err != nil {
		return fmt.Errorf("failed to write temp file: %v", err)
	}
	vmProgFile, err := inst.Copy(progFile)
	if err != nil {
		return fmt.Errorf("failed to copy to VM: %v", err)
	}
	reporter, err := report.NewReporter(mgrcfg.TargetOS, mgrcfg.Kernel_Src,
		filepath.Dir(mgrcfg.Vmlinux), nil, mgrcfg.ParsedIgnores)
	if err != nil {
		return err
	}

	Logf(0, "job: testing syzkaller program...")
	testTime := 10 * time.Minute
	if len(req.ReproC) != 0 {
		testTime /= 2
	}
	// TODO(dvyukov): we currently ignore ReproOpts in req.
	// Proably we should test with both default opts and req.ReproOpts.
	cmdSyz := fmt.Sprintf("%v -executor %v -arch=%v -procs=%v -sandbox=%v -repeat=0 -cover=0 %v",
		execprogBin, executorBin, mgrcfg.TargetArch, mgrcfg.Procs, mgrcfg.Sandbox, vmProgFile)
	if crashed, err := job.testProgram(inst, cmdSyz, reporter, testTime); crashed || err != nil {
		return err
	}

	if len(req.ReproC) != 0 {
		Logf(0, "job: testing C program...")
		cFile := filepath.Join(mgrcfg.Workdir, "repro.c")
		if err := osutil.WriteFile(cFile, req.ReproC); err != nil {
			return fmt.Errorf("failed to write temp file: %v", err)
		}
		target, err := prog.GetTarget(mgrcfg.TargetOS, mgrcfg.TargetArch)
		if err != nil {
			return err
		}
		bin, err := csource.Build(target, "c", cFile)
		if err != nil {
			return err
		}
		vmBin, err := inst.Copy(bin)
		if err != nil {
			return fmt.Errorf("failed to copy test binary to VM: %v", err)
		}
		if crashed, err := job.testProgram(inst, vmBin, reporter, testTime); crashed || err != nil {
			return err
		}
	}
	return nil
}

func (job *Job) testProgram(inst *vm.Instance, command string, reporter report.Reporter,
	testTime time.Duration) (bool, error) {
	outc, errc, err := inst.Run(testTime, nil, command)
	if err != nil {
		return false, fmt.Errorf("failed to run binary in VM: %v", err)
	}
	title, report, output, crashed, _ := vm.MonitorExecution(outc, errc, reporter)
	if crashed {
		job.resp.CrashTitle = title
		job.resp.CrashReport = report
		job.resp.CrashLog = output
	}
	return crashed, nil
}
<commit_msg>syz-ci: pre-fill kernel/syzkaller commits in job build<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/google/syzkaller/dashboard/dashapi"
	"github.com/google/syzkaller/pkg/config"
	"github.com/google/syzkaller/pkg/csource"
	"github.com/google/syzkaller/pkg/git"
	"github.com/google/syzkaller/pkg/kernel"
	. "github.com/google/syzkaller/pkg/log"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/pkg/report"
	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/syz-manager/mgrconfig"
	"github.com/google/syzkaller/vm"
)

type JobProcessor struct {
	managers []*Manager
	dash     *dashapi.Dashboard
}

func newJobProcessor(cfg *Config, managers []*Manager) *JobProcessor {
	jp := &JobProcessor{
		managers: managers,
	}
	if cfg.Dashboard_Addr != "" && cfg.Dashboard_Client != "" {
		jp.dash = dashapi.New(cfg.Dashboard_Client, cfg.Dashboard_Addr, cfg.Dashboard_Key)
	}
	return jp
}

func (jp *JobProcessor) loop(stop chan struct{}) {
	if jp.dash == nil {
		return
	}
	ticker := time.NewTicker(time.Minute)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			jp.poll()
		case <-stop:
			Logf(0, "job loop stopped")
			return
		}
	}
}

func (jp *JobProcessor) poll() {
	var names []string
	for _, mgr := range jp.managers {
		names = append(names, mgr.name)
	}
	req, err := jp.dash.JobPoll(names)
	if err != nil {
		Logf(0, "failed to poll jobs: %v", err)
		return
	}
	if req.ID == "" {
		return
	}
	var mgr *Manager
	for _, m := range jp.managers {
		if m.name == req.Manager {
			mgr = m
			break
		}
	}
	if mgr == nil {
		Logf(0, "got job for unknown manager: %v", req.Manager)
		return
	}
	job := &Job{
		req: req,
		mgr: mgr,
	}
	Logf(0, "starting job %v for manager %v on %v/%v",
		req.ID, req.Manager, req.KernelRepo, req.KernelBranch)
	resp := job.process()
	Logf(0, "done job %v: commit %v, crash %q, error: %s",
		resp.ID, resp.Build.KernelCommit, resp.CrashTitle, resp.Error)
	if err := jp.dash.JobDone(resp); err != nil {
		Logf(0, "failed to mark job as done: %v", err)
		return
	}
}

type Job struct {
	req    *dashapi.JobPollResp
	resp   *dashapi.JobDoneReq
	mgr    *Manager
	mgrcfg *mgrconfig.Config
}

func (job *Job) process() *dashapi.JobDoneReq {
	req, mgr := job.req, job.mgr
	build := dashapi.Build{
		Manager:         mgr.name,
		ID:              req.ID,
		OS:              mgr.managercfg.TargetOS,
		Arch:            mgr.managercfg.TargetArch,
		VMArch:          mgr.managercfg.TargetVMArch,
		CompilerID:      mgr.compilerID,
		KernelRepo:      req.KernelRepo,
		KernelBranch:    req.KernelBranch,
		KernelCommit:    "[unknown]",
		SyzkallerCommit: "[unknown]",
	}
	job.resp = &dashapi.JobDoneReq{
		ID:    req.ID,
		Build: build,
	}
	required := []struct {
		name string
		ok   bool
	}{
		{"kernel repository", req.KernelRepo != ""},
		{"kernel branch", req.KernelBranch != ""},
		{"kernel config", len(req.KernelConfig) != 0},
		{"syzkaller commit", req.SyzkallerCommit != ""},
		{"test patch", len(req.Patch) != 0},
		{"reproducer options", len(req.ReproOpts) != 0},
		{"reproducer program", len(req.ReproSyz) != 0},
	}
	for _, req := range required {
		if !req.ok {
			job.resp.Error = []byte(req.name + " is empty")
			return job.resp
		}
	}
	// TODO(dvyukov): this will only work for qemu/gce,
	// because e.g. adb requires unique device IDs and we can't use what
	// manager already uses. For qemu/gce this is also bad, because we
	// override resource limits specified in config (e.g. can OOM), but works.
	switch typ := mgr.managercfg.Type; typ {
	case "gce", "qemu":
	default:
		job.resp.Error = []byte(fmt.Sprintf("testing is not yet supported for %v machine type.", typ))
		return job.resp
	}
	if err := job.buildImage(); err != nil {
		job.resp.Error = []byte(err.Error())
		return job.resp
	}
	if err := job.test(); err != nil {
		job.resp.Error = []byte(err.Error())
		return job.resp
	}
	return job.resp
}

func (job *Job) buildImage() error {
	kernelBuildSem <- struct{}{}
	defer func() { <-kernelBuildSem }()
	req, resp, mgr := job.req, job.resp, job.mgr

	// TODO(dvyukov): build syzkaller on req.SyzkallerCommit and use it.
	// Newer syzkaller may not parse an old reproducer program.
	syzkallerCommit, _ := readTag(filepath.FromSlash("syzkaller/current/tag"))
	if syzkallerCommit == "" {
		return fmt.Errorf("failed to read syzkaller build tag")
	}
	resp.Build.SyzkallerCommit = syzkallerCommit

	dir := osutil.Abs(filepath.Join("jobs", mgr.managercfg.TargetOS))
	kernelDir := filepath.Join(dir, "kernel")
	if err := osutil.MkdirAll(kernelDir); err != nil {
		return fmt.Errorf("failed to create temp dir: %v", err)
	}
	imageDir := filepath.Join(dir, "image")
	os.RemoveAll(imageDir)
	if err := osutil.MkdirAll(imageDir); err != nil {
		return fmt.Errorf("failed to create temp dir: %v", err)
	}
	workDir := filepath.Join(dir, "workdir")
	os.RemoveAll(workDir)
	if err := osutil.MkdirAll(workDir); err != nil {
		return fmt.Errorf("failed to create temp dir: %v", err)
	}

	Logf(0, "job: fetching kernel...")
	kernelCommit, err := git.Checkout(kernelDir, req.KernelRepo, req.KernelBranch)
	if err != nil {
		return fmt.Errorf("failed to checkout kernel repo: %v", err)
	}
	resp.Build.KernelCommit = kernelCommit

	if err := git.Patch(kernelDir, req.Patch); err != nil {
		return err
	}

	Logf(0, "job: building kernel...")
	configFile := filepath.Join(dir, "kernel.config")
	if err := osutil.WriteFile(configFile, req.KernelConfig); err != nil {
		return fmt.Errorf("failed to write temp file: %v", err)
	}
	if err := kernel.Build(kernelDir, mgr.mgrcfg.Compiler, configFile); err != nil {
		return fmt.Errorf("kernel build failed: %v", err)
	}
	kernelConfig, err := ioutil.ReadFile(filepath.Join(kernelDir, ".config"))
	if err != nil {
		return fmt.Errorf("failed to read config file: %v", err)
	}
	resp.Build.KernelConfig = kernelConfig

	Logf(0, "job: creating image...")
	image := filepath.Join(imageDir, "image")
	key := filepath.Join(imageDir, "key")
	err = kernel.CreateImage(kernelDir, mgr.mgrcfg.Userspace,
		mgr.mgrcfg.Kernel_Cmdline, mgr.mgrcfg.Kernel_Sysctl, image, key)
	if err != nil {
		return fmt.Errorf("image build failed: %v", err)
	}
	// TODO(dvyukov): test that the image is good (boots and we can ssh into it).

	mgrcfg := new(mgrconfig.Config)
	*mgrcfg = *mgr.managercfg
	mgrcfg.Name += "-job"
	mgrcfg.Workdir = workDir
	mgrcfg.Vmlinux = filepath.Join(kernelDir, "vmlinux")
	mgrcfg.Kernel_Src = kernelDir
	mgrcfg.Syzkaller = filepath.FromSlash("syzkaller/current")
	mgrcfg.Image = image
	mgrcfg.Sshkey = key

	// Reload config to fill derived fields (ugly hack).
	cfgdata, err := config.SaveData(mgrcfg)
	if err != nil {
		return fmt.Errorf("failed to save manager config: %v", err)
	}
	if job.mgrcfg, err = mgrconfig.LoadData(cfgdata); err != nil {
		return fmt.Errorf("failed to reload manager config: %v", err)
	}
	return nil
}

func (job *Job) test() error {
	req, mgrcfg := job.req, job.mgrcfg

	Logf(0, "job: booting VM...")
	vmEnv := mgrconfig.CreateVMEnv(mgrcfg, false)
	vmPool, err := vm.Create(mgrcfg.Type, vmEnv)
	if err != nil {
		return fmt.Errorf("failed to create VM pool: %v", err)
	}
	inst, err := vmPool.Create(0)
	if err != nil {
		return fmt.Errorf("failed to create VM: %v", err)
	}
	defer inst.Close()

	Logf(0, "job: copying binaries...")
	execprogBin, err := inst.Copy(mgrcfg.SyzExecprogBin)
	if err != nil {
		return fmt.Errorf("failed to copy test binary to VM: %v", err)
	}
	executorBin, err := inst.Copy(mgrcfg.SyzExecutorBin)
	if err != nil {
		return fmt.Errorf("failed to copy test binary to VM: %v", err)
	}
	progFile := filepath.Join(mgrcfg.Workdir, "repro.prog")
	if err := osutil.WriteFile(progFile, req.ReproSyz); err != nil {
		return fmt.Errorf("failed to write temp file: %v", err)
	}
	vmProgFile, err := inst.Copy(progFile)
	if err != nil {
		return fmt.Errorf("failed to copy to VM: %v", err)
	}
	reporter, err := report.NewReporter(mgrcfg.TargetOS, mgrcfg.Kernel_Src,
		filepath.Dir(mgrcfg.Vmlinux), nil, mgrcfg.ParsedIgnores)
	if err != nil {
		return err
	}

	Logf(0, "job: testing syzkaller program...")
	testTime := 10 * time.Minute
	if len(req.ReproC) != 0 {
		testTime /= 2
	}
	// TODO(dvyukov): we currently ignore ReproOpts in req.
	// Proably we should test with both default opts and req.ReproOpts.
	cmdSyz := fmt.Sprintf("%v -executor %v -arch=%v -procs=%v -sandbox=%v -repeat=0 -cover=0 %v",
		execprogBin, executorBin, mgrcfg.TargetArch, mgrcfg.Procs, mgrcfg.Sandbox, vmProgFile)
	if crashed, err := job.testProgram(inst, cmdSyz, reporter, testTime); crashed || err != nil {
		return err
	}

	if len(req.ReproC) != 0 {
		Logf(0, "job: testing C program...")
		cFile := filepath.Join(mgrcfg.Workdir, "repro.c")
		if err := osutil.WriteFile(cFile, req.ReproC); err != nil {
			return fmt.Errorf("failed to write temp file: %v", err)
		}
		target, err := prog.GetTarget(mgrcfg.TargetOS, mgrcfg.TargetArch)
		if err != nil {
			return err
		}
		bin, err := csource.Build(target, "c", cFile)
		if err != nil {
			return err
		}
		vmBin, err := inst.Copy(bin)
		if err != nil {
			return fmt.Errorf("failed to copy test binary to VM: %v", err)
		}
		if crashed, err := job.testProgram(inst, vmBin, reporter, testTime); crashed || err != nil {
			return err
		}
	}
	return nil
}

func (job *Job) testProgram(inst *vm.Instance, command string, reporter report.Reporter,
	testTime time.Duration) (bool, error) {
	outc, errc, err := inst.Run(testTime, nil, command)
	if err != nil {
		return false, fmt.Errorf("failed to run binary in VM: %v", err)
	}
	title, report, output, crashed, _ := vm.MonitorExecution(outc, errc, reporter)
	if crashed {
		job.resp.CrashTitle = title
		job.resp.CrashReport = report
		job.resp.CrashLog = output
	}
	return crashed, nil
}
<|endoftext|>


<commit_before>package server

import (
	"google.golang.org/grpc"

	"go.pachyderm.com/pachyderm/src/pps"
	"go.pedge.io/google-protobuf"
	"golang.org/x/net/context"
)

type localAPIClient struct {
	apiServer pps.APIServer
}

func newLocalAPIClient(apiServer pps.APIServer) *localAPIClient {
	return &localAPIClient{apiServer}
}

func (a *localAPIClient) CreateJob(ctx context.Context, request *pps.CreateJobRequest, _ ...grpc.CallOption) (response *pps.Job, err error) {
	return a.apiServer.CreateJob(ctx, request)
}

func (a *localAPIClient) GetJob(ctx context.Context, request *pps.GetJobRequest, _ ...grpc.CallOption) (response *pps.Job, err error) {
	return a.apiServer.GetJob(ctx, request)
}

func (a *localAPIClient) GetJobsByPipelineName(ctx context.Context, request *pps.GetJobsByPipelineNameRequest, _ ...grpc.CallOption) (response *pps.Jobs, err error) {
	return a.apiServer.GetJobsByPipelineName(ctx, request)
}

func (a *localAPIClient) StartJob(ctx context.Context, request *pps.StartJobRequest, _ ...grpc.CallOption) (response *google_protobuf.Empty, err error) {
	return a.apiServer.StartJob(ctx, request)
}

func (a *localAPIClient) GetJobStatus(ctx context.Context, request *pps.GetJobStatusRequest, _ ...grpc.CallOption) (response *pps.JobStatus, err error) {
	return a.apiServer.GetJobStatus(ctx, request)
}

func (a *localAPIClient) GetJobLogs(request *pps.GetJobLogsRequest, responseServer pps.API_GetJobLogsServer) (err error) {
}

func (a *localAPIClient) CreatePipeline(ctx context.Context, request *pps.CreatePipelineRequest, _ ...grpc.CallOption) (response *pps.Pipeline, err error) {
	return a.apiServer.CreatePipeline(ctx, request)
}

func (a *localAPIClient) GetPipeline(ctx context.Context, request *pps.GetPipelineRequest, _ ...grpc.CallOption) (response *pps.Pipeline, err error) {
	return a.apiServer.GetPipeline(ctx, request)
}

func (a *localAPIClient) GetAllPipelines(ctx context.Context, request *google_protobuf.Empty, _ ...grpc.CallOption) (response *pps.Pipelines, err error) {
	return a.apiServer.GetAllPipelines(ctx, request)
}
<commit_msg>continue local api client for pps<commit_after>package server

import (
	"google.golang.org/grpc"

	"go.pachyderm.com/pachyderm/src/pps"
	"go.pedge.io/google-protobuf"
	"go.pedge.io/proto/stream"
	"golang.org/x/net/context"
)

type localAPIClient struct {
	apiServer pps.APIServer
}

func newLocalAPIClient(apiServer pps.APIServer) *localAPIClient {
	return &localAPIClient{apiServer}
}

func (a *localAPIClient) CreateJob(ctx context.Context, request *pps.CreateJobRequest, _ ...grpc.CallOption) (response *pps.Job, err error) {
	return a.apiServer.CreateJob(ctx, request)
}

func (a *localAPIClient) GetJob(ctx context.Context, request *pps.GetJobRequest, _ ...grpc.CallOption) (response *pps.Job, err error) {
	return a.apiServer.GetJob(ctx, request)
}

func (a *localAPIClient) GetJobsByPipelineName(ctx context.Context, request *pps.GetJobsByPipelineNameRequest, _ ...grpc.CallOption) (response *pps.Jobs, err error) {
	return a.apiServer.GetJobsByPipelineName(ctx, request)
}

func (a *localAPIClient) StartJob(ctx context.Context, request *pps.StartJobRequest, _ ...grpc.CallOption) (response *google_protobuf.Empty, err error) {
	return a.apiServer.StartJob(ctx, request)
}

func (a *localAPIClient) GetJobStatus(ctx context.Context, request *pps.GetJobStatusRequest, _ ...grpc.CallOption) (response *pps.JobStatus, err error) {
	return a.apiServer.GetJobStatus(ctx, request)
}

func (a *localAPIClient) GetJobLogs(ctx context.Context, request *pps.GetJobLogsRequest, _ ...grpc.CallOption) (client pps.API_GetJobLogsClient, err error) {
	steamingBytesRelayer := protostream.NewStreamingBytesRelayer(ctx)
	if err := a.apiServer.GetJobLogs(request, steamingBytesRelayer); err != nil {
		return nil, err
	}
	return steamingBytesRelayer, nil
}

func (a *localAPIClient) CreatePipeline(ctx context.Context, request *pps.CreatePipelineRequest, _ ...grpc.CallOption) (response *pps.Pipeline, err error) {
	return a.apiServer.CreatePipeline(ctx, request)
}

func (a *localAPIClient) GetPipeline(ctx context.Context, request *pps.GetPipelineRequest, _ ...grpc.CallOption) (response *pps.Pipeline, err error) {
	return a.apiServer.GetPipeline(ctx, request)
}

func (a *localAPIClient) GetAllPipelines(ctx context.Context, request *google_protobuf.Empty, _ ...grpc.CallOption) (response *pps.Pipelines, err error) {
	return a.apiServer.GetAllPipelines(ctx, request)
}
<|endoftext|>


<commit_before>// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package filter

import (
	"github.com/goharbor/harbor/src/replication/model"
	"github.com/goharbor/harbor/src/replication/util"
	"strings"
)

// DoFilterArtifacts filter the artifacts according to the filters
func DoFilterArtifacts(artifacts []*model.Artifact, filters []*model.Filter) ([]*model.Artifact, error) {
	fl, err := BuildArtifactFilters(filters)
	if err != nil {
		return nil, err
	}
	return fl.Filter(artifacts)
}

// BuildArtifactFilters from the defined filters
func BuildArtifactFilters(filters []*model.Filter) (ArtifactFilters, error) {
	var fs ArtifactFilters
	for _, filter := range filters {
		var f ArtifactFilter
		switch filter.Type {
		case model.FilterTypeLabel:
			f = &artifactLabelFilter{
				labels: filter.Value.([]string),
			}
		case model.FilterTypeTag:
			f = &artifactTagFilter{
				pattern: filter.Value.(string),
			}
		case model.FilterTypeResource:
			v := filter.Value.(model.ResourceType)
			if v != model.ResourceTypeArtifact && v != model.ResourceTypeChart {
				f = &artifactTypeFilter{
					types: []string{string(v)},
				}
			}
		}
		if f != nil {
			fs = append(fs, f)
		}
	}
	return fs, nil
}

// ArtifactFilter filter the artifacts
type ArtifactFilter interface {
	Filter([]*model.Artifact) ([]*model.Artifact, error)
}

// ArtifactFilters is an array of artifact filter
type ArtifactFilters []ArtifactFilter

// Filter artifacts
func (a ArtifactFilters) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
	var err error
	for _, filter := range a {
		artifacts, err = filter.Filter(artifacts)
		if err != nil {
			return nil, err
		}
	}
	return artifacts, nil
}

type artifactTypeFilter struct {
	types []string
}

func (a *artifactTypeFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
	if len(a.types) == 0 {
		return artifacts, nil
	}
	var result []*model.Artifact
	for _, artifact := range artifacts {
		for _, t := range a.types {
			if strings.ToLower(artifact.Type) == strings.ToLower(t) {
				result = append(result, artifact)
				continue
			}
		}
	}
	return result, nil
}

// filter the artifacts according to the labels. Only the artifact contains all labels defined
// in the filter is the valid one
type artifactLabelFilter struct {
	labels []string
}

func (a *artifactLabelFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
	if len(a.labels) == 0 {
		return artifacts, nil
	}
	var result []*model.Artifact
	for _, artifact := range artifacts {
		labels := map[string]struct{}{}
		for _, label := range artifact.Labels {
			labels[label] = struct{}{}
		}
		match := true
		for _, label := range a.labels {
			if _, exist := labels[label]; !exist {
				match = false
				break
			}
		}
		// add the artifact to the result list if it contains all labels defined for the filter
		if match {
			result = append(result, artifact)
		}
	}
	return result, nil
}

// filter artifacts according to whether the artifact is tagged or untagged artifact
type artifactTaggedFilter struct {
	tagged bool
}

func (a *artifactTaggedFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
	var result []*model.Artifact
	for _, artifact := range artifacts {
		if a.tagged && len(artifact.Tags) > 0 ||
			!a.tagged && len(artifact.Tags) == 0 {
			result = append(result, artifact)
		}
	}
	return result, nil
}

type artifactTagFilter struct {
	pattern string
}

func (a *artifactTagFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
	if len(a.pattern) == 0 {
		return artifacts, nil
	}
	var result []*model.Artifact
	for _, artifact := range artifacts {
		var tags []string
		for _, tag := range artifact.Tags {
			match, err := util.Match(a.pattern, tag)
			if err != nil {
				return nil, err
			}
			if match {
				tags = append(tags, tag)
				continue
			}
		}
		if len(tags) == 0 {
			continue
		}
		// copy a new artifact here to avoid changing the original one
		result = append(result, &model.Artifact{
			Type:   artifact.Type,
			Digest: artifact.Digest,
			Labels: artifact.Labels,
			Tags:   tags,
		})
	}
	return result, nil
}
<commit_msg>Make sure the tag filter have the same behavior for empty value and *<commit_after>// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package filter

import (
	"github.com/goharbor/harbor/src/replication/model"
	"github.com/goharbor/harbor/src/replication/util"
	"strings"
)

// DoFilterArtifacts filter the artifacts according to the filters
func DoFilterArtifacts(artifacts []*model.Artifact, filters []*model.Filter) ([]*model.Artifact, error) {
	fl, err := BuildArtifactFilters(filters)
	if err != nil {
		return nil, err
	}
	return fl.Filter(artifacts)
}

// BuildArtifactFilters from the defined filters
func BuildArtifactFilters(filters []*model.Filter) (ArtifactFilters, error) {
	var fs ArtifactFilters
	for _, filter := range filters {
		var f ArtifactFilter
		switch filter.Type {
		case model.FilterTypeLabel:
			f = &artifactLabelFilter{
				labels: filter.Value.([]string),
			}
		case model.FilterTypeTag:
			f = &artifactTagFilter{
				pattern: filter.Value.(string),
			}
		case model.FilterTypeResource:
			v := filter.Value.(model.ResourceType)
			if v != model.ResourceTypeArtifact && v != model.ResourceTypeChart {
				f = &artifactTypeFilter{
					types: []string{string(v)},
				}
			}
		}
		if f != nil {
			fs = append(fs, f)
		}
	}
	return fs, nil
}

// ArtifactFilter filter the artifacts
type ArtifactFilter interface {
	Filter([]*model.Artifact) ([]*model.Artifact, error)
}

// ArtifactFilters is an array of artifact filter
type ArtifactFilters []ArtifactFilter

// Filter artifacts
func (a ArtifactFilters) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
	var err error
	for _, filter := range a {
		artifacts, err = filter.Filter(artifacts)
		if err != nil {
			return nil, err
		}
	}
	return artifacts, nil
}

type artifactTypeFilter struct {
	types []string
}

func (a *artifactTypeFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
	if len(a.types) == 0 {
		return artifacts, nil
	}
	var result []*model.Artifact
	for _, artifact := range artifacts {
		for _, t := range a.types {
			if strings.ToLower(artifact.Type) == strings.ToLower(t) {
				result = append(result, artifact)
				continue
			}
		}
	}
	return result, nil
}

// filter the artifacts according to the labels. Only the artifact contains all labels defined
// in the filter is the valid one
type artifactLabelFilter struct {
	labels []string
}

func (a *artifactLabelFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
	if len(a.labels) == 0 {
		return artifacts, nil
	}
	var result []*model.Artifact
	for _, artifact := range artifacts {
		labels := map[string]struct{}{}
		for _, label := range artifact.Labels {
			labels[label] = struct{}{}
		}
		match := true
		for _, label := range a.labels {
			if _, exist := labels[label]; !exist {
				match = false
				break
			}
		}
		// add the artifact to the result list if it contains all labels defined for the filter
		if match {
			result = append(result, artifact)
		}
	}
	return result, nil
}

// filter artifacts according to whether the artifact is tagged or untagged artifact
type artifactTaggedFilter struct {
	tagged bool
}

func (a *artifactTaggedFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
	var result []*model.Artifact
	for _, artifact := range artifacts {
		if a.tagged && len(artifact.Tags) > 0 ||
			!a.tagged && len(artifact.Tags) == 0 {
			result = append(result, artifact)
		}
	}
	return result, nil
}

type artifactTagFilter struct {
	pattern string
}

func (a *artifactTagFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
	if len(a.pattern) == 0 {
		return artifacts, nil
	}
	var result []*model.Artifact
	for _, artifact := range artifacts {
		// untagged artifact
		if len(artifact.Tags) == 0 {
			match, err := util.Match(a.pattern, "")
			if err != nil {
				return nil, err
			}
			if match {
				result = append(result, artifact)
			}
			continue
		}

		// tagged artifact
		var tags []string
		for _, tag := range artifact.Tags {
			match, err := util.Match(a.pattern, tag)
			if err != nil {
				return nil, err
			}
			if match {
				tags = append(tags, tag)
				continue
			}
		}
		if len(tags) == 0 {
			continue
		}
		// copy a new artifact here to avoid changing the original one
		result = append(result, &model.Artifact{
			Type:   artifact.Type,
			Digest: artifact.Digest,
			Labels: artifact.Labels,
			Tags:   tags,
		})
	}
	return result, nil
}
<|endoftext|>


<commit_before>package toscalib

// We define a type status used in the PropertyDefinition
type Status int64

// Valid values for Status as described in Appendix 5.7.3
const (
	Supported    Status = 1
	Unsupported  Status = 2
	Experimental Status = 3
	Deprecated   Status = 4
)

// ConstraintClauses definition as described in Appendix 5.2.
// This is a map where the index is a string that may have a value in
// {"equal","greater_than", ...} (see Appendix 5.2) a,s value is an interface
// for the definition.
// Example: ConstraintClauses may be [ "greater_than": 3 ]
type ConstraintClauses map[string]interface{}

// Evaluate the constraint and return a boolean
func (this *ConstraintClauses) Evaluate(interface{}) bool { return true }

// TODO: implement the Mashaler YAML interface for the constraint type
func (this *ConstraintClauses) UnmarshalYAML() {}

// PropertyDefinition as described in Appendix 5.7:
// A property definition defines a named, typed value and related data
// that can be associated with an entity defined in this specification
// (e.g., Node Types, Relation ship Types, Capability Types, etc.).
// Properties are used by template authors to provide input values to
// TOSCA entities which indicate their “desired state” when they are instantiated.
// The value of a property can be retrieved using the
// get_property function within TOSCA Service Templates
type PropertyDefinition struct {
	Type        string            `yaml:"type"`                  // The required data type for the property
	Description string            `yaml:"description,omitempty"` // The optional description for the property.
	Required    bool              `yaml:"required"`              // An optional key that declares a property as required ( true) or not ( false) Default: true
	Default     interface{}       `yaml:"default"`
	Status      Status            `yaml:"status"`
	Constraints ConstraintClauses `yaml:"constraints,inline,omitempty"`
	EntrySchema string            `yaml:"entry_schema,omitempty"`
}

// Input corresponds to  `yaml:"inputs,omitempty"`
type Input struct {
	Type             string            `yaml:"type"`
	Description      string            `yaml:"description,omitempty"` // Not required
	Constraints      ConstraintClauses `yaml:"constraints,omitempty,inline"`
	ValidSourceTypes interface{}       `yaml:"valid_source_types,omitempty"`
	Occurrences      interface{}       `yaml:"occurrences,omitempty"`
}

type Output struct {
	Value       interface{} `yaml:"value"`
	Description string      `yaml:"description"`
}

// TODO: Implement the type as defined in Appendix 5.9
type AttributeDefinition interface{}

// RequirementDefinition as described in Appendix 6.2
type RequirementDefinition struct {
	Capability   string     `yaml:"capability"`             // The required reserved keyname used that can be used to provide the name of a valid Capability Type that can fulfil the requirement
	node         string     `yaml:"node,omitempty"`         // The optional reserved keyname used to provide the name of a valid Node Type that contains the capability definition that can be used to fulfil the requirement
	Relationship string     `yaml:"relationship,omitempty"` //The optional reserved keyname used to provide the name of a valid Relationship Type to construct when fulfilling the requirement.
	Occurrences  ToscaRange `yaml:"occurences,omitempty"`   // The optional minimum and maximum occurrences for the requirement.  Note: the keyword UNBOUNDED is also supported to represent any positive integer
}

//TODO: Appendix 6.1
type CapabilityDefinition interface{}

// TODO: Appendix 5.12
type InterfaceDefinition interface{}

// TODO: Appendix 5.5
type ArtifactDefinition interface{}

// TODO Appendix 5.4
// A node filter definition defines criteria for selection of a TOSCA Node Template based upon the template’s property values, capabilities and capability properties.
type NodeFilter interface{}

// NodeType as described is Appendix 6.8.
// A Node Type is a reusable entity that defines the type of one or more Node Templates. As such, a Node Type defines the structure of observable properties via a Properties Definition, the Requirements and Capabilities of the node as well as its supported interfaces.
type NodeType struct {
	DerivedFrom  string                           `yaml:"derived_from,omitempty"` // An optional parent Node Type name this new Node Type derives from
	Description  string                           `yaml:"description,omitempty"`  // An optional description for the Node Type
	Properties   map[string]PropertyDefinition    `yaml:"properties,omitempty"`   // An optional list of property definitions for the Node Type.
	Attributes   map[string]AttributeDefinition   `yaml:"attributes,omitempty"`   // An optional list of attribute definitions for the Node Type.
	Requirements map[string]RequirementDefinition `yaml:"requirements,omitempty"` // An optional sequenced list of requirement definitions for the Node Type
	Capabilities map[string]CapabilityDefinition  `yaml:"capabilities,omitempty"` // An optional list of capability definitions for the Node Type
	Interfaces   map[string]InterfaceDefinition   `yaml:"interfaces,omitempty"`   // An optional list of interface definitions supported by the Node Type
	Artifacts    map[string]ArtifactDefinition    `yaml:"artifacts,omitempty" `   // An optional list of named artifact definitions for the Node Type
	Copy         string                           `yaml:"copy"`                   // The optional (symbolic) name of another node template to copy into (all keynames and values) and use as a basis for this node template.
}

// DataType as described in Appendix 6.5
// A Data Type definition defines the schema for new named datatypes in TOSCA.
type DataType struct {
	DerivedFrom string                        `yaml:"derived_from,omitempty"` // The optional key used when a datatype is derived from an existing TOSCA Data Type.
	Description string                        `yaml:"description,omitempty"`  // The optional description for the Data Type.
	Constraints ConstraintClauses             `yaml:"constraints"`            // The optional list of sequenced constraint clauses for the Data Type.
	Properties  map[string]PropertyDefinition `yaml:"properties"`             // The optional list property definitions that comprise the schema for a complex Data Type in TOSCA.
}

// NodeTemplate as described in Appendix 7.3
// A Node Template specifies the occurrence of a manageable software component as part of an application’s topology model which is defined in a TOSCA Service Template.  A Node template is an instance of a specified Node Type and can provide customized properties, constraints or operations which override the defaults provided by its Node Type and its implementations.
type NodeTemplate struct {
	Type         string                         `yaml:"type"`                   // The required name of the Node Type the Node Template is based upon.
	Decription   string                         `yaml:"description,omitempty"`  // An optional description for the Node Template.
	Directives   []string                       `yaml:"directives,omitempty"`   // An optional list of directive values to provide processing instructions to orchestrators and tooling.
	Properties   map[string]interface{}         `yaml:"properties,omitempty"`   // An optional list of property value assignments for the Node Template.
	Attributes   map[string]interface{}         `yaml:"attributes,omitempty"`   // An optional list of attribute value assignments for the Node Template.
	Requirements interface{}                    `yaml:"requirements,omitempty"` // An optional sequenced list of requirement assignments for the Node Template.
	Capabilities map[string]interface{}         `yaml:"capabilities,omitempty"` // An optional list of capability assignments for the Node Template.
	Interfaces   map[string]InterfaceDefinition `yaml:"interfaces"`             // An optional list of named interface definitions for the Node Template.
	Artifcats    map[string]ArtifactDefinition  `yaml:"artifcats,omitempty"`    // An optional list of named artifact definitions for the Node Template.
	NodeFilter   map[string]NodeFilter          `yaml:"node_filter,omitempty"`  // The optional filter definition that TOSCA orchestrators would use to select the correct target node.  This keyname is only valid if the directive has the value of “selectable” set.
}

// RepositoryDefinition as desribed in Appendix 5.6
// A repository definition defines a named external repository which contains deployment and implementation artifacts that are referenced within the TOSCA Service Template.
type RepositoryDefinition struct {
	Description string               `yaml:"description,omitempty"` // The optional description for the repository.
	Url         string               `yaml:"url"`                   // The required URL or network address used to access the repository.
	Credential  CredentialDefinition `yaml:"credential"`            // The optional Credential used to authorize access to the repository.
}

// RelationshipType as described in appendix 6.9
// A Relationship Type is a reusable entity that defines the type of one or more relationships between Node Types or Node Templates.
// TODO
type RelationshipType interface{}

// CapabilityType as described in appendix 6.6
//A Capability Type is a reusable entity that describes a kind of capability that a Node Type can declare to expose.  Requirements (implicit or explicit) that are declared as part of one node can be matched to (i.e., fulfilled by) the Capabilities declared by another node.
// TODO
type CapabilityType interface{}

// ArtifactType as described in appendix 6.3
//An Artifact Type is a reusable entity that defines the type of one or more files which Node Types or Node Templates can have dependent relationships and used during operations such as during installation or deployment.
// TODO
type ArtifactType interface{}

// InterfaceType as described in Appendix A 6.4
// An Interface Type is a reusable entity that describes a set of operations that can be used to interact with or manage a node or relationship in a TOSCA topology.
type InterfaceType struct {
	Inputs map[string]PropertyDefinition `yaml:"inputs"` // The optional list of input parameter definitions.
}

// TopologyTemplateType as described in appendix A 8
// This section defines the topology template of a cloud application. The main ingredients of the topology template are node templates representing components of the application and relationship templates representing links between the components. These elements are defined in the nested node_templates section and the nested relationship_templates sections, respectively.  Furthermore, a topology template allows for defining input parameters, output parameters as well as grouping of node templates.
type TopologyTemplateType struct {
	Inputs        map[string]Input        `yaml:"inputs,omitempty"`
	NodeTemplates map[string]NodeTemplate `yaml:"node_templates"`
	Outputs       map[string]Output       `yaml:"outputs,omitempty"`
}

// TopologyStructure as defined in
//http://docs.oasis-open.org/tosca/TOSCA-Simple-Profile-YAML/v1.0/csd03/TOSCA-Simple-Profile-YAML-v1.0-csd03.html
type TopologyTemplateStruct struct {
	DefinitionsVersion string                          `yaml:"tosca_definitions_version"` // A.9.3.1 tosca_definitions_version
	Description        string                          `yaml:"description,omitempty"`
	Imports            []string                        `yaml:"imports,omitempty"`            // Declares import statements external TOSCA Definitions documents. For example, these may be file location or URIs relative to the service template file within the same TOSCA CSAR file.
	Repositories       map[string]RepositoryDefinition `yaml:"repositories,omitempty"`       // Declares the list of external repositories which contain artifacts that are referenced in the service template along with their addresses and necessary credential information used to connect to them in order to retrieve the artifacts.
	DataTypes          map[string]DataType             `yaml:"data_types,omitempty"`         // Declares a list of optional TOSCA Data Type definitions.
	NodeTypes          map[string]NodeType             `yaml:"node_types,omitempty"`         // This section contains a set of node type definitions for use in service templates.
	RelationshipTypes  map[string]RelationshipType     `yaml:"relationship_types,omitempty"` // This section contains a set of relationship type definitions for use in service templates.
	CapabilityTypes    map[string]CapabilityType       `yaml:"capability_types,omitempty"`   // This section contains an optional list of capability type definitions for use in service templates.
	ArtifactTypes      map[string]ArtifactType         `yaml:"artifact_types,omitempty"`     // This section contains an optional list of artifact type definitions for use in service templates
	DlsDefinitions     interface{}                     `yaml:"dsl_definitions,omitempty"`    // Declares optional DSL-specific definitions and conventions.  For example, in YAML, this allows defining reusable YAML macros (i.e., YAML alias anchors) for use throughout the TOSCA Service Template.
	InterfaceTypes     map[string]InterfaceType        `yaml:"interface_types,omitempty"`    // This section contains an optional list of interface type definitions for use in service templates.
	TopologyTemplate   TopologyTemplateType            `yaml:"topology_template"`            // Defines the topology template of an application or service, consisting of node templates that represent the application’s or service’s components, as well as relationship templates representing relations between the components.
}
<commit_msg>Added an "omitempty" for the interface field in the node template<commit_after>package toscalib

// We define a type status used in the PropertyDefinition
type Status int64

// Valid values for Status as described in Appendix 5.7.3
const (
	Supported    Status = 1
	Unsupported  Status = 2
	Experimental Status = 3
	Deprecated   Status = 4
)

// ConstraintClauses definition as described in Appendix 5.2.
// This is a map where the index is a string that may have a value in
// {"equal","greater_than", ...} (see Appendix 5.2) a,s value is an interface
// for the definition.
// Example: ConstraintClauses may be [ "greater_than": 3 ]
type ConstraintClauses map[string]interface{}

// Evaluate the constraint and return a boolean
func (this *ConstraintClauses) Evaluate(interface{}) bool { return true }

// TODO: implement the Mashaler YAML interface for the constraint type
func (this *ConstraintClauses) UnmarshalYAML() {}

// PropertyDefinition as described in Appendix 5.7:
// A property definition defines a named, typed value and related data
// that can be associated with an entity defined in this specification
// (e.g., Node Types, Relation ship Types, Capability Types, etc.).
// Properties are used by template authors to provide input values to
// TOSCA entities which indicate their “desired state” when they are instantiated.
// The value of a property can be retrieved using the
// get_property function within TOSCA Service Templates
type PropertyDefinition struct {
	Type        string            `yaml:"type"`                  // The required data type for the property
	Description string            `yaml:"description,omitempty"` // The optional description for the property.
	Required    bool              `yaml:"required"`              // An optional key that declares a property as required ( true) or not ( false) Default: true
	Default     interface{}       `yaml:"default"`
	Status      Status            `yaml:"status"`
	Constraints ConstraintClauses `yaml:"constraints,inline,omitempty"`
	EntrySchema string            `yaml:"entry_schema,omitempty"`
}

// Input corresponds to  `yaml:"inputs,omitempty"`
type Input struct {
	Type             string            `yaml:"type"`
	Description      string            `yaml:"description,omitempty"` // Not required
	Constraints      ConstraintClauses `yaml:"constraints,omitempty,inline"`
	ValidSourceTypes interface{}       `yaml:"valid_source_types,omitempty"`
	Occurrences      interface{}       `yaml:"occurrences,omitempty"`
}

type Output struct {
	Value       interface{} `yaml:"value"`
	Description string      `yaml:"description"`
}

// TODO: Implement the type as defined in Appendix 5.9
type AttributeDefinition interface{}

// RequirementDefinition as described in Appendix 6.2
type RequirementDefinition struct {
	Capability   string     `yaml:"capability"`             // The required reserved keyname used that can be used to provide the name of a valid Capability Type that can fulfil the requirement
	node         string     `yaml:"node,omitempty"`         // The optional reserved keyname used to provide the name of a valid Node Type that contains the capability definition that can be used to fulfil the requirement
	Relationship string     `yaml:"relationship,omitempty"` //The optional reserved keyname used to provide the name of a valid Relationship Type to construct when fulfilling the requirement.
	Occurrences  ToscaRange `yaml:"occurences,omitempty"`   // The optional minimum and maximum occurrences for the requirement.  Note: the keyword UNBOUNDED is also supported to represent any positive integer
}

//TODO: Appendix 6.1
type CapabilityDefinition interface{}

// TODO: Appendix 5.12
type InterfaceDefinition interface{}

// TODO: Appendix 5.5
type ArtifactDefinition interface{}

// TODO Appendix 5.4
// A node filter definition defines criteria for selection of a TOSCA Node Template based upon the template’s property values, capabilities and capability properties.
type NodeFilter interface{}

// NodeType as described is Appendix 6.8.
// A Node Type is a reusable entity that defines the type of one or more Node Templates. As such, a Node Type defines the structure of observable properties via a Properties Definition, the Requirements and Capabilities of the node as well as its supported interfaces.
type NodeType struct {
	DerivedFrom  string                           `yaml:"derived_from,omitempty"` // An optional parent Node Type name this new Node Type derives from
	Description  string                           `yaml:"description,omitempty"`  // An optional description for the Node Type
	Properties   map[string]PropertyDefinition    `yaml:"properties,omitempty"`   // An optional list of property definitions for the Node Type.
	Attributes   map[string]AttributeDefinition   `yaml:"attributes,omitempty"`   // An optional list of attribute definitions for the Node Type.
	Requirements map[string]RequirementDefinition `yaml:"requirements,omitempty"` // An optional sequenced list of requirement definitions for the Node Type
	Capabilities map[string]CapabilityDefinition  `yaml:"capabilities,omitempty"` // An optional list of capability definitions for the Node Type
	Interfaces   map[string]InterfaceDefinition   `yaml:"interfaces,omitempty"`   // An optional list of interface definitions supported by the Node Type
	Artifacts    map[string]ArtifactDefinition    `yaml:"artifacts,omitempty" `   // An optional list of named artifact definitions for the Node Type
	Copy         string                           `yaml:"copy"`                   // The optional (symbolic) name of another node template to copy into (all keynames and values) and use as a basis for this node template.
}

// DataType as described in Appendix 6.5
// A Data Type definition defines the schema for new named datatypes in TOSCA.
type DataType struct {
	DerivedFrom string                        `yaml:"derived_from,omitempty"` // The optional key used when a datatype is derived from an existing TOSCA Data Type.
	Description string                        `yaml:"description,omitempty"`  // The optional description for the Data Type.
	Constraints ConstraintClauses             `yaml:"constraints"`            // The optional list of sequenced constraint clauses for the Data Type.
	Properties  map[string]PropertyDefinition `yaml:"properties"`             // The optional list property definitions that comprise the schema for a complex Data Type in TOSCA.
}

// NodeTemplate as described in Appendix 7.3
// A Node Template specifies the occurrence of a manageable software component as part of an application’s topology model which is defined in a TOSCA Service Template.  A Node template is an instance of a specified Node Type and can provide customized properties, constraints or operations which override the defaults provided by its Node Type and its implementations.
type NodeTemplate struct {
	Type         string                         `yaml:"type"`                   // The required name of the Node Type the Node Template is based upon.
	Decription   string                         `yaml:"description,omitempty"`  // An optional description for the Node Template.
	Directives   []string                       `yaml:"directives,omitempty"`   // An optional list of directive values to provide processing instructions to orchestrators and tooling.
	Properties   map[string]interface{}         `yaml:"properties,omitempty"`   // An optional list of property value assignments for the Node Template.
	Attributes   map[string]interface{}         `yaml:"attributes,omitempty"`   // An optional list of attribute value assignments for the Node Template.
	Requirements interface{}                    `yaml:"requirements,omitempty"` // An optional sequenced list of requirement assignments for the Node Template.
	Capabilities map[string]interface{}         `yaml:"capabilities,omitempty"` // An optional list of capability assignments for the Node Template.
	Interfaces   map[string]InterfaceDefinition `yaml:"interfaces,omitempty"`   // An optional list of named interface definitions for the Node Template.
	Artifcats    map[string]ArtifactDefinition  `yaml:"artifcats,omitempty"`    // An optional list of named artifact definitions for the Node Template.
	NodeFilter   map[string]NodeFilter          `yaml:"node_filter,omitempty"`  // The optional filter definition that TOSCA orchestrators would use to select the correct target node.  This keyname is only valid if the directive has the value of “selectable” set.
}

// RepositoryDefinition as desribed in Appendix 5.6
// A repository definition defines a named external repository which contains deployment and implementation artifacts that are referenced within the TOSCA Service Template.
type RepositoryDefinition struct {
	Description string               `yaml:"description,omitempty"` // The optional description for the repository.
	Url         string               `yaml:"url"`                   // The required URL or network address used to access the repository.
	Credential  CredentialDefinition `yaml:"credential"`            // The optional Credential used to authorize access to the repository.
}

// RelationshipType as described in appendix 6.9
// A Relationship Type is a reusable entity that defines the type of one or more relationships between Node Types or Node Templates.
// TODO
type RelationshipType interface{}

// CapabilityType as described in appendix 6.6
//A Capability Type is a reusable entity that describes a kind of capability that a Node Type can declare to expose.  Requirements (implicit or explicit) that are declared as part of one node can be matched to (i.e., fulfilled by) the Capabilities declared by another node.
// TODO
type CapabilityType interface{}

// ArtifactType as described in appendix 6.3
//An Artifact Type is a reusable entity that defines the type of one or more files which Node Types or Node Templates can have dependent relationships and used during operations such as during installation or deployment.
// TODO
type ArtifactType interface{}

// InterfaceType as described in Appendix A 6.4
// An Interface Type is a reusable entity that describes a set of operations that can be used to interact with or manage a node or relationship in a TOSCA topology.
type InterfaceType struct {
	Inputs map[string]PropertyDefinition `yaml:"inputs"` // The optional list of input parameter definitions.
}

// TopologyTemplateType as described in appendix A 8
// This section defines the topology template of a cloud application. The main ingredients of the topology template are node templates representing components of the application and relationship templates representing links between the components. These elements are defined in the nested node_templates section and the nested relationship_templates sections, respectively.  Furthermore, a topology template allows for defining input parameters, output parameters as well as grouping of node templates.
type TopologyTemplateType struct {
	Inputs        map[string]Input        `yaml:"inputs,omitempty"`
	NodeTemplates map[string]NodeTemplate `yaml:"node_templates"`
	Outputs       map[string]Output       `yaml:"outputs,omitempty"`
}

// TopologyStructure as defined in
//http://docs.oasis-open.org/tosca/TOSCA-Simple-Profile-YAML/v1.0/csd03/TOSCA-Simple-Profile-YAML-v1.0-csd03.html
type TopologyTemplateStruct struct {
	DefinitionsVersion string                          `yaml:"tosca_definitions_version"` // A.9.3.1 tosca_definitions_version
	Description        string                          `yaml:"description,omitempty"`
	Imports            []string                        `yaml:"imports,omitempty"`            // Declares import statements external TOSCA Definitions documents. For example, these may be file location or URIs relative to the service template file within the same TOSCA CSAR file.
	Repositories       map[string]RepositoryDefinition `yaml:"repositories,omitempty"`       // Declares the list of external repositories which contain artifacts that are referenced in the service template along with their addresses and necessary credential information used to connect to them in order to retrieve the artifacts.
	DataTypes          map[string]DataType             `yaml:"data_types,omitempty"`         // Declares a list of optional TOSCA Data Type definitions.
	NodeTypes          map[string]NodeType             `yaml:"node_types,omitempty"`         // This section contains a set of node type definitions for use in service templates.
	RelationshipTypes  map[string]RelationshipType     `yaml:"relationship_types,omitempty"` // This section contains a set of relationship type definitions for use in service templates.
	CapabilityTypes    map[string]CapabilityType       `yaml:"capability_types,omitempty"`   // This section contains an optional list of capability type definitions for use in service templates.
	ArtifactTypes      map[string]ArtifactType         `yaml:"artifact_types,omitempty"`     // This section contains an optional list of artifact type definitions for use in service templates
	DlsDefinitions     interface{}                     `yaml:"dsl_definitions,omitempty"`    // Declares optional DSL-specific definitions and conventions.  For example, in YAML, this allows defining reusable YAML macros (i.e., YAML alias anchors) for use throughout the TOSCA Service Template.
	InterfaceTypes     map[string]InterfaceType        `yaml:"interface_types,omitempty"`    // This section contains an optional list of interface type definitions for use in service templates.
	TopologyTemplate   TopologyTemplateType            `yaml:"topology_template"`            // Defines the topology template of an application or service, consisting of node templates that represent the application’s or service’s components, as well as relationship templates representing relations between the components.
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"regexp"
	"testing"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/transfer"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSTransferServer_basic(t *testing.T) {
	var conf transfer.DescribedServer
	rName := acctest.RandString(5)

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:      func() { testAccPreCheck(t) },
		IDRefreshName: "aws_transfer_server.foo",
		Providers:     testAccProviders,
		CheckDestroy:  testAccCheckAWSTransferServerDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSTransferServerConfig_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSTransferServerExists("aws_transfer_server.foo", &conf),
					testAccMatchResourceAttrRegionalARN("aws_transfer_server.foo", "arn", "transfer", regexp.MustCompile(`server/.+`)),
					resource.TestCheckResourceAttrSet(
						"aws_transfer_server.foo", "endpoint"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "identity_provider_type", "SERVICE_MANAGED"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.%", "1"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.NAME", "tf-acc-test-transfer-server"),
				),
			},
			{
				ResourceName:      "aws_transfer_server.foo",
				ImportState:       true,
				ImportStateVerify: true,
			},
			{
				Config: testAccAWSTransferServerConfig_basicUpdate(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSTransferServerExists("aws_transfer_server.foo", &conf),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.%", "2"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.NAME", "tf-acc-test-transfer-server"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.ENV", "test"),
					resource.TestCheckResourceAttrSet(
						"aws_transfer_server.foo", "logging_role"),
				),
			},
		},
	})
}

func TestAccAWSTransferServer_apigateway(t *testing.T) {
	var conf transfer.DescribedServer
	rName := acctest.RandString(5)

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:      func() { testAccPreCheck(t) },
		IDRefreshName: "aws_transfer_server.foo",
		Providers:     testAccProviders,
		CheckDestroy:  testAccCheckAWSTransferServerDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSTransferServerConfig_apigateway(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSTransferServerExists("aws_transfer_server.foo", &conf),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "identity_provider_type", "API_GATEWAY"),
					resource.TestCheckResourceAttrSet(
						"aws_transfer_server.foo", "invocation_role"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.%", "2"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.NAME", "tf-acc-test-transfer-server"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.TYPE", "apigateway"),
				),
			},
		},
	})
}

func TestAccAWSTransferServer_disappears(t *testing.T) {
	var conf transfer.DescribedServer

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSTransferServerDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSTransferServerConfig_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSTransferServerExists("aws_transfer_server.foo", &conf),
					testAccCheckAWSTransferServerDisappears(&conf),
				),
				ExpectNonEmptyPlan: true,
			},
		},
	})
}

func testAccCheckAWSTransferServerExists(n string, res *transfer.DescribedServer) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No Transfer Server ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).transferconn

		describe, err := conn.DescribeServer(&transfer.DescribeServerInput{
			ServerId: aws.String(rs.Primary.ID),
		})

		if err != nil {
			return err
		}

		*res = *describe.Server

		return nil
	}
}

func testAccCheckAWSTransferServerDisappears(conf *transfer.DescribedServer) resource.TestCheckFunc {

	return func(s *terraform.State) error {
		conn := testAccProvider.Meta().(*AWSClient).transferconn

		params := &transfer.DeleteServerInput{
			ServerId: conf.ServerId,
		}

		_, err := conn.DeleteServer(params)
		if err != nil {
			return err
		}

		return resource.Retry(10*time.Minute, func() *resource.RetryError {
			params := &transfer.DescribeServerInput{
				ServerId: conf.ServerId,
			}
			resp, err := conn.DescribeServer(params)
			if err != nil {
				cgw, ok := err.(awserr.Error)
				if ok && cgw.Code() == transfer.ErrCodeResourceNotFoundException {
					return nil
				}
				if ok && (cgw.Code() == transfer.ErrCodeInvalidRequestException) {
					return resource.NonRetryableError(
						fmt.Errorf("Waiting for Transfer Server to be in the correct state: %v", err))
				}
				return resource.RetryableError(fmt.Errorf(
					"Waiting for Transfer Server: %v", conf.ServerId))
			}
			if resp.Server == nil {
				return nil
			}
			return resource.RetryableError(fmt.Errorf(
				"Waiting for Transfer Server: %v", conf.ServerId))
		})

	}
}

func testAccCheckAWSTransferServerDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).transferconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_transfer_server" {
			continue
		}

		_, err := conn.DescribeServer(&transfer.DescribeServerInput{
			ServerId: aws.String(rs.Primary.ID),
		})

		if isAWSErr(err, transfer.ErrCodeResourceNotFoundException, "") {
			continue
		}

		if err != nil {
			return err
		}
	}

	return nil
}

const testAccAWSTransferServerConfig_basic = `
resource "aws_transfer_server" "foo" {
  identity_provider_type = "SERVICE_MANAGED"

  tags {
	NAME     = "tf-acc-test-transfer-server"
  }
}
`

func testAccAWSTransferServerConfig_basicUpdate(rName string) string {

	return fmt.Sprintf(`

resource "aws_iam_role" "foo" {
	name = "tf-test-transfer-server-iam-role-%s"

	assume_role_policy = <<EOF
{
	"Version": "2012-10-17",
	"Statement": [
		{
		"Effect": "Allow",
		"Principal": {
			"Service": "transfer.amazonaws.com"
		},
		"Action": "sts:AssumeRole"
		}
	]
}
EOF
}

resource "aws_iam_role_policy" "foo" {
	name = "tf-test-transfer-server-iam-policy-%s"
	role = "${aws_iam_role.foo.id}"
	policy = <<POLICY
{
	"Version": "2012-10-17",
	"Statement": [
		{
		"Sid": "AllowFullAccesstoCloudWatchLogs",
		"Effect": "Allow",
		"Action": [
			"logs:*"
		],
		"Resource": "*"
		}
	]
}
POLICY
}

resource "aws_transfer_server" "foo" {
  identity_provider_type = "SERVICE_MANAGED"
  logging_role = "${aws_iam_role.foo.arn}"

  tags {
	NAME   = "tf-acc-test-transfer-server"
	ENV    = "test"
  }
}
`, rName, rName)
}

func testAccAWSTransferServerConfig_apigateway(rName string) string {

	return fmt.Sprintf(`
resource "aws_api_gateway_rest_api" "test" {
	name = "test"
}

resource "aws_api_gateway_resource" "test" {
	rest_api_id = "${aws_api_gateway_rest_api.test.id}"
	parent_id = "${aws_api_gateway_rest_api.test.root_resource_id}"
	path_part = "test"
}

resource "aws_api_gateway_method" "test" {
	rest_api_id = "${aws_api_gateway_rest_api.test.id}"
	resource_id = "${aws_api_gateway_resource.test.id}"
	http_method = "GET"
	authorization = "NONE"
}

resource "aws_api_gateway_method_response" "error" {
	rest_api_id = "${aws_api_gateway_rest_api.test.id}"
	resource_id = "${aws_api_gateway_resource.test.id}"
	http_method = "${aws_api_gateway_method.test.http_method}"
	status_code = "400"
}

resource "aws_api_gateway_integration" "test" {
	rest_api_id = "${aws_api_gateway_rest_api.test.id}"
	resource_id = "${aws_api_gateway_resource.test.id}"
	http_method = "${aws_api_gateway_method.test.http_method}"

	type = "HTTP"
	uri = "https://www.google.de"
	integration_http_method = "GET"
}

resource "aws_api_gateway_integration_response" "test" {
	rest_api_id = "${aws_api_gateway_rest_api.test.id}"
	resource_id = "${aws_api_gateway_resource.test.id}"
	http_method = "${aws_api_gateway_integration.test.http_method}"
	status_code = "${aws_api_gateway_method_response.error.status_code}"
}

resource "aws_api_gateway_deployment" "test" {
	depends_on = ["aws_api_gateway_integration.test"]

	rest_api_id = "${aws_api_gateway_rest_api.test.id}"
	stage_name = "test"
	description = "%s"
	stage_description = "%s"

  variables = {
    "a" = "2"
  }
}

resource "aws_iam_role" "foo" {
	name = "tf-test-transfer-server-iam-role-for-apigateway-%s"

	assume_role_policy = <<EOF
{
	"Version": "2012-10-17",
	"Statement": [
		{
		"Effect": "Allow",
		"Principal": {
			"Service": "transfer.amazonaws.com"
		},
		"Action": "sts:AssumeRole"
		}
	]
}
EOF
}

resource "aws_iam_role_policy" "foo" {
	name = "tf-test-transfer-server-iam-policy-%s"
	role = "${aws_iam_role.foo.id}"
	policy = <<POLICY
{
	"Version": "2012-10-17",
	"Statement": [
		{
		"Sid": "AllowFullAccesstoCloudWatchLogs",
		"Effect": "Allow",
		"Action": [
			"logs:*"
		],
		"Resource": "*"
		}
	]
}
POLICY
}

resource "aws_transfer_server" "foo" {
	identity_provider_type	= "API_GATEWAY"
	url 				   	= "https://${aws_api_gateway_rest_api.test.id}.execute-api.us-west-2.amazonaws.com${aws_api_gateway_resource.test.path}"
	invocation_role 	   	= "${aws_iam_role.foo.arn}"
	logging_role 		   	= "${aws_iam_role.foo.arn}"

	tags {
	  NAME     = "tf-acc-test-transfer-server"
	  TYPE	   = "apigateway"
	}
}
`, rName, rName, rName, rName)

}
<commit_msg>Remove unused package<commit_after>package aws

import (
	"fmt"
	"regexp"
	"testing"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/transfer"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSTransferServer_basic(t *testing.T) {
	var conf transfer.DescribedServer
	rName := acctest.RandString(5)

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:      func() { testAccPreCheck(t) },
		IDRefreshName: "aws_transfer_server.foo",
		Providers:     testAccProviders,
		CheckDestroy:  testAccCheckAWSTransferServerDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSTransferServerConfig_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSTransferServerExists("aws_transfer_server.foo", &conf),
					testAccMatchResourceAttrRegionalARN("aws_transfer_server.foo", "arn", "transfer", regexp.MustCompile(`server/.+`)),
					resource.TestCheckResourceAttrSet(
						"aws_transfer_server.foo", "endpoint"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "identity_provider_type", "SERVICE_MANAGED"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.%", "1"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.NAME", "tf-acc-test-transfer-server"),
				),
			},
			{
				ResourceName:      "aws_transfer_server.foo",
				ImportState:       true,
				ImportStateVerify: true,
			},
			{
				Config: testAccAWSTransferServerConfig_basicUpdate(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSTransferServerExists("aws_transfer_server.foo", &conf),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.%", "2"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.NAME", "tf-acc-test-transfer-server"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.ENV", "test"),
					resource.TestCheckResourceAttrSet(
						"aws_transfer_server.foo", "logging_role"),
				),
			},
		},
	})
}

func TestAccAWSTransferServer_apigateway(t *testing.T) {
	var conf transfer.DescribedServer
	rName := acctest.RandString(5)

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:      func() { testAccPreCheck(t) },
		IDRefreshName: "aws_transfer_server.foo",
		Providers:     testAccProviders,
		CheckDestroy:  testAccCheckAWSTransferServerDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSTransferServerConfig_apigateway(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSTransferServerExists("aws_transfer_server.foo", &conf),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "identity_provider_type", "API_GATEWAY"),
					resource.TestCheckResourceAttrSet(
						"aws_transfer_server.foo", "invocation_role"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.%", "2"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.NAME", "tf-acc-test-transfer-server"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.TYPE", "apigateway"),
				),
			},
		},
	})
}

func TestAccAWSTransferServer_disappears(t *testing.T) {
	var conf transfer.DescribedServer

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSTransferServerDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSTransferServerConfig_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSTransferServerExists("aws_transfer_server.foo", &conf),
					testAccCheckAWSTransferServerDisappears(&conf),
				),
				ExpectNonEmptyPlan: true,
			},
		},
	})
}

func testAccCheckAWSTransferServerExists(n string, res *transfer.DescribedServer) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No Transfer Server ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).transferconn

		describe, err := conn.DescribeServer(&transfer.DescribeServerInput{
			ServerId: aws.String(rs.Primary.ID),
		})

		if err != nil {
			return err
		}

		*res = *describe.Server

		return nil
	}
}

func testAccCheckAWSTransferServerDisappears(conf *transfer.DescribedServer) resource.TestCheckFunc {

	return func(s *terraform.State) error {
		conn := testAccProvider.Meta().(*AWSClient).transferconn

		params := &transfer.DeleteServerInput{
			ServerId: conf.ServerId,
		}

		_, err := conn.DeleteServer(params)
		if err != nil {
			return err
		}

		return resource.Retry(10*time.Minute, func() *resource.RetryError {
			params := &transfer.DescribeServerInput{
				ServerId: conf.ServerId,
			}
			resp, err := conn.DescribeServer(params)
			if err != nil {
				cgw, ok := err.(awserr.Error)
				if ok && cgw.Code() == transfer.ErrCodeResourceNotFoundException {
					return nil
				}
				if ok && (cgw.Code() == transfer.ErrCodeInvalidRequestException) {
					return resource.NonRetryableError(
						fmt.Errorf("Waiting for Transfer Server to be in the correct state: %v", err))
				}
				return resource.RetryableError(fmt.Errorf(
					"Waiting for Transfer Server: %v", conf.ServerId))
			}
			if resp.Server == nil {
				return nil
			}
			return resource.RetryableError(fmt.Errorf(
				"Waiting for Transfer Server: %v", conf.ServerId))
		})

	}
}

func testAccCheckAWSTransferServerDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).transferconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_transfer_server" {
			continue
		}

		_, err := conn.DescribeServer(&transfer.DescribeServerInput{
			ServerId: aws.String(rs.Primary.ID),
		})

		if isAWSErr(err, transfer.ErrCodeResourceNotFoundException, "") {
			continue
		}

		if err != nil {
			return err
		}
	}

	return nil
}

const testAccAWSTransferServerConfig_basic = `
resource "aws_transfer_server" "foo" {
  identity_provider_type = "SERVICE_MANAGED"

  tags {
	NAME     = "tf-acc-test-transfer-server"
  }
}
`

func testAccAWSTransferServerConfig_basicUpdate(rName string) string {

	return fmt.Sprintf(`

resource "aws_iam_role" "foo" {
	name = "tf-test-transfer-server-iam-role-%s"

	assume_role_policy = <<EOF
{
	"Version": "2012-10-17",
	"Statement": [
		{
		"Effect": "Allow",
		"Principal": {
			"Service": "transfer.amazonaws.com"
		},
		"Action": "sts:AssumeRole"
		}
	]
}
EOF
}

resource "aws_iam_role_policy" "foo" {
	name = "tf-test-transfer-server-iam-policy-%s"
	role = "${aws_iam_role.foo.id}"
	policy = <<POLICY
{
	"Version": "2012-10-17",
	"Statement": [
		{
		"Sid": "AllowFullAccesstoCloudWatchLogs",
		"Effect": "Allow",
		"Action": [
			"logs:*"
		],
		"Resource": "*"
		}
	]
}
POLICY
}

resource "aws_transfer_server" "foo" {
  identity_provider_type = "SERVICE_MANAGED"
  logging_role = "${aws_iam_role.foo.arn}"

  tags {
	NAME   = "tf-acc-test-transfer-server"
	ENV    = "test"
  }
}
`, rName, rName)
}

func testAccAWSTransferServerConfig_apigateway(rName string) string {

	return fmt.Sprintf(`
resource "aws_api_gateway_rest_api" "test" {
	name = "test"
}

resource "aws_api_gateway_resource" "test" {
	rest_api_id = "${aws_api_gateway_rest_api.test.id}"
	parent_id = "${aws_api_gateway_rest_api.test.root_resource_id}"
	path_part = "test"
}

resource "aws_api_gateway_method" "test" {
	rest_api_id = "${aws_api_gateway_rest_api.test.id}"
	resource_id = "${aws_api_gateway_resource.test.id}"
	http_method = "GET"
	authorization = "NONE"
}

resource "aws_api_gateway_method_response" "error" {
	rest_api_id = "${aws_api_gateway_rest_api.test.id}"
	resource_id = "${aws_api_gateway_resource.test.id}"
	http_method = "${aws_api_gateway_method.test.http_method}"
	status_code = "400"
}

resource "aws_api_gateway_integration" "test" {
	rest_api_id = "${aws_api_gateway_rest_api.test.id}"
	resource_id = "${aws_api_gateway_resource.test.id}"
	http_method = "${aws_api_gateway_method.test.http_method}"

	type = "HTTP"
	uri = "https://www.google.de"
	integration_http_method = "GET"
}

resource "aws_api_gateway_integration_response" "test" {
	rest_api_id = "${aws_api_gateway_rest_api.test.id}"
	resource_id = "${aws_api_gateway_resource.test.id}"
	http_method = "${aws_api_gateway_integration.test.http_method}"
	status_code = "${aws_api_gateway_method_response.error.status_code}"
}

resource "aws_api_gateway_deployment" "test" {
	depends_on = ["aws_api_gateway_integration.test"]

	rest_api_id = "${aws_api_gateway_rest_api.test.id}"
	stage_name = "test"
	description = "%s"
	stage_description = "%s"

  variables = {
    "a" = "2"
  }
}

resource "aws_iam_role" "foo" {
	name = "tf-test-transfer-server-iam-role-for-apigateway-%s"

	assume_role_policy = <<EOF
{
	"Version": "2012-10-17",
	"Statement": [
		{
		"Effect": "Allow",
		"Principal": {
			"Service": "transfer.amazonaws.com"
		},
		"Action": "sts:AssumeRole"
		}
	]
}
EOF
}

resource "aws_iam_role_policy" "foo" {
	name = "tf-test-transfer-server-iam-policy-%s"
	role = "${aws_iam_role.foo.id}"
	policy = <<POLICY
{
	"Version": "2012-10-17",
	"Statement": [
		{
		"Sid": "AllowFullAccesstoCloudWatchLogs",
		"Effect": "Allow",
		"Action": [
			"logs:*"
		],
		"Resource": "*"
		}
	]
}
POLICY
}

resource "aws_transfer_server" "foo" {
	identity_provider_type	= "API_GATEWAY"
	url 				   	= "https://${aws_api_gateway_rest_api.test.id}.execute-api.us-west-2.amazonaws.com${aws_api_gateway_resource.test.path}"
	invocation_role 	   	= "${aws_iam_role.foo.arn}"
	logging_role 		   	= "${aws_iam_role.foo.arn}"

	tags {
	  NAME     = "tf-acc-test-transfer-server"
	  TYPE	   = "apigateway"
	}
}
`, rName, rName, rName, rName)

}
<|endoftext|>


<commit_before>package valente

import (
	"errors"
	"log"
	"time"

	"github.com/trumae/valente/action"

	"golang.org/x/net/websocket"
)

const (
	endofmessage = "___ENDOFMESSAGE___"
)

var (
	ProtocolError = errors.New("Protocol Error")
)

//HandlerFunc is a function of handle an event received into websocket.Conn
type HandlerFunc func(*websocket.Conn, *App, []string)

//Form represents the unit of user interaction
type Form interface {
	AddEventHandler(evt string, f HandlerFunc) Form
	Run(ws *websocket.Conn, app *App) error
	Initialize(ws *websocket.Conn) Form
	Render(ws *websocket.Conn, params []string) error
}

//FormImpl its a simple Form
type FormImpl struct {
	trans map[string]HandlerFunc
}

//AddEventHandler add an f function to handle evt event
func (form FormImpl) AddEventHandler(evt string, f HandlerFunc) Form {
	if form.trans == nil {
		form.trans = map[string]HandlerFunc{}
	}
	form.trans[evt] = f
	return form
}

//Run execs the form
func (form FormImpl) Run(ws *websocket.Conn, app *App) error {
	msgs := []string{}
	for {
		msg := ""
		err := websocket.Message.Receive(ws, &msg)
		if err != nil {
			log.Println("Error on WS Receive", err)
			return err
		}
		if msg == endofmessage {
			break
		} else {
			msgs = append(msgs, msg)
		}
	}
	log.Printf("currentForm = %v msgs = %v\n", app.CurrentForm, msgs)
	if len(msgs) < 1 {
		return ProtocolError
	}

	f, present := form.trans[msgs[0]]
	if present {
		f(ws, app, msgs)
	} else {
		log.Println("Evt not found", msgs[0])
	}
	return nil
}

//Initialize inits the form
func (form FormImpl) Initialize(ws *websocket.Conn) Form {
	log.Println("FormImpl Initialize")
	return form
}

func (form FormImpl) Render(ws *websocket.Conn, params []string) error {
	log.Println("FormImpl Render")
	return nil
}

//App is a Web Application representation
type App struct {
	WS          *websocket.Conn
	Forms       map[string]Form
	Data        map[string]interface{}
	CurrentForm Form
}

//GoTo replace the current form into app
func (app *App) GoTo(formName string, params []string) error {
	log.Println("App goto", formName)
	form, present := app.Forms[formName]
	if present {
		app.CurrentForm = form.Initialize(app.WS)
		form.Render(app.WS, params)
		action.UnblockUI(app.WS)
	} else {
		log.Println("[ERROR] Form not registred", formName)
	}
	return nil
}

//Run handle events
func (app *App) Run() {
	app.Data = map[string]interface{}{}
	go func() {
		c := time.Tick(10 * time.Second)
		for _ = range c {
			err := action.Exec(app.WS, "1 == 1;")
			if err != nil {
				log.Println("Error in connection probe", err)
				return
			}
		}
	}()

	for {
		err := app.CurrentForm.Run(app.WS, app)
		if err != nil {
			return
		}
	}
}

//Initialize inits the App
func (app *App) Initialize() {
	log.Println("App Initialize")
}

//AddForm add a new form to App
func (app *App) AddForm(name string, f Form) {
	log.Println("AddForm", name, f)
	if app.Forms == nil {
		app.Forms = map[string]Form{}
	}

	app.Forms[name] = f
}
<commit_msg>app ref on render method<commit_after>package valente

import (
	"errors"
	"log"
	"time"

	"github.com/trumae/valente/action"

	"golang.org/x/net/websocket"
)

const (
	endofmessage = "___ENDOFMESSAGE___"
)

var (
	ProtocolError = errors.New("Protocol Error")
)

//HandlerFunc is a function of handle an event received into websocket.Conn
type HandlerFunc func(*websocket.Conn, *App, []string)

//Form represents the unit of user interaction
type Form interface {
	AddEventHandler(evt string, f HandlerFunc) Form
	Run(ws *websocket.Conn, app *App) error
	Initialize(ws *websocket.Conn) Form
	Render(ws *websocket.Conn, app *App, params []string) error
}

//FormImpl its a simple Form
type FormImpl struct {
	trans map[string]HandlerFunc
}

//AddEventHandler add an f function to handle evt event
func (form FormImpl) AddEventHandler(evt string, f HandlerFunc) Form {
	if form.trans == nil {
		form.trans = map[string]HandlerFunc{}
	}
	form.trans[evt] = f
	return form
}

//Run execs the form
func (form FormImpl) Run(ws *websocket.Conn, app *App) error {
	msgs := []string{}
	for {
		msg := ""
		err := websocket.Message.Receive(ws, &msg)
		if err != nil {
			log.Println("Error on WS Receive", err)
			return err
		}
		if msg == endofmessage {
			break
		} else {
			msgs = append(msgs, msg)
		}
	}
	log.Printf("currentForm = %v msgs = %v\n", app.CurrentForm, msgs)
	if len(msgs) < 1 {
		return ProtocolError
	}

	f, present := form.trans[msgs[0]]
	if present {
		f(ws, app, msgs)
	} else {
		log.Println("Evt not found", msgs[0])
	}
	return nil
}

//Initialize inits the form
func (form FormImpl) Initialize(ws *websocket.Conn) Form {
	log.Println("FormImpl Initialize")
	return form
}

func (form FormImpl) Render(ws *websocket.Conn, app *App, params []string) error {
	log.Println("FormImpl Render")
	return nil
}

//App is a Web Application representation
type App struct {
	WS          *websocket.Conn
	Forms       map[string]Form
	Data        map[string]interface{}
	CurrentForm Form
}

//GoTo replace the current form into app
func (app *App) GoTo(formName string, params []string) error {
	log.Println("App goto", formName)
	form, present := app.Forms[formName]
	if present {
		app.CurrentForm = form.Initialize(app.WS)
		err := form.Render(app.WS, app, params)
		if err != nil {
			log.Println("Error on goto", err)
		}
		action.UnblockUI(app.WS)
	} else {
		log.Println("[ERROR] Form not registred", formName)
	}
	return nil
}

//Run handle events
func (app *App) Run() {
	app.Data = map[string]interface{}{}
	go func() {
		c := time.Tick(10 * time.Second)
		for _ = range c {
			err := action.Exec(app.WS, "1 == 1;")
			if err != nil {
				log.Println("Error in connection probe", err)
				return
			}
		}
	}()

	for {
		err := app.CurrentForm.Run(app.WS, app)
		if err != nil {
			return
		}
	}
}

//Initialize inits the App
func (app *App) Initialize() {
	log.Println("App Initialize")
}

//AddForm add a new form to App
func (app *App) AddForm(name string, f Form) {
	log.Println("AddForm", name, f)
	if app.Forms == nil {
		app.Forms = map[string]Form{}
	}

	app.Forms[name] = f
}
<|endoftext|>


<commit_before>package vcs

import "os/exec"

type GitUpdate struct {
	Branch string
}
type GitPull struct {
}
type GitPullUpdate struct {
	Branch string
}
type GitBranch struct {
}
type GitFetch struct {
}
type GitPush struct {
	Branch string
}

func (cmd GitUpdate) Execute(path string) {
	args := []string{"-C", path, "checkout"}

	if cmd.Branch != "" {
		args = append(args, cmd.Branch)
	} else {
		args = append(args, "HEAD")
	}

	systemCmd := exec.Command("git", args...)
	execCommand(path, systemCmd)
}

func (cmd GitPull) Execute(path string) {
	args := []string{"-C", path, "pull"}
	systemCmd := exec.Command("git", args...)
	execCommand(path, systemCmd)
}

func (cmd GitPullUpdate) Execute(path string) {
	GitUpdate{Branch: cmd.Branch}.Execute(path)
	GitPull{}.Execute(path)
}

func (cmd GitBranch) Execute(path string) {
	args := []string{"-C", path, "rev-parse", "--abbrev-ref", "HEAD"}
	systemCmd := exec.Command("git", args...)
	execCommand(path, systemCmd)
}

func (cmd GitFetch) Execute(path string) {
	args := []string{"-C", path, "fetch", "--tags", "--prune"}
	systemCmd := exec.Command("git", args...)
	execCommand(path, systemCmd)
}

func (cmd GitPush) Execute(path string) {
	args := []string{"-C", path, "push", "--set-upstream"}

	if cmd.Branch != "" {
		args = append(args, "origin", cmd.Branch)
	} else {
		args = append(args, "--all")
	}

	systemCmd := exec.Command("git", args...)
	execCommand(path, systemCmd)
}
<commit_msg>add GitFetch in GitPullUpdate<commit_after>package vcs

import "os/exec"

type GitUpdate struct {
	Branch string
}
type GitPull struct {
}
type GitPullUpdate struct {
	Branch string
}
type GitBranch struct {
}
type GitFetch struct {
}
type GitPush struct {
	Branch string
}

func (cmd GitUpdate) Execute(path string) {
	args := []string{"-C", path, "checkout"}

	if cmd.Branch != "" {
		args = append(args, cmd.Branch)
	} else {
		args = append(args, "HEAD")
	}

	systemCmd := exec.Command("git", args...)
	execCommand(path, systemCmd)
}

func (cmd GitPull) Execute(path string) {
	args := []string{"-C", path, "pull"}
	systemCmd := exec.Command("git", args...)
	execCommand(path, systemCmd)
}

func (cmd GitPullUpdate) Execute(path string) {
	GitFetch{}.Execute(path)
	GitUpdate{Branch: cmd.Branch}.Execute(path)
	GitPull{}.Execute(path)
}

func (cmd GitBranch) Execute(path string) {
	args := []string{"-C", path, "rev-parse", "--abbrev-ref", "HEAD"}
	systemCmd := exec.Command("git", args...)
	execCommand(path, systemCmd)
}

func (cmd GitFetch) Execute(path string) {
	args := []string{"-C", path, "fetch", "--tags", "--prune"}
	systemCmd := exec.Command("git", args...)
	execCommand(path, systemCmd)
}

func (cmd GitPush) Execute(path string) {
	args := []string{"-C", path, "push", "--set-upstream"}

	if cmd.Branch != "" {
		args = append(args, "origin", cmd.Branch)
	} else {
		args = append(args, "--all")
	}

	systemCmd := exec.Command("git", args...)
	execCommand(path, systemCmd)
}
<|endoftext|>


<commit_before>package gohttp

const (
	VERSION = "0.1.8.062618_beta"
)
<commit_msg>update version<commit_after>package gohttp

const (
	VERSION = "0.1.9.070518_beta"
)
<|endoftext|>


<commit_before>package punter

const Version = "1.1.0"
<commit_msg>1.2.0<commit_after>package punter

const Version = "1.2.0"
<|endoftext|>


<commit_before>package main

const VERSION = "0.6.4"
<commit_msg>:+1: Bump up the version to 0.7.0-alpha1<commit_after>package main

const VERSION = "0.7.0-alpha1"
<|endoftext|>


<commit_before>package main

// Name is the name of this app
const Name string = "gckdir"

// Version is the version of this app.
const Version string = "0.6.1"
<commit_msg>chore: update the version.<commit_after>package main

// Name is the name of this app
const Name string = "gckdir"

// Version is the version of this app.
const Version string = "0.7.0"
<|endoftext|>


<commit_before>package main

const Version = "0.4.0"
<commit_msg>Bump to v0.4.1<commit_after>package main

const Version = "0.4.1"
<|endoftext|>


<commit_before>package egoscale

// Version of the library
const Version = "0.15.0"
<commit_msg>prepare release: 0.16.0<commit_after>package egoscale

// Version of the library
const Version = "0.16.0"
<|endoftext|>


<commit_before>package main
var VERSION string = "v0.4.0.8"
<commit_msg>Forcing recompile<commit_after>package main
var VERSION string = "v0.4.0.10"
<|endoftext|>


<commit_before>package dns

import "fmt"

// Version is current version of this library.
var Version = v{1, 1, 27}

// v holds the version of this library.
type v struct {
	Major, Minor, Patch int
}

func (v v) String() string {
	return fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)
}
<commit_msg>Release 1.1.28<commit_after>package dns

import "fmt"

// Version is current version of this library.
var Version = v{1, 1, 28}

// v holds the version of this library.
type v struct {
	Major, Minor, Patch int
}

func (v v) String() string {
	return fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)
}
<|endoftext|>