<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/url"
	"os"
	"strconv"

	"github.com/koding/kite/config"
	"github.com/koding/kite/reverseproxy"
)

var (
	flagCertFile    = flag.String("cert", "", "Cert file to be used for HTTPS")
	flagKeyFile     = flag.String("key", "", "Key file to be used for HTTPS")
	flagIp          = flag.String("ip", "0.0.0.0", "Listening IP")
	flagPort        = flag.Int("port", 3999, "Server port")
	flagPublicHost  = flag.String("host", "127.0.0.1:3999", "Public host of Proxy.")
	flagRegion      = flag.String("region", "", "Change region")
	flagEnvironment = flag.String("env", "development", "Change development")
	flagVersion     = flag.Bool("version", false, "Show version and exit")
)

func main() {
	flag.Parse()

	if *flagVersion {
		fmt.Println(reverseproxy.Version)
		os.Exit(0)
	}

	if *flagRegion == "" || *flagEnvironment == "" {
		log.Fatal("Please specify envrionment via -env and region via -region. Aborting.")
	}

	scheme := "ws"
	if *flagCertFile != "" && *flagKeyFile != "" {
		scheme = "wss"
	}

	conf := config.MustGet()
	conf.IP = *flagIp
	conf.Port = *flagPort
	conf.Region = *flagRegion
	conf.Environment = *flagEnvironment

	r := reverseproxy.New(conf)
	r.PublicHost = *flagPublicHost
	r.Scheme = scheme

	registerURL := &url.URL{
		Scheme: scheme,
		Host:   r.PublicHost + ":" + strconv.Itoa(*flagPort),
		Path:   "/kite",
	}

	r.Kite.Log.Info("Registering with register url %s", registerURL)
	if err := r.Kite.RegisterForever(registerURL); err != nil {
		r.Kite.Log.Fatal("Registering to Kontrol: %s", err)
	}

	if *flagCertFile == "" || *flagKeyFile == "" {
		log.Println("No cert/key files are defined. Running proxy unsecure.")
		err := r.ListenAndServe()
		if err != nil {
			log.Fatal("ListenAndServe: ", err)
		}
	} else {
		err := r.ListenAndServeTLS(*flagCertFile, *flagKeyFile)
		if err != nil {
			log.Fatal("ListenAndServe: ", err)
		}
	}
}
<commit_msg>reverseproxy/main: we now use sockjs, no more ws/wss<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/url"
	"os"
	"strconv"

	"github.com/koding/kite/config"
	"github.com/koding/kite/reverseproxy"
)

var (
	flagCertFile    = flag.String("cert", "", "Cert file to be used for HTTPS")
	flagKeyFile     = flag.String("key", "", "Key file to be used for HTTPS")
	flagIp          = flag.String("ip", "0.0.0.0", "Listening IP")
	flagPort        = flag.Int("port", 3999, "Server port")
	flagPublicHost  = flag.String("host", "127.0.0.1:3999", "Public host of Proxy.")
	flagRegion      = flag.String("region", "", "Change region")
	flagEnvironment = flag.String("env", "development", "Change development")
	flagVersion     = flag.Bool("version", false, "Show version and exit")
)

func main() {
	flag.Parse()

	if *flagVersion {
		fmt.Println(reverseproxy.Version)
		os.Exit(0)
	}

	if *flagRegion == "" || *flagEnvironment == "" {
		log.Fatal("Please specify envrionment via -env and region via -region. Aborting.")
	}

	scheme := "http"
	if *flagCertFile != "" && *flagKeyFile != "" {
		scheme = "https"
	}

	conf := config.MustGet()
	conf.IP = *flagIp
	conf.Port = *flagPort
	conf.Region = *flagRegion
	conf.Environment = *flagEnvironment

	r := reverseproxy.New(conf)
	r.PublicHost = *flagPublicHost
	r.Scheme = scheme

	registerURL := &url.URL{
		Scheme: scheme,
		Host:   r.PublicHost + ":" + strconv.Itoa(*flagPort),
		Path:   "/kite",
	}

	r.Kite.Log.Info("Registering with register url %s", registerURL)
	if err := r.Kite.RegisterForever(registerURL); err != nil {
		r.Kite.Log.Fatal("Registering to Kontrol: %s", err)
	}

	if *flagCertFile == "" || *flagKeyFile == "" {
		log.Println("No cert/key files are defined. Running proxy unsecure.")
		err := r.ListenAndServe()
		if err != nil {
			log.Fatal("ListenAndServe: ", err)
		}
	} else {
		err := r.ListenAndServeTLS(*flagCertFile, *flagKeyFile)
		if err != nil {
			log.Fatal("ListenAndServe: ", err)
		}
	}
}
<|endoftext|>


<commit_before>package resource

import (
	"github.com/tsinghua-io/api-server/adapter/tsinghua.edu.cn/cic/learn"
	"github.com/tsinghua-io/api-server/util"
	"golang.org/x/text/language"
	"net/http"
)

var Attended = Resource{
	"GET": util.AuthNeededHandler(GetAttended),
}

var GetAttended = learn.HandlerFunc(func(rw http.ResponseWriter, req *http.Request, ada *learn.Adapter) {
	sem := req.URL.Query().Get("semester")
	english := (util.Language(req) == language.English)

	switch sem {
	case "", "latest":
		if this, next, status, err := ada.NowAttended(english); len(next) == 0 {
			util.JSON(rw, this, status, err)
		} else {
			util.JSON(rw, next, status, err)
		}
	case "now":
		this, _, status, err := ada.NowAttended(english)
		util.JSON(rw, this, status, err)
	case "next":
		_, next, status, err := ada.NowAttended(english)
		util.JSON(rw, next, status, err)
	case "past":
		past, status, err := ada.PastAttended(english)
		util.JSON(rw, past, status, err)
	default:
		courses, status, err := ada.Attended(sem, english)
		util.JSON(rw, courses, status, err)
	}
})
<commit_msg>add all attended end point.<commit_after>package resource

import (
	"github.com/tsinghua-io/api-server/adapter/tsinghua.edu.cn/cic/learn"
	"github.com/tsinghua-io/api-server/util"
	"golang.org/x/text/language"
	"net/http"
)

var Attended = Resource{
	"GET": util.AuthNeededHandler(GetAttended),
}

var GetAttended = learn.HandlerFunc(func(rw http.ResponseWriter, req *http.Request, ada *learn.Adapter) {
	sem := req.URL.Query().Get("semester")
	english := (util.Language(req) == language.English)

	switch sem {
	case "", "latest":
		if this, next, status, err := ada.NowAttended(english); len(next) == 0 {
			util.JSON(rw, this, status, err)
		} else {
			util.JSON(rw, next, status, err)
		}
	case "this":
		this, _, status, err := ada.NowAttended(english)
		util.JSON(rw, this, status, err)
	case "next":
		_, next, status, err := ada.NowAttended(english)
		util.JSON(rw, next, status, err)
	case "past":
		past, status, err := ada.PastAttended(english)
		util.JSON(rw, past, status, err)
	case "all":
		all, status, err := ada.AllAttended(english)
		util.JSON(rw, all, status, err)
	default:
		courses, status, err := ada.Attended(sem, english)
		util.JSON(rw, courses, status, err)
	}
})
<|endoftext|>


<commit_before>package terraform_vix

import (
	"fmt"
	"log"
	"os/user"
	"path/filepath"
	"strconv"
	"time"

	"github.com/c4milo/govix"
	"github.com/c4milo/terraform_vix/helper"
	"github.com/dustin/go-humanize"
	"github.com/hashicorp/terraform/flatmap"
	"github.com/hashicorp/terraform/helper/config"
	"github.com/hashicorp/terraform/helper/diff"
	"github.com/hashicorp/terraform/terraform"
)

func resource_vix_vm_validation() *config.Validator {
	return &config.Validator{
		Required: []string{
			"name",
			"image.*",
			"image.*.url",
			"image.*.checksum",
			"image.*.checksum_type",
		},
		Optional: []string{
			"description",
			"image.*.password",
			"cpus",
			"memory",
			"upgrade_vhardware",
			"tools_init_timeout",
			"network_driver",
			"networks.*",
			"sharedfolders",
			"sharedfolder.*",
			"gui",
		},
	}
}

func resource_vix_vm_create(
	s *terraform.ResourceState,
	d *terraform.ResourceDiff,
	meta interface{}) (*terraform.ResourceState, error) {
	// Merge the diff into the state so that we have all the attributes
	// properly.
	rs := s.MergeDiff(d)

	name := rs.Attributes["name"]
	description := rs.Attributes["description"]
	cpus, err := strconv.ParseUint(rs.Attributes["cpus"], 0, 8)
	memory := rs.Attributes["memory"]
	toolsInitTimeout, err := time.ParseDuration(rs.Attributes["tools_init_timeout"])
	upgradehw, err := strconv.ParseBool(rs.Attributes["upgrade_vhardware"])
	//netdrv := rs.Attributes["network_driver"]
	launchGUI, err := strconv.ParseBool(rs.Attributes["gui"])
	sharedfolders, err := strconv.ParseBool(rs.Attributes["sharedfolders"])
	var networks []string

	if err != nil {
		return nil, err
	}

	if raw := flatmap.Expand(rs.Attributes, "networks"); raw != nil {
		if nets, ok := raw.([]interface{}); ok {
			for _, net := range nets {
				str, ok := net.(string)
				if !ok {
					continue
				}

				networks = append(networks, str)
			}
		}
	}

	// This is nasty but there doesn't seem to be a cleaner way to extract stuff
	// from the TF configuration
	image := flatmap.Expand(rs.Attributes, "image").([]interface{})[0].(map[string]interface{})

	log.Printf("[DEBUG] networks => %v", networks)

	if len(networks) == 0 {
		networks = append(networks, "bridged")
	}

	usr, err := user.Current()
	if err != nil {
		return nil, err
	}

	// FIXME(c4milo): There is an issue here whenever count is greater than 1
	// please see: https://github.com/hashicorp/terraform/issues/141
	vmPath := filepath.Join(usr.HomeDir, fmt.Sprintf(".terraform/vix/vms/%s", name))

	imageConfig := helper.FetchConfig{
		URL:          image["url"].(string),
		Checksum:     image["checksum"].(string),
		ChecksumType: image["checksum_type"].(string),
		DownloadPath: vmPath,
	}

	vmPath, err = helper.FetchFile(imageConfig)
	if err != nil {
		return nil, err
	}

	// FIXME(c4milo): This has an edge case when a resource with the same
	// name is declared with a different image box, it will return multiple
	// vmx files.
	pattern := filepath.Join(vmPath, "/**/*.vmx")

	log.Printf("[DEBUG] Finding VMX file in %s", pattern)
	files, _ := filepath.Glob(pattern)

	log.Printf("[DEBUG] VMX files found %v", files)

	if len(files) == 0 {
		return nil, fmt.Errorf("[ERROR] VMX file was not found: %s", pattern)
	}

	// Gets VIX instance
	p := meta.(*ResourceProvider)
	client := p.client

	if ((client.Provider & vix.VMWARE_VI_SERVER) == 0) ||
		((client.Provider & vix.VMWARE_SERVER) == 0) {
		log.Printf("[INFO] Registering VM in host's inventory...")
		err = client.RegisterVm(files[0])
		if err != nil {
			return nil, err
		}
	}

	log.Printf("[INFO] Opening virtual machine from %s", files[0])

	vm, err := client.OpenVm(files[0], image["password"].(string))
	if err != nil {
		return nil, err
	}
	defer client.Disconnect()

	memoryInMb, err := humanize.ParseBytes(memory)
	if err != nil {
		log.Printf("[WARN] Unable to set memory size, defaulting to 1g: %s", err)
		memoryInMb = 1024
	} else {
		memoryInMb = (memoryInMb / 1024) / 1024
	}

	log.Printf("[DEBUG] Setting memory size to %d megabytes", memoryInMb)
	vm.SetMemorySize(uint(memoryInMb))

	log.Printf("[DEBUG] Setting vcpus to %d", cpus)
	vm.SetNumberVcpus(uint8(cpus))

	log.Printf("[DEBUG] Setting annotation to %s", description)
	vm.SetAnnotation(description)

	// for _, netType := range networks {
	// 	adapter := &vix.NetworkAdapter{
	// 		//VSwitch:        vix.VSwitch{},
	// 		StartConnected: true,
	// 	}

	// 	switch netdrv {
	// 	case "e1000":
	// 		adapter.Vdevice = vix.NETWORK_DEVICE_E1000
	// 	case "vmxnet3":
	// 		adapter.Vdevice = vix.NETWORK_DEVICE_VMXNET3
	// 	default:
	// 		adapter.Vdevice = vix.NETWORK_DEVICE_E1000
	// 	}

	// 	switch netType {
	// 	case "hostonly":
	// 		adapter.ConnType = vix.NETWORK_HOSTONLY
	// 	case "bridged":
	// 		adapter.ConnType = vix.NETWORK_BRIDGED
	// 	case "nat":
	// 		adapter.ConnType = vix.NETWORK_NAT
	// 	default:
	// 		adapter.ConnType = vix.NETWORK_CUSTOM

	// 	}

	// 	err = vm.AddNetworkAdapter(adapter)
	// 	if err != nil {
	// 		return nil, err
	// 	}
	// }

	running, err := vm.IsRunning()
	if err != nil {
		return nil, err
	}

	if !running {
		if upgradehw &&
			((client.Provider & vix.VMWARE_PLAYER) == 0) {

			log.Println("[INFO] Upgrading virtual hardware...")
			err = vm.UpgradeVHardware()
			if err != nil {
				return nil, err
			}
		}

		log.Println("[INFO] Powering virtual machine on...")
		var options vix.VMPowerOption

		if launchGUI {
			log.Println("[INFO] Preparing to launch GUI...")
			options |= vix.VMPOWEROP_LAUNCH_GUI
		}

		options |= vix.VMPOWEROP_NORMAL

		err = vm.PowerOn(options)
		if err != nil {
			return rs, err
		}

		err = vm.WaitForToolsInGuest(toolsInitTimeout)
		if err != nil {
			log.Println("[WARN] VMware Tools initialization timed out.")
			if sharedfolders {
				log.Println("[WARN] Enabling shared folders is not possible.")
			}
			return rs, nil
		}

		if sharedfolders {
			log.Println("[DEBUG] Enabling shared folders...")

			err = vm.EnableSharedFolders(sharedfolders)
			if err != nil {
				return nil, err
			}
		}
	} else {
		log.Println("[INFO] Virtual machine is already powered on")
	}

	//rs.ConnInfo["type"] = "ssh"
	// rs.ConnInfo["host"] = ?

	return rs, nil
}

func resource_vix_vm_update(
	s *terraform.ResourceState,
	d *terraform.ResourceDiff,
	meta interface{}) (*terraform.ResourceState, error) {
	//p := meta.(*ResourceProvider)

	return nil, nil
}

func resource_vix_vm_destroy(
	s *terraform.ResourceState,
	meta interface{}) error {
	// p := meta.(*ResourceProvider)
	// client := p.client

	return nil
}

func resource_vix_vm_diff(
	s *terraform.ResourceState,
	c *terraform.ResourceConfig,
	meta interface{}) (*terraform.ResourceDiff, error) {

	b := &diff.ResourceBuilder{
		// We have to choose whether a change in an attribute triggers a new
		// resource creation or updates the existing resource.
		Attrs: map[string]diff.AttrType{
			"name":               diff.AttrTypeCreate,
			"description":        diff.AttrTypeUpdate,
			"tools_init_timeout": diff.AttrTypeUpdate,
			"image":              diff.AttrTypeCreate,
			"cpus":               diff.AttrTypeUpdate,
			"memory":             diff.AttrTypeUpdate,
			"networks":           diff.AttrTypeUpdate,
			"upgrade_vhardware":  diff.AttrTypeUpdate,
			"network_driver":     diff.AttrTypeUpdate,
			"sharedfolders":      diff.AttrTypeUpdate,
			"gui":                diff.AttrTypeUpdate,
		},

		ComputedAttrs: []string{
			"ip_address",
		},
	}

	return b.Diff(s, c)
}

func resource_vix_vm_refresh(
	s *terraform.ResourceState,
	meta interface{}) (*terraform.ResourceState, error) {

	return nil, nil
}

func resource_vix_vm_update_state(
	s *terraform.ResourceState,
	vm *vix.VM) (*terraform.ResourceState, error) {

	return nil, nil
}
<commit_msg>Adds some logging for debugging purposes<commit_after>package terraform_vix

import (
	"fmt"
	"log"
	"os/user"
	"path/filepath"
	"strconv"
	"time"

	"github.com/c4milo/govix"
	"github.com/c4milo/terraform_vix/helper"
	"github.com/dustin/go-humanize"
	"github.com/hashicorp/terraform/flatmap"
	"github.com/hashicorp/terraform/helper/config"
	"github.com/hashicorp/terraform/helper/diff"
	"github.com/hashicorp/terraform/terraform"
)

func resource_vix_vm_validation() *config.Validator {
	return &config.Validator{
		Required: []string{
			"name",
			"image.*",
			"image.*.url",
			"image.*.checksum",
			"image.*.checksum_type",
		},
		Optional: []string{
			"description",
			"image.*.password",
			"cpus",
			"memory",
			"upgrade_vhardware",
			"tools_init_timeout",
			"network_driver",
			"networks.*",
			"sharedfolders",
			"sharedfolder.*",
			"gui",
		},
	}
}

func resource_vix_vm_create(
	s *terraform.ResourceState,
	d *terraform.ResourceDiff,
	meta interface{}) (*terraform.ResourceState, error) {
	// Merge the diff into the state so that we have all the attributes
	// properly.
	rs := s.MergeDiff(d)

	name := rs.Attributes["name"]
	description := rs.Attributes["description"]
	cpus, err := strconv.ParseUint(rs.Attributes["cpus"], 0, 8)
	memory := rs.Attributes["memory"]
	toolsInitTimeout, err := time.ParseDuration(rs.Attributes["tools_init_timeout"])
	upgradehw, err := strconv.ParseBool(rs.Attributes["upgrade_vhardware"])
	//netdrv := rs.Attributes["network_driver"]
	launchGUI, err := strconv.ParseBool(rs.Attributes["gui"])
	sharedfolders, err := strconv.ParseBool(rs.Attributes["sharedfolders"])
	var networks []string

	if err != nil {
		return nil, err
	}

	if raw := flatmap.Expand(rs.Attributes, "networks"); raw != nil {
		if nets, ok := raw.([]interface{}); ok {
			for _, net := range nets {
				str, ok := net.(string)
				if !ok {
					continue
				}

				networks = append(networks, str)
			}
		}
	}

	// This is nasty but there doesn't seem to be a cleaner way to extract stuff
	// from the TF configuration
	image := flatmap.Expand(rs.Attributes, "image").([]interface{})[0].(map[string]interface{})

	log.Printf("[DEBUG] networks => %v", networks)

	if len(networks) == 0 {
		networks = append(networks, "bridged")
	}

	usr, err := user.Current()
	if err != nil {
		return nil, err
	}

	// FIXME(c4milo): There is an issue here whenever count is greater than 1
	// please see: https://github.com/hashicorp/terraform/issues/141
	vmPath := filepath.Join(usr.HomeDir, fmt.Sprintf(".terraform/vix/vms/%s", name))

	imageConfig := helper.FetchConfig{
		URL:          image["url"].(string),
		Checksum:     image["checksum"].(string),
		ChecksumType: image["checksum_type"].(string),
		DownloadPath: vmPath,
	}

	vmPath, err = helper.FetchFile(imageConfig)
	if err != nil {
		return nil, err
	}

	// FIXME(c4milo): This has an edge case when a resource with the same
	// name is declared with a different image box, it will return multiple
	// vmx files.
	pattern := filepath.Join(vmPath, "/**/*.vmx")

	log.Printf("[DEBUG] Finding VMX file in %s", pattern)
	files, _ := filepath.Glob(pattern)

	log.Printf("[DEBUG] VMX files found %v", files)

	if len(files) == 0 {
		return nil, fmt.Errorf("[ERROR] VMX file was not found: %s", pattern)
	}

	// Gets VIX instance
	p := meta.(*ResourceProvider)
	client := p.client

	if ((client.Provider & vix.VMWARE_VI_SERVER) == 0) ||
		((client.Provider & vix.VMWARE_SERVER) == 0) {
		log.Printf("[INFO] Registering VM in host's inventory...")
		err = client.RegisterVm(files[0])
		if err != nil {
			return nil, err
		}
	}

	log.Printf("[INFO] Opening virtual machine from %s", files[0])

	vm, err := client.OpenVm(files[0], image["password"].(string))
	if err != nil {
		return nil, err
	}
	defer client.Disconnect()

	memoryInMb, err := humanize.ParseBytes(memory)
	if err != nil {
		log.Printf("[WARN] Unable to set memory size, defaulting to 1g: %s", err)
		memoryInMb = 1024
	} else {
		memoryInMb = (memoryInMb / 1024) / 1024
	}

	log.Printf("[DEBUG] Setting memory size to %d megabytes", memoryInMb)
	vm.SetMemorySize(uint(memoryInMb))

	log.Printf("[DEBUG] Setting vcpus to %d", cpus)
	vm.SetNumberVcpus(uint8(cpus))

	log.Printf("[DEBUG] Setting annotation to %s", description)
	vm.SetAnnotation(description)

	// for _, netType := range networks {
	// 	adapter := &vix.NetworkAdapter{
	// 		//VSwitch:        vix.VSwitch{},
	// 		StartConnected: true,
	// 	}

	// 	switch netdrv {
	// 	case "e1000":
	// 		adapter.Vdevice = vix.NETWORK_DEVICE_E1000
	// 	case "vmxnet3":
	// 		adapter.Vdevice = vix.NETWORK_DEVICE_VMXNET3
	// 	default:
	// 		adapter.Vdevice = vix.NETWORK_DEVICE_E1000
	// 	}

	// 	switch netType {
	// 	case "hostonly":
	// 		adapter.ConnType = vix.NETWORK_HOSTONLY
	// 	case "bridged":
	// 		adapter.ConnType = vix.NETWORK_BRIDGED
	// 	case "nat":
	// 		adapter.ConnType = vix.NETWORK_NAT
	// 	default:
	// 		adapter.ConnType = vix.NETWORK_CUSTOM

	// 	}

	// 	err = vm.AddNetworkAdapter(adapter)
	// 	if err != nil {
	// 		return nil, err
	// 	}
	// }

	running, err := vm.IsRunning()
	if err != nil {
		return nil, err
	}

	if !running {
		if upgradehw &&
			((client.Provider & vix.VMWARE_PLAYER) == 0) {

			log.Println("[INFO] Upgrading virtual hardware...")
			err = vm.UpgradeVHardware()
			if err != nil {
				return nil, err
			}
		}

		log.Println("[INFO] Powering virtual machine on...")
		var options vix.VMPowerOption

		if launchGUI {
			log.Println("[INFO] Preparing to launch GUI...")
			options |= vix.VMPOWEROP_LAUNCH_GUI
		}

		options |= vix.VMPOWEROP_NORMAL

		err = vm.PowerOn(options)
		if err != nil {
			return rs, err
		}

		log.Println("[INFO] Waiting for VMware Tools to initialize...")
		err = vm.WaitForToolsInGuest(toolsInitTimeout)
		if err != nil {
			log.Println("[WARN] VMware Tools initialization timed out.")
			if sharedfolders {
				log.Println("[WARN] Enabling shared folders is not possible.")
			}
			return rs, nil
		}

		if sharedfolders {
			log.Println("[DEBUG] Enabling shared folders...")

			err = vm.EnableSharedFolders(sharedfolders)
			if err != nil {
				return nil, err
			}
		}
	} else {
		log.Println("[INFO] Virtual machine is already powered on")
	}

	//rs.ConnInfo["type"] = "ssh"
	// rs.ConnInfo["host"] = ?

	return rs, nil
}

func resource_vix_vm_update(
	s *terraform.ResourceState,
	d *terraform.ResourceDiff,
	meta interface{}) (*terraform.ResourceState, error) {
	//p := meta.(*ResourceProvider)

	return nil, nil
}

func resource_vix_vm_destroy(
	s *terraform.ResourceState,
	meta interface{}) error {
	// p := meta.(*ResourceProvider)
	// client := p.client

	return nil
}

func resource_vix_vm_diff(
	s *terraform.ResourceState,
	c *terraform.ResourceConfig,
	meta interface{}) (*terraform.ResourceDiff, error) {

	b := &diff.ResourceBuilder{
		// We have to choose whether a change in an attribute triggers a new
		// resource creation or updates the existing resource.
		Attrs: map[string]diff.AttrType{
			"name":               diff.AttrTypeCreate,
			"description":        diff.AttrTypeUpdate,
			"tools_init_timeout": diff.AttrTypeUpdate,
			"image":              diff.AttrTypeCreate,
			"cpus":               diff.AttrTypeUpdate,
			"memory":             diff.AttrTypeUpdate,
			"networks":           diff.AttrTypeUpdate,
			"upgrade_vhardware":  diff.AttrTypeUpdate,
			"network_driver":     diff.AttrTypeUpdate,
			"sharedfolders":      diff.AttrTypeUpdate,
			"gui":                diff.AttrTypeUpdate,
		},

		ComputedAttrs: []string{
			"ip_address",
		},
	}

	return b.Diff(s, c)
}

func resource_vix_vm_refresh(
	s *terraform.ResourceState,
	meta interface{}) (*terraform.ResourceState, error) {

	return nil, nil
}

func resource_vix_vm_update_state(
	s *terraform.ResourceState,
	vm *vix.VM) (*terraform.ResourceState, error) {

	return nil, nil
}
<|endoftext|>


<commit_before>package rest

import (
	"fmt"
	"github.com/the-information/ori/config"
	"golang.org/x/net/context"
	"net/http"
)

// Middleware ensures all requests are formatted properly for a REST/JSON API.
// It requires that inbound requests meet all of the following conditions:
//
//	The request accepts application/json.
//	The request's content type is application/json, if it has a body.
//	The request is encoded in UTF-8.
//	The request accepts UTF-8.
//	The request is properly configured for CORS, if it requires it.
//
// If any of these conditions is not met, Rest will respond with an appropriate
// HTTP error code and error message. Otherwise it will pass control down the line.
func Middleware(ctx context.Context, w http.ResponseWriter, r *http.Request) context.Context {

	w.Header().Set("Accept", "application/json")
	w.Header().Set("Accept-Charset", "UTF-8")
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	var conf config.Global

	if err := config.Get(ctx, &conf); err != nil {
		panic("Could not retrieve Configuration for Rest middleware: " + err.Error())
	}

	if !AcceptsJson(r) || !AcceptsUtf8(r) {

		// If the requester does not accept JSON in the UTF-8	character set, respond with
		// 406 Not Acceptable

		w.WriteHeader(http.StatusNotAcceptable)
		w.Write([]byte(`{"message": "This API only responds with application/json in UTF-8"}`))
		return nil

	} else if r.Method != "HEAD" && r.Method != "GET" && r.Method != "OPTIONS" && !ContentIsJson(r) {

		// If the requester has sent something other than application/json, respond with
		// 415 Unsupported Media Type

		w.WriteHeader(http.StatusUnsupportedMediaType)
		w.Write([]byte(`{"message": "This API only accepts application/json in UTF-8"}`))
		return nil

	} else if r.Header.Get("Origin") != "" && !HasValidOrigin(r, conf.ValidOriginSuffix) {

		// If the requester has sent Origin and the origin is invalid, respond with
		// 400 Bad Request

		w.Header().Set("Access-Control-Allow-Origin", conf.ValidOriginSuffix)
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, `{"message": "Invalid Origin header; this API only accepts %s and its subdomains"}`, conf.ValidOriginSuffix)
		return nil

	} else {

		// The request passes all checks; it can now be processed

		// Since "Access-Control-Allow-Origin" passed, set the request Origin
		// as an allowed origin
		if r.Header.Get("Origin") != "" {
			w.Header().Set("Access-Control-Allow-Origin", r.Header.Get("Origin"))
		}

		if r.Method == "OPTIONS" {
			// Options call. Intercept and do not forward.
			w.Header().Set("Access-Control-Allow-Headers", "Accept, Accept-Charset, Content-Type, Authorization, X-CORS")
			w.Header().Set("Access-Control-Expose-Headers", "Content-Type, Accept, Accept-Charset, Link, Location, X-CORS")
			w.Header().Set("Access-Control-Max-Age", "3600")
			w.WriteHeader(http.StatusNoContent)
			return nil
		}

		return ctx

	}

}
<commit_msg>add access-control-allow-credentials<commit_after>package rest

import (
	"fmt"
	"github.com/the-information/ori/config"
	"golang.org/x/net/context"
	"net/http"
)

// Middleware ensures all requests are formatted properly for a REST/JSON API.
// It requires that inbound requests meet all of the following conditions:
//
//	The request accepts application/json.
//	The request's content type is application/json, if it has a body.
//	The request is encoded in UTF-8.
//	The request accepts UTF-8.
//	The request is properly configured for CORS, if it requires it.
//
// If any of these conditions is not met, Rest will respond with an appropriate
// HTTP error code and error message. Otherwise it will pass control down the line.
func Middleware(ctx context.Context, w http.ResponseWriter, r *http.Request) context.Context {

	w.Header().Set("Accept", "application/json")
	w.Header().Set("Accept-Charset", "UTF-8")
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	var conf config.Global

	if err := config.Get(ctx, &conf); err != nil {
		panic("Could not retrieve Configuration for Rest middleware: " + err.Error())
	}

	if !AcceptsJson(r) || !AcceptsUtf8(r) {

		// If the requester does not accept JSON in the UTF-8	character set, respond with
		// 406 Not Acceptable

		w.WriteHeader(http.StatusNotAcceptable)
		w.Write([]byte(`{"message": "This API only responds with application/json in UTF-8"}`))
		return nil

	} else if r.Method != "HEAD" && r.Method != "GET" && r.Method != "OPTIONS" && !ContentIsJson(r) {

		// If the requester has sent something other than application/json, respond with
		// 415 Unsupported Media Type

		w.WriteHeader(http.StatusUnsupportedMediaType)
		w.Write([]byte(`{"message": "This API only accepts application/json in UTF-8"}`))
		return nil

	} else if r.Header.Get("Origin") != "" && !HasValidOrigin(r, conf.ValidOriginSuffix) {

		// If the requester has sent Origin and the origin is invalid, respond with
		// 400 Bad Request

		w.Header().Set("Access-Control-Allow-Origin", conf.ValidOriginSuffix)
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, `{"message": "Invalid Origin header; this API only accepts %s and its subdomains"}`, conf.ValidOriginSuffix)
		return nil

	} else {

		// The request passes all checks; it can now be processed

		// Since "Access-Control-Allow-Origin" passed, set the request Origin
		// as an allowed origin
		if r.Header.Get("Origin") != "" {
			w.Header().Set("Access-Control-Allow-Origin", r.Header.Get("Origin"))
			w.Header().Set("Access-Control-Allow-Credentials", "true")
		}

		if r.Method == "OPTIONS" {
			// Options call. Intercept and do not forward.
			w.Header().Set("Access-Control-Allow-Headers", "Accept, Accept-Charset, Content-Type, Authorization, X-CORS")
			w.Header().Set("Access-Control-Expose-Headers", "Content-Type, Accept, Accept-Charset, Link, Location, X-CORS")
			w.Header().Set("Access-Control-Max-Age", "3600")
			w.WriteHeader(http.StatusNoContent)
			return nil
		}

		return ctx

	}

}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcs

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"

	"github.com/jacobsa/gcloud/httputil"
	"golang.org/x/net/context"
	"google.golang.org/api/googleapi"
	storagev1 "google.golang.org/api/storage/v1"
)

// Bucket represents a GCS bucket, pre-bound with a bucket name and necessary
// authorization information.
//
// Each method that may block accepts a context object that is used for
// deadlines and cancellation. Users need not package authorization information
// into the context object (using cloud.WithContext or similar).
//
// All methods are safe for concurrent access.
type Bucket interface {
	Name() string

	// Create a reader for the contents of a particular generation of an object.
	// On a nil error, the caller must arrange for the reader to be closed when
	// it is no longer needed.
	//
	// Non-existent objects cause either this method or the first read from the
	// resulting reader to return an error of type *NotFoundError.
	//
	// Official documentation:
	//     https://cloud.google.com/storage/docs/json_api/v1/objects/get
	NewReader(
		ctx context.Context,
		req *ReadObjectRequest) (io.ReadCloser, error)

	// Create or overwrite an object according to the supplied request. The new
	// object is guaranteed to exist immediately for the purposes of reading (and
	// eventually for listing) after this method returns a nil error. It is
	// guaranteed not to exist before req.Contents returns io.EOF.
	//
	// Official documentation:
	//     https://cloud.google.com/storage/docs/json_api/v1/objects/insert
	//     https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload
	CreateObject(
		ctx context.Context,
		req *CreateObjectRequest) (*Object, error)

	// Copy an object to a new name, preserving all metadata. Any existing
	// generation of the destination name will be overwritten.
	//
	// Returns a record for the new object.
	//
	// Official documentation:
	//     https://cloud.google.com/storage/docs/json_api/v1/objects/copy
	CopyObject(
		ctx context.Context,
		req *CopyObjectRequest) (*Object, error)

	// Compose one or more source objects into a single destination object by
	// concatenating. Any existing generation of the destination name will be
	// overwritten.
	//
	// Returns a record for the new object.
	//
	// Official documentation:
	//     https://cloud.google.com/storage/docs/json_api/v1/objects/compose
	ComposeObjects(
		ctx context.Context,
		req *ComposeObjectsRequest) (*Object, error)

	// Return current information about the object with the given name.
	//
	// Official documentation:
	//     https://cloud.google.com/storage/docs/json_api/v1/objects/get
	StatObject(
		ctx context.Context,
		req *StatObjectRequest) (*Object, error)

	// List the objects in the bucket that meet the criteria defined by the
	// request, returning a result object that contains the results and
	// potentially a cursor for retrieving the next portion of the larger set of
	// results.
	//
	// Official documentation:
	//     https://cloud.google.com/storage/docs/json_api/v1/objects/list
	ListObjects(
		ctx context.Context,
		req *ListObjectsRequest) (*Listing, error)

	// Update the object specified by newAttrs.Name, patching using the non-zero
	// fields of newAttrs.
	//
	// Official documentation:
	//     https://cloud.google.com/storage/docs/json_api/v1/objects/patch
	UpdateObject(
		ctx context.Context,
		req *UpdateObjectRequest) (*Object, error)

	// Delete an object. Non-existence of the object is not treated as an error.
	//
	// Official documentation:
	//     https://cloud.google.com/storage/docs/json_api/v1/objects/delete
	DeleteObject(
		ctx context.Context,
		req *DeleteObjectRequest) error
}

type bucket struct {
	client         *http.Client
	url            *url.URL
	userAgent      string
	name           string
	billingProject string
}

func (b *bucket) Name() string {
	return b.name
}

func (b *bucket) ListObjects(
	ctx context.Context,
	req *ListObjectsRequest) (listing *Listing, err error) {
	// Construct an appropriate URL (cf. http://goo.gl/aVSAhT).
	opaque := fmt.Sprintf(
		"//%s/storage/v1/b/%s/o",
		b.url.Host,
		httputil.EncodePathSegment(b.Name()))

	query := make(url.Values)
	query.Set("projection", req.ProjectionVal.String())

	if req.Prefix != "" {
		query.Set("prefix", req.Prefix)
	}

	if req.Delimiter != "" {
		query.Set("delimiter", req.Delimiter)
		query.Set("includeTrailingDelimiter",
			fmt.Sprintf("%v", req.IncludeTrailingDelimiter))
	}

	if req.ContinuationToken != "" {
		query.Set("pageToken", req.ContinuationToken)
	}

	if req.MaxResults != 0 {
		query.Set("maxResults", fmt.Sprintf("%v", req.MaxResults))
	}

	if b.billingProject != "" {
		query.Set("userProject", b.billingProject)
	}

	url := &url.URL{
		Scheme:   b.url.Scheme,
		Host:     b.url.Host,
		Opaque:   opaque,
		RawQuery: query.Encode(),
	}

	// Create an HTTP request.
	httpReq, err := httputil.NewRequest(ctx, "GET", url, nil, 0, b.userAgent)
	if err != nil {
		err = fmt.Errorf("httputil.NewRequest: %v", err)
		return
	}

	// Call the server.
	httpRes, err := b.client.Do(httpReq)
	if err != nil {
		return
	}

	defer googleapi.CloseBody(httpRes)

	// Check for HTTP-level errors.
	if err = googleapi.CheckResponse(httpRes); err != nil {
		return
	}

	// Parse the response.
	var rawListing *storagev1.Objects
	if err = json.NewDecoder(httpRes.Body).Decode(&rawListing); err != nil {
		return
	}

	// Convert the response.
	if listing, err = toListing(rawListing); err != nil {
		return
	}

	return
}

func (b *bucket) StatObject(
	ctx context.Context,
	req *StatObjectRequest) (o *Object, err error) {
	// Construct an appropriate URL (cf. http://goo.gl/MoITmB).
	opaque := fmt.Sprintf(
		"//%s/storage/v1/b/%s/o/%s",
		b.url.Host,
		httputil.EncodePathSegment(b.Name()),
		httputil.EncodePathSegment(req.Name))

	query := make(url.Values)
	query.Set("projection", "full")

	if b.billingProject != "" {
		query.Set("userProject", b.billingProject)
	}

	url := &url.URL{
		Scheme:   b.url.Scheme,
		Host:     b.url.Host,
		Opaque:   opaque,
		RawQuery: query.Encode(),
	}

	// Create an HTTP request.
	httpReq, err := httputil.NewRequest(ctx, "GET", url, nil, 0, b.userAgent)
	if err != nil {
		err = fmt.Errorf("httputil.NewRequest: %v", err)
		return
	}

	// Execute the HTTP request.
	httpRes, err := b.client.Do(httpReq)
	if err != nil {
		return
	}

	defer googleapi.CloseBody(httpRes)

	// Check for HTTP-level errors.
	if err = googleapi.CheckResponse(httpRes); err != nil {
		// Special case: handle not found errors.
		if typed, ok := err.(*googleapi.Error); ok {
			if typed.Code == http.StatusNotFound {
				err = &NotFoundError{Err: typed}
			}
		}

		return
	}

	// Parse the response.
	var rawObject *storagev1.Object
	if err = json.NewDecoder(httpRes.Body).Decode(&rawObject); err != nil {
		return
	}

	// Convert the response.
	if o, err = toObject(rawObject); err != nil {
		err = fmt.Errorf("toObject: %v", err)
		return
	}

	return
}

func (b *bucket) DeleteObject(
	ctx context.Context,
	req *DeleteObjectRequest) (err error) {
	// Construct an appropriate URL (cf. http://goo.gl/TRQJjZ).
	opaque := fmt.Sprintf(
		"//%s/storage/v1/b/%s/o/%s",
		b.url.Host,
		httputil.EncodePathSegment(b.Name()),
		httputil.EncodePathSegment(req.Name))

	query := make(url.Values)

	if req.Generation != 0 {
		query.Set("generation", fmt.Sprintf("%d", req.Generation))
	}

	if req.MetaGenerationPrecondition != nil {
		query.Set(
			"ifMetagenerationMatch",
			fmt.Sprintf("%d", *req.MetaGenerationPrecondition))
	}

	if b.billingProject != "" {
		query.Set("userProject", b.billingProject)
	}

	url := &url.URL{
		Scheme:   b.url.Scheme,
		Host:     b.url.Host,
		Opaque:   opaque,
		RawQuery: query.Encode(),
	}

	// Create an HTTP request.
	httpReq, err := httputil.NewRequest(ctx, "DELETE", url, nil, 0, b.userAgent)
	if err != nil {
		err = fmt.Errorf("httputil.NewRequest: %v", err)
		return
	}

	// Execute the HTTP request.
	httpRes, err := b.client.Do(httpReq)
	if err != nil {
		return
	}

	defer googleapi.CloseBody(httpRes)

	// Check for HTTP-level errors.
	err = googleapi.CheckResponse(httpRes)

	// Special case: we want deletes to be idempotent.
	if typed, ok := err.(*googleapi.Error); ok {
		if typed.Code == http.StatusNotFound {
			err = nil
		}
	}

	// Special case: handle precondition errors.
	if typed, ok := err.(*googleapi.Error); ok {
		if typed.Code == http.StatusPreconditionFailed {
			err = &PreconditionError{Err: typed}
		}
	}

	// Propagate other errors.
	if err != nil {
		return
	}

	return
}

func newBucket(
	client *http.Client,
	url *url.URL,
	userAgent string,
	name string,
	billingProject string) Bucket {
	return &bucket{
		client:         client,
		url:            url,
		userAgent:      userAgent,
		name:           name,
		billingProject: billingProject,
	}
}
<commit_msg>StatObject method with Go client library<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcs

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"time"

	"cloud.google.com/go/storage"
	"github.com/jacobsa/gcloud/httputil"
	"golang.org/x/net/context"
	"google.golang.org/api/googleapi"
	storagev1 "google.golang.org/api/storage/v1"
)

// Bucket represents a GCS bucket, pre-bound with a bucket name and necessary
// authorization information.
//
// Each method that may block accepts a context object that is used for
// deadlines and cancellation. Users need not package authorization information
// into the context object (using cloud.WithContext or similar).
//
// All methods are safe for concurrent access.
type Bucket interface {
	Name() string

	// Create a reader for the contents of a particular generation of an object.
	// On a nil error, the caller must arrange for the reader to be closed when
	// it is no longer needed.
	//
	// Non-existent objects cause either this method or the first read from the
	// resulting reader to return an error of type *NotFoundError.
	//
	// Official documentation:
	//     https://cloud.google.com/storage/docs/json_api/v1/objects/get
	NewReader(
			ctx context.Context,
			req *ReadObjectRequest) (io.ReadCloser, error)

	// Create or overwrite an object according to the supplied request. The new
	// object is guaranteed to exist immediately for the purposes of reading (and
	// eventually for listing) after this method returns a nil error. It is
	// guaranteed not to exist before req.Contents returns io.EOF.
	//
	// Official documentation:
	//     https://cloud.google.com/storage/docs/json_api/v1/objects/insert
	//     https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload
	CreateObject(
			ctx context.Context,
			req *CreateObjectRequest) (*Object, error)

	// Copy an object to a new name, preserving all metadata. Any existing
	// generation of the destination name will be overwritten.
	//
	// Returns a record for the new object.
	//
	// Official documentation:
	//     https://cloud.google.com/storage/docs/json_api/v1/objects/copy
	CopyObject(
			ctx context.Context,
			req *CopyObjectRequest) (*Object, error)

	// Compose one or more source objects into a single destination object by
	// concatenating. Any existing generation of the destination name will be
	// overwritten.
	//
	// Returns a record for the new object.
	//
	// Official documentation:
	//     https://cloud.google.com/storage/docs/json_api/v1/objects/compose
	ComposeObjects(
			ctx context.Context,
			req *ComposeObjectsRequest) (*Object, error)

	// Return current information about the object with the given name.
	//
	// Official documentation:
	//     https://cloud.google.com/storage/docs/json_api/v1/objects/get
	StatObject(
			ctx context.Context,
			req *StatObjectRequest) (*Object, error)

	// List the objects in the bucket that meet the criteria defined by the
	// request, returning a result object that contains the results and
	// potentially a cursor for retrieving the next portion of the larger set of
	// results.
	//
	// Official documentation:
	//     https://cloud.google.com/storage/docs/json_api/v1/objects/list
	ListObjects(
			ctx context.Context,
			req *ListObjectsRequest) (*Listing, error)

	// Update the object specified by newAttrs.Name, patching using the non-zero
	// fields of newAttrs.
	//
	// Official documentation:
	//     https://cloud.google.com/storage/docs/json_api/v1/objects/patch
	UpdateObject(
			ctx context.Context,
			req *UpdateObjectRequest) (*Object, error)

	// Delete an object. Non-existence of the object is not treated as an error.
	//
	// Official documentation:
	//     https://cloud.google.com/storage/docs/json_api/v1/objects/delete
	DeleteObject(
			ctx context.Context,
			req *DeleteObjectRequest) error
}

type bucket struct {
	client         *http.Client
	url            *url.URL
	userAgent      string
	name           string
	billingProject string
}

func ObjectAttrsToBucketObject(attrs *storage.ObjectAttrs) *Object {
	// Converting []ACLRule returned by the Go Client into []*storagev1.ObjectAccessControl which complies with GCSFuse type.
	var Acl []*storagev1.ObjectAccessControl
	for _, element := range attrs.ACL {
		currACL := &storagev1.ObjectAccessControl{
			Entity:   string(element.Entity),
			EntityId: element.EntityID,
			Role:     string(element.Role),
			Domain:   element.Domain,
			Email:    element.Email,
			ProjectTeam: &storagev1.ObjectAccessControlProjectTeam{
				ProjectNumber: element.ProjectTeam.ProjectNumber,
				Team:          element.ProjectTeam.Team,
			},
		}
		Acl = append(Acl, currACL)
	}

	// Converting MD5[] slice to MD5[md5.Size] type fixed array as accepted by GCSFuse.
	var MD5 [md5.Size]byte
	copy(MD5[:], attrs.MD5)

	// Setting the parameters in Object and doing conversions as necessary.
	return &Object{
		Name:            attrs.Name,
		ContentType:     attrs.ContentType,
		ContentLanguage: attrs.ContentLanguage,
		CacheControl:    attrs.CacheControl,
		Owner:           attrs.Owner,
		Size:            uint64(attrs.Size),
		ContentEncoding: attrs.ContentEncoding,
		MD5:             &MD5,
		CRC32C:          &attrs.CRC32C,
		MediaLink:       attrs.MediaLink,
		Metadata:        attrs.Metadata,
		Generation:      attrs.Generation,
		MetaGeneration:  attrs.Metageneration,
		StorageClass:    attrs.StorageClass,
		Deleted:         attrs.Deleted,
		Updated:         attrs.Updated,
		//ComponentCount: , (Field not found in attrs returned by Go Client.)
		ContentDisposition: attrs.ContentDisposition,
		CustomTime:         string(attrs.CustomTime.Format(time.RFC3339)),
		EventBasedHold:     attrs.EventBasedHold,
		Acl:                Acl,
	}
}

func (b *bucket) Name() string {
	return b.name
}

func (b *bucket) ListObjects(
		ctx context.Context,
		req *ListObjectsRequest) (listing *Listing, err error) {
	// Construct an appropriate URL (cf. http://goo.gl/aVSAhT).
	opaque := fmt.Sprintf(
		"//%s/storage/v1/b/%s/o",
		b.url.Host,
		httputil.EncodePathSegment(b.Name()))

	query := make(url.Values)
	query.Set("projection", req.ProjectionVal.String())

	if req.Prefix != "" {
		query.Set("prefix", req.Prefix)
	}

	if req.Delimiter != "" {
		query.Set("delimiter", req.Delimiter)
		query.Set("includeTrailingDelimiter",
			fmt.Sprintf("%v", req.IncludeTrailingDelimiter))
	}

	if req.ContinuationToken != "" {
		query.Set("pageToken", req.ContinuationToken)
	}

	if req.MaxResults != 0 {
		query.Set("maxResults", fmt.Sprintf("%v", req.MaxResults))
	}

	if b.billingProject != "" {
		query.Set("userProject", b.billingProject)
	}

	url := &url.URL{
		Scheme:   b.url.Scheme,
		Host:     b.url.Host,
		Opaque:   opaque,
		RawQuery: query.Encode(),
	}

	// Create an HTTP request.
	httpReq, err := httputil.NewRequest(ctx, "GET", url, nil, 0, b.userAgent)
	if err != nil {
		err = fmt.Errorf("httputil.NewRequest: %v", err)
		return
	}

	// Call the server.
	httpRes, err := b.client.Do(httpReq)
	if err != nil {
		return
	}

	defer googleapi.CloseBody(httpRes)

	// Check for HTTP-level errors.
	if err = googleapi.CheckResponse(httpRes); err != nil {
		return
	}

	// Parse the response.
	var rawListing *storagev1.Objects
	if err = json.NewDecoder(httpRes.Body).Decode(&rawListing); err != nil {
		return
	}

	// Convert the response.
	if listing, err = toListing(rawListing); err != nil {
		return
	}

	return
}

func (b *bucket) StatObject(
		ctx context.Context,
		req *StatObjectRequest) (o *Object, err error) {
	// Construct an appropriate URL (cf. http://goo.gl/MoITmB).
	opaque := fmt.Sprintf(
		"//%s/storage/v1/b/%s/o/%s",
		b.url.Host,
		httputil.EncodePathSegment(b.Name()),
		httputil.EncodePathSegment(req.Name))

	query := make(url.Values)
	query.Set("projection", "full")

	if b.billingProject != "" {
		query.Set("userProject", b.billingProject)
	}

	url := &url.URL{
		Scheme:   b.url.Scheme,
		Host:     b.url.Host,
		Opaque:   opaque,
		RawQuery: query.Encode(),
	}

	// Create an HTTP request.
	httpReq, err := httputil.NewRequest(ctx, "GET", url, nil, 0, b.userAgent)
	if err != nil {
		err = fmt.Errorf("httputil.NewRequest: %v", err)
		return
	}

	// Execute the HTTP request.
	httpRes, err := b.client.Do(httpReq)
	if err != nil {
		return
	}

	defer googleapi.CloseBody(httpRes)

	// Check for HTTP-level errors.
	if err = googleapi.CheckResponse(httpRes); err != nil {
		// Special case: handle not found errors.
		if typed, ok := err.(*googleapi.Error); ok {
			if typed.Code == http.StatusNotFound {
				err = &NotFoundError{Err: typed}
			}
		}

		return
	}

	// Parse the response.
	var rawObject *storagev1.Object
	if err = json.NewDecoder(httpRes.Body).Decode(&rawObject); err != nil {
		return
	}

	// Convert the response.
	if o, err = toObject(rawObject); err != nil {
		err = fmt.Errorf("toObject: %v", err)
		return
	}

	return
}

func (b *bucket) DeleteObject(
		ctx context.Context,
		req *DeleteObjectRequest) (err error) {
	// Construct an appropriate URL (cf. http://goo.gl/TRQJjZ).
	opaque := fmt.Sprintf(
		"//%s/storage/v1/b/%s/o/%s",
		b.url.Host,
		httputil.EncodePathSegment(b.Name()),
		httputil.EncodePathSegment(req.Name))

	query := make(url.Values)

	if req.Generation != 0 {
		query.Set("generation", fmt.Sprintf("%d", req.Generation))
	}

	if req.MetaGenerationPrecondition != nil {
		query.Set(
			"ifMetagenerationMatch",
			fmt.Sprintf("%d", *req.MetaGenerationPrecondition))
	}

	if b.billingProject != "" {
		query.Set("userProject", b.billingProject)
	}

	url := &url.URL{
		Scheme:   b.url.Scheme,
		Host:     b.url.Host,
		Opaque:   opaque,
		RawQuery: query.Encode(),
	}

	// Create an HTTP request.
	httpReq, err := httputil.NewRequest(ctx, "DELETE", url, nil, 0, b.userAgent)
	if err != nil {
		err = fmt.Errorf("httputil.NewRequest: %v", err)
		return
	}

	// Execute the HTTP request.
	httpRes, err := b.client.Do(httpReq)
	if err != nil {
		return
	}

	defer googleapi.CloseBody(httpRes)

	// Check for HTTP-level errors.
	err = googleapi.CheckResponse(httpRes)

	// Special case: we want deletes to be idempotent.
	if typed, ok := err.(*googleapi.Error); ok {
		if typed.Code == http.StatusNotFound {
			err = nil
		}
	}

	// Special case: handle precondition errors.
	if typed, ok := err.(*googleapi.Error); ok {
		if typed.Code == http.StatusPreconditionFailed {
			err = &PreconditionError{Err: typed}
		}
	}

	// Propagate other errors.
	if err != nil {
		return
	}

	return
}

func newBucket(
		client *http.Client,
		url *url.URL,
		userAgent string,
		name string,
		billingProject string) Bucket {
	return &bucket{
		client:         client,
		url:            url,
		userAgent:      userAgent,
		name:           name,
		billingProject: billingProject,
	}
}<|endoftext|>


<commit_before>package coreapi

import (
	"context"
	"fmt"

	coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
	corerepo "github.com/ipfs/go-ipfs/core/corerepo"
	merkledag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag"
	bserv "gx/ipfs/QmcRecCZWM2NZfCQrCe97Ch3Givv8KKEP82tGUDntzdLFe/go-blockservice"

	cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid"
	offline "gx/ipfs/QmR5miWuikPxWyUrzMYJVmFUcD44pGdtc98h9Qsbp4YcJw/go-ipfs-exchange-offline"
)

type PinAPI CoreAPI

func (api *PinAPI) Add(ctx context.Context, p coreiface.Path, opts ...caopts.PinAddOption) error {
	settings, err := caopts.PinAddOptions(opts...)
	if err != nil {
		return err
	}

	defer api.node.Blockstore.PinLock().Unlock()

	rp, err := api.core().ResolvePath(ctx, p)
	if err != nil {
		return err
	}

	_, err = corerepo.Pin(api.node, api.core(), ctx, []string{rp.Cid().String()}, settings.Recursive)
	if err != nil {
		return err
	}

	return api.node.Pinning.Flush()
}

func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]coreiface.Pin, error) {
	settings, err := caopts.PinLsOptions(opts...)
	if err != nil {
		return nil, err
	}

	switch settings.Type {
	case "all", "direct", "indirect", "recursive":
	default:
		return nil, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", settings.Type)
	}

	return api.pinLsAll(settings.Type, ctx)
}

func (api *PinAPI) Rm(ctx context.Context, p coreiface.Path) error {
	_, err := corerepo.Unpin(api.node, api.core(), ctx, []string{p.String()}, true)
	if err != nil {
		return err
	}

	return api.node.Pinning.Flush()
}

func (api *PinAPI) Update(ctx context.Context, from coreiface.Path, to coreiface.Path, opts ...caopts.PinUpdateOption) error {
	settings, err := caopts.PinUpdateOptions(opts...)
	if err != nil {
		return err
	}

	fp, err := api.core().ResolvePath(ctx, from)
	if err != nil {
		return err
	}

	tp, err := api.core().ResolvePath(ctx, to)
	if err != nil {
		return err
	}

	err = api.node.Pinning.Update(ctx, fp.Cid(), tp.Cid(), settings.Unpin)
	if err != nil {
		return err
	}

	return api.node.Pinning.Flush()
}

type pinStatus struct {
	cid      cid.Cid
	ok       bool
	badNodes []coreiface.BadPinNode
}

// BadNode is used in PinVerifyRes
type badNode struct {
	path coreiface.ResolvedPath
	err  error
}

func (s *pinStatus) Ok() bool {
	return s.ok
}

func (s *pinStatus) BadNodes() []coreiface.BadPinNode {
	return s.badNodes
}

func (n *badNode) Path() coreiface.ResolvedPath {
	return n.path
}

func (n *badNode) Err() error {
	return n.err
}

func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, error) {
	visited := make(map[string]*pinStatus)
	bs := api.node.Blocks.Blockstore()
	DAG := merkledag.NewDAGService(bserv.New(bs, offline.Exchange(bs)))
	getLinks := merkledag.GetLinksWithDAG(DAG)
	recPins := api.node.Pinning.RecursiveKeys()

	var checkPin func(root cid.Cid) *pinStatus
	checkPin = func(root cid.Cid) *pinStatus {
		key := root.String()
		if status, ok := visited[key]; ok {
			return status
		}

		links, err := getLinks(ctx, root)
		if err != nil {
			status := &pinStatus{ok: false, cid: root}
			status.badNodes = []coreiface.BadPinNode{&badNode{path: coreiface.IpldPath(root), err: err}}
			visited[key] = status
			return status
		}

		status := &pinStatus{ok: true, cid: root}
		for _, lnk := range links {
			res := checkPin(lnk.Cid)
			if !res.ok {
				status.ok = false
				status.badNodes = append(status.badNodes, res.badNodes...)
			}
		}

		visited[key] = status
		return status
	}

	out := make(chan coreiface.PinStatus)
	go func() {
		defer close(out)
		for _, c := range recPins {
			out <- checkPin(c)
		}
	}()

	return out, nil
}

type pinInfo struct {
	pinType string
	path    coreiface.ResolvedPath
}

func (p *pinInfo) Path() coreiface.ResolvedPath {
	return p.path
}

func (p *pinInfo) Type() string {
	return p.pinType
}

func (api *PinAPI) pinLsAll(typeStr string, ctx context.Context) ([]coreiface.Pin, error) {

	keys := make(map[string]*pinInfo)

	AddToResultKeys := func(keyList []cid.Cid, typeStr string) {
		for _, c := range keyList {
			keys[c.String()] = &pinInfo{
				pinType: typeStr,
				path:    coreiface.IpldPath(c),
			}
		}
	}

	if typeStr == "direct" || typeStr == "all" {
		AddToResultKeys(api.node.Pinning.DirectKeys(), "direct")
	}
	if typeStr == "indirect" || typeStr == "all" {
		set := cid.NewSet()
		for _, k := range api.node.Pinning.RecursiveKeys() {
			err := merkledag.EnumerateChildren(ctx, merkledag.GetLinksWithDAG(api.node.DAG), k, set.Visit)
			if err != nil {
				return nil, err
			}
		}
		AddToResultKeys(set.Keys(), "indirect")
	}
	if typeStr == "recursive" || typeStr == "all" {
		AddToResultKeys(api.node.Pinning.RecursiveKeys(), "recursive")
	}

	out := make([]coreiface.Pin, 0, len(keys))
	for _, v := range keys {
		out = append(out, v)
	}

	return out, nil
}

func (api *PinAPI) core() coreiface.CoreAPI {
	return (*CoreAPI)(api)
}
<commit_msg>take the pinlock when updating pins<commit_after>package coreapi

import (
	"context"
	"fmt"

	coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
	corerepo "github.com/ipfs/go-ipfs/core/corerepo"
	merkledag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag"
	bserv "gx/ipfs/QmcRecCZWM2NZfCQrCe97Ch3Givv8KKEP82tGUDntzdLFe/go-blockservice"

	cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid"
	offline "gx/ipfs/QmR5miWuikPxWyUrzMYJVmFUcD44pGdtc98h9Qsbp4YcJw/go-ipfs-exchange-offline"
)

type PinAPI CoreAPI

func (api *PinAPI) Add(ctx context.Context, p coreiface.Path, opts ...caopts.PinAddOption) error {
	settings, err := caopts.PinAddOptions(opts...)
	if err != nil {
		return err
	}

	defer api.node.Blockstore.PinLock().Unlock()

	rp, err := api.core().ResolvePath(ctx, p)
	if err != nil {
		return err
	}

	_, err = corerepo.Pin(api.node, api.core(), ctx, []string{rp.Cid().String()}, settings.Recursive)
	if err != nil {
		return err
	}

	return api.node.Pinning.Flush()
}

func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]coreiface.Pin, error) {
	settings, err := caopts.PinLsOptions(opts...)
	if err != nil {
		return nil, err
	}

	switch settings.Type {
	case "all", "direct", "indirect", "recursive":
	default:
		return nil, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", settings.Type)
	}

	return api.pinLsAll(settings.Type, ctx)
}

func (api *PinAPI) Rm(ctx context.Context, p coreiface.Path) error {
	_, err := corerepo.Unpin(api.node, api.core(), ctx, []string{p.String()}, true)
	if err != nil {
		return err
	}

	return api.node.Pinning.Flush()
}

func (api *PinAPI) Update(ctx context.Context, from coreiface.Path, to coreiface.Path, opts ...caopts.PinUpdateOption) error {
	settings, err := caopts.PinUpdateOptions(opts...)
	if err != nil {
		return err
	}

	defer api.node.Blockstore.PinLock().Unlock()

	fp, err := api.core().ResolvePath(ctx, from)
	if err != nil {
		return err
	}

	tp, err := api.core().ResolvePath(ctx, to)
	if err != nil {
		return err
	}

	err = api.node.Pinning.Update(ctx, fp.Cid(), tp.Cid(), settings.Unpin)
	if err != nil {
		return err
	}

	return api.node.Pinning.Flush()
}

type pinStatus struct {
	cid      cid.Cid
	ok       bool
	badNodes []coreiface.BadPinNode
}

// BadNode is used in PinVerifyRes
type badNode struct {
	path coreiface.ResolvedPath
	err  error
}

func (s *pinStatus) Ok() bool {
	return s.ok
}

func (s *pinStatus) BadNodes() []coreiface.BadPinNode {
	return s.badNodes
}

func (n *badNode) Path() coreiface.ResolvedPath {
	return n.path
}

func (n *badNode) Err() error {
	return n.err
}

func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, error) {
	visited := make(map[string]*pinStatus)
	bs := api.node.Blocks.Blockstore()
	DAG := merkledag.NewDAGService(bserv.New(bs, offline.Exchange(bs)))
	getLinks := merkledag.GetLinksWithDAG(DAG)
	recPins := api.node.Pinning.RecursiveKeys()

	var checkPin func(root cid.Cid) *pinStatus
	checkPin = func(root cid.Cid) *pinStatus {
		key := root.String()
		if status, ok := visited[key]; ok {
			return status
		}

		links, err := getLinks(ctx, root)
		if err != nil {
			status := &pinStatus{ok: false, cid: root}
			status.badNodes = []coreiface.BadPinNode{&badNode{path: coreiface.IpldPath(root), err: err}}
			visited[key] = status
			return status
		}

		status := &pinStatus{ok: true, cid: root}
		for _, lnk := range links {
			res := checkPin(lnk.Cid)
			if !res.ok {
				status.ok = false
				status.badNodes = append(status.badNodes, res.badNodes...)
			}
		}

		visited[key] = status
		return status
	}

	out := make(chan coreiface.PinStatus)
	go func() {
		defer close(out)
		for _, c := range recPins {
			out <- checkPin(c)
		}
	}()

	return out, nil
}

type pinInfo struct {
	pinType string
	path    coreiface.ResolvedPath
}

func (p *pinInfo) Path() coreiface.ResolvedPath {
	return p.path
}

func (p *pinInfo) Type() string {
	return p.pinType
}

func (api *PinAPI) pinLsAll(typeStr string, ctx context.Context) ([]coreiface.Pin, error) {

	keys := make(map[string]*pinInfo)

	AddToResultKeys := func(keyList []cid.Cid, typeStr string) {
		for _, c := range keyList {
			keys[c.String()] = &pinInfo{
				pinType: typeStr,
				path:    coreiface.IpldPath(c),
			}
		}
	}

	if typeStr == "direct" || typeStr == "all" {
		AddToResultKeys(api.node.Pinning.DirectKeys(), "direct")
	}
	if typeStr == "indirect" || typeStr == "all" {
		set := cid.NewSet()
		for _, k := range api.node.Pinning.RecursiveKeys() {
			err := merkledag.EnumerateChildren(ctx, merkledag.GetLinksWithDAG(api.node.DAG), k, set.Visit)
			if err != nil {
				return nil, err
			}
		}
		AddToResultKeys(set.Keys(), "indirect")
	}
	if typeStr == "recursive" || typeStr == "all" {
		AddToResultKeys(api.node.Pinning.RecursiveKeys(), "recursive")
	}

	out := make([]coreiface.Pin, 0, len(keys))
	for _, v := range keys {
		out = append(out, v)
	}

	return out, nil
}

func (api *PinAPI) core() coreiface.CoreAPI {
	return (*CoreAPI)(api)
}
<|endoftext|>


<commit_before>/* Copyright 2016-2017 Vector Creations Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package gomatrixserverlib

import (
	"encoding/json"
	"fmt"
	"golang.org/x/crypto/ed25519"
)

// A KeyID is the ID of a ed25519 key used to sign JSON.
// The key IDs have a format of "ed25519:[0-9A-Za-z]+"
// If we switch to using a different signing algorithm then we will change the
// prefix used.
type KeyID string

// SignJSON signs a JSON object returning a copy signed with the given key.
// https://matrix.org/docs/spec/server_server/unstable.html#signing-json
func SignJSON(signingName string, keyID KeyID, privateKey ed25519.PrivateKey, message []byte) ([]byte, error) {
	var object map[string]*json.RawMessage
	var signatures map[string]map[KeyID]Base64String
	if err := json.Unmarshal(message, &object); err != nil {
		return nil, err
	}

	rawUnsigned, hasUnsigned := object["unsigned"]
	delete(object, "unsigned")

	if rawSignatures := object["signatures"]; rawSignatures != nil {
		if err := json.Unmarshal(*rawSignatures, &signatures); err != nil {
			return nil, err
		}
		delete(object, "signatures")
	} else {
		signatures = map[string]map[KeyID]Base64String{}
	}

	unsorted, err := json.Marshal(object)
	if err != nil {
		return nil, err
	}

	canonical, err := CanonicalJSON(unsorted)
	if err != nil {
		return nil, err
	}

	signature := Base64String(ed25519.Sign(privateKey, canonical))

	signaturesForEntity := signatures[signingName]
	if signaturesForEntity != nil {
		signaturesForEntity[keyID] = signature
	} else {
		signatures[signingName] = map[KeyID]Base64String{keyID: signature}
	}

	var rawSignatures json.RawMessage
	rawSignatures, err = json.Marshal(signatures)
	if err != nil {
		return nil, err
	}

	object["signatures"] = &rawSignatures
	if hasUnsigned {
		object["unsigned"] = rawUnsigned
	}

	return json.Marshal(object)
}

// ListKeyIDs lists the key IDs a given entity has signed a message with.
func ListKeyIDs(signingName string, message []byte) ([]KeyID, error) {
	var object struct {
		Signatures map[string]map[KeyID]json.RawMessage `json:"signatures"`
	}
	if err := json.Unmarshal(message, &object); err != nil {
		return nil, err
	}
	var result []KeyID
	for keyID := range object.Signatures[signingName] {
		result = append(result, keyID)
	}
	return result, nil
}

// VerifyJSON checks that the entity has signed the message using a particular key.
func VerifyJSON(signingName string, keyID KeyID, publicKey ed25519.PublicKey, message []byte) error {
	var object map[string]*json.RawMessage
	var signatures map[string]map[KeyID]Base64String
	if err := json.Unmarshal(message, &object); err != nil {
		return err
	}
	delete(object, "unsigned")

	if object["signatures"] == nil {
		return fmt.Errorf("No signatures")
	}

	if err := json.Unmarshal(*object["signatures"], &signatures); err != nil {
		return err
	}
	delete(object, "signatures")

	signature, ok := signatures[signingName][keyID]
	if !ok {
		return fmt.Errorf("No signature from %q with ID %q", signingName, keyID)
	}

	if len(signature) != ed25519.SignatureSize {
		return fmt.Errorf("Bad signature length from %q with ID %q", signingName, keyID)
	}

	unsorted, err := json.Marshal(object)
	if err != nil {
		return err
	}

	canonical, err := CanonicalJSON(unsorted)
	if err != nil {
		return err
	}

	if !ed25519.Verify(publicKey, canonical, signature) {
		return fmt.Errorf("Bad signature from %q with ID %q", signingName, keyID)
	}

	return nil
}
<commit_msg>Add comments to explain more of SignJSON and VerifyJSON (#48)<commit_after>/* Copyright 2016-2017 Vector Creations Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package gomatrixserverlib

import (
	"encoding/json"
	"fmt"
	"golang.org/x/crypto/ed25519"
)

// A KeyID is the ID of a ed25519 key used to sign JSON.
// The key IDs have a format of "ed25519:[0-9A-Za-z]+"
// If we switch to using a different signing algorithm then we will change the
// prefix used.
type KeyID string

// SignJSON signs a JSON object returning a copy signed with the given key.
// https://matrix.org/docs/spec/server_server/unstable.html#signing-json
func SignJSON(signingName string, keyID KeyID, privateKey ed25519.PrivateKey, message []byte) ([]byte, error) {
	// Unpack the top-level key of the JSON object without unpacking the contents of the keys.
	// This allows us to add and remove the top-level keys from the JSON object.
	// It also ensures that the JSON is actually a valid JSON object.
	var object map[string]*json.RawMessage
	var signatures map[string]map[KeyID]Base64String
	if err := json.Unmarshal(message, &object); err != nil {
		return nil, err
	}

	// We don't sign the contents of the unsigned key so we remove it.
	rawUnsigned, hasUnsigned := object["unsigned"]
	delete(object, "unsigned")

	// Parse the existing signatures if they exist.
	// Signing a JSON object adds our signature to the existing
	// signature rather than replacing it.
	if rawSignatures := object["signatures"]; rawSignatures != nil {
		if err := json.Unmarshal(*rawSignatures, &signatures); err != nil {
			return nil, err
		}
		delete(object, "signatures")
	} else {
		signatures = map[string]map[KeyID]Base64String{}
	}

	// Encode the JSON object without the "signatures" key or
	// the "unsigned" key in the canonical format.
	unsorted, err := json.Marshal(object)
	if err != nil {
		return nil, err
	}
	canonical, err := CanonicalJSON(unsorted)
	if err != nil {
		return nil, err
	}

	// Sign the canonical JSON with the ed25519 key.
	signature := Base64String(ed25519.Sign(privateKey, canonical))

	// Add the signature to the "signature" key.
	signaturesForEntity := signatures[signingName]
	if signaturesForEntity != nil {
		signaturesForEntity[keyID] = signature
	} else {
		signatures[signingName] = map[KeyID]Base64String{keyID: signature}
	}
	var rawSignatures json.RawMessage
	rawSignatures, err = json.Marshal(signatures)
	if err != nil {
		return nil, err
	}
	object["signatures"] = &rawSignatures

	// Add the unsigned key back if it was present.
	if hasUnsigned {
		object["unsigned"] = rawUnsigned
	}

	return json.Marshal(object)
}

// ListKeyIDs lists the key IDs a given entity has signed a message with.
func ListKeyIDs(signingName string, message []byte) ([]KeyID, error) {
	var object struct {
		Signatures map[string]map[KeyID]json.RawMessage `json:"signatures"`
	}
	if err := json.Unmarshal(message, &object); err != nil {
		return nil, err
	}
	var result []KeyID
	for keyID := range object.Signatures[signingName] {
		result = append(result, keyID)
	}
	return result, nil
}

// VerifyJSON checks that the entity has signed the message using a particular key.
func VerifyJSON(signingName string, keyID KeyID, publicKey ed25519.PublicKey, message []byte) error {
	// Unpack the top-level key of the JSON object without unpacking the contents of the keys.
	// This allows us to add and remove the top-level keys from the JSON object.
	// It also ensures that the JSON is actually a valid JSON object.
	var object map[string]*json.RawMessage
	var signatures map[string]map[KeyID]Base64String
	if err := json.Unmarshal(message, &object); err != nil {
		return err
	}

	// Check that there is a signature from the entity that we are expecting a signature from.
	if object["signatures"] == nil {
		return fmt.Errorf("No signatures")
	}
	if err := json.Unmarshal(*object["signatures"], &signatures); err != nil {
		return err
	}
	signature, ok := signatures[signingName][keyID]
	if !ok {
		return fmt.Errorf("No signature from %q with ID %q", signingName, keyID)
	}
	if len(signature) != ed25519.SignatureSize {
		return fmt.Errorf("Bad signature length from %q with ID %q", signingName, keyID)
	}

	// The "unsigned" key and "signatures" keys aren't covered by the signature so remove them.
	delete(object, "unsigned")
	delete(object, "signatures")

	// Encode the JSON without the "unsigned" and "signatures" keys in the canonical format.
	unsorted, err := json.Marshal(object)
	if err != nil {
		return err
	}
	canonical, err := CanonicalJSON(unsorted)
	if err != nil {
		return err
	}

	// Verify the ed25519 signature.
	if !ed25519.Verify(publicKey, canonical, signature) {
		return fmt.Errorf("Bad signature from %q with ID %q", signingName, keyID)
	}

	return nil
}
<|endoftext|>


<commit_before>package core

import (
	"log"
	"os"
	"path/filepath"
	"sync"
)

const (
	maxConcurrentScans = 64
)

type File struct {
	Name   string
	Parent *File
	Size   int64
	IsDir  bool
	Files  []*File
}

func (f *File) Path() string {
	if f.Parent == nil {
		return f.Name
	}
	return filepath.Join(f.Parent.Path(), f.Name)
}

func (f *File) UpdateSize() {
	if !f.IsDir {
		return
	}
	var size int64
	for _, child := range f.Files {
		child.UpdateSize()
		size += child.Size
	}
	f.Size = size
}

type ReadDir func(dirname string) ([]os.FileInfo, error)

func GetSubTree(path string, parent *File, readDir ReadDir, ignoredFolders map[string]struct{}) *File {
	var mutex sync.Mutex
	var wg sync.WaitGroup
	c := make(chan bool, maxConcurrentScans)
	root := getSubTreeConcurrently(path, parent, readDir, ignoredFolders, c, &mutex, &wg)
	wg.Wait()
	root.UpdateSize()
	return root
}

func getSubTreeConcurrently(path string, parent *File, readDir ReadDir, ignoredFolders map[string]struct{}, c chan bool, mutex *sync.Mutex, wg *sync.WaitGroup) *File {
	ret := &File{}
	entries, err := readDir(path)
	if err != nil {
		log.Println(err)
		return ret
	}
	dirName, name := filepath.Split(path)
	ret.Files = make([]*File, 0, len(entries))
	for _, entry := range entries {
		if entry.IsDir() {
			if _, ignored := ignoredFolders[entry.Name()]; ignored {
				continue
			}
			subDir := filepath.Join(path, entry.Name())
			wg.Add(1)
			go func() {
				c <- true
				subfolder := getSubTreeConcurrently(subDir, ret, readDir, ignoredFolders, c, mutex, wg)
				mutex.Lock()
				ret.Files = append(ret.Files, subfolder)
				mutex.Unlock()
				<-c
				wg.Done()
			}()
		} else {
			size := entry.Size()
			file := &File{
				entry.Name(),
				ret,
				size,
				false,
				[]*File{},
			}
			mutex.Lock()
			ret.Files = append(ret.Files, file)
			mutex.Unlock()
		}
	}
	if parent != nil {
		ret.Name = name
		ret.Parent = parent
	} else {
		// Root dir
		ret.Name = filepath.Join(dirName, name)
	}
	ret.IsDir = true
	return ret
}
<commit_msg>Polishing getSubTreeConcurrently for now<commit_after>package core

import (
	"log"
	"os"
	"path/filepath"
	"sync"
)

const (
	maxConcurrentScans = 64
)

type File struct {
	Name   string
	Parent *File
	Size   int64
	IsDir  bool
	Files  []*File
}

func (f *File) Path() string {
	if f.Parent == nil {
		return f.Name
	}
	return filepath.Join(f.Parent.Path(), f.Name)
}

func (f *File) UpdateSize() {
	if !f.IsDir {
		return
	}
	var size int64
	for _, child := range f.Files {
		child.UpdateSize()
		size += child.Size
	}
	f.Size = size
}

type ReadDir func(dirname string) ([]os.FileInfo, error)

func GetSubTree(path string, parent *File, readDir ReadDir, ignoredFolders map[string]struct{}) *File {
	var mutex sync.Mutex
	var wg sync.WaitGroup
	c := make(chan bool, maxConcurrentScans)
	root := getSubTreeConcurrently(path, parent, readDir, ignoredFolders, c, &mutex, &wg)
	wg.Wait()
	root.UpdateSize()
	return root
}

func getSubTreeConcurrently(path string, parent *File, readDir ReadDir, ignoredFolders map[string]struct{}, c chan bool, mutex *sync.Mutex, wg *sync.WaitGroup) *File {
	result := &File{}
	entries, err := readDir(path)
	if err != nil {
		log.Println(err)
		return result
	}
	dirName, name := filepath.Split(path)
	result.Files = make([]*File, 0, len(entries))
	for _, entry := range entries {
		if entry.IsDir() {
			if _, ignored := ignoredFolders[entry.Name()]; ignored {
				continue
			}
			subFodlerPath := filepath.Join(path, entry.Name())
			wg.Add(1)
			go func() {
				c <- true
				subFolder := getSubTreeConcurrently(subFodlerPath, result, readDir, ignoredFolders, c, mutex, wg)
				mutex.Lock()
				result.Files = append(result.Files, subFolder)
				mutex.Unlock()
				<-c
				wg.Done()
			}()
		} else {
			size := entry.Size()
			file := &File{
				entry.Name(),
				result,
				size,
				false,
				[]*File{},
			}
			mutex.Lock()
			result.Files = append(result.Files, file)
			mutex.Unlock()
		}
	}
	if parent != nil {
		result.Name = name
		result.Parent = parent
	} else {
		// Root dir
		result.Name = filepath.Join(dirName, name)
	}
	result.IsDir = true
	return result
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"io/ioutil"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/iron-io/go/common"
	"github.com/iron-io/titan_go"
)

type Tasker struct {
	api *titan.JobsApi
}

// Titan tasker.
func NewTasker() *Tasker {
	// FIXME(nikhil): Build API from path obtained from config.
	api := titan.NewJobsApiWithBasePath("http://localhost:8080")
	return &Tasker{api}
}

func (t *Tasker) Job(ctx *common.Context) *titan.Job {
	var job *titan.Job
	for {
		jobs, err := t.api.JobsGet(1)
		if err != nil {
			log.Errorln("Tasker JobsGet", "err", err)
		} else if len(jobs.Jobs) > 0 {
			job = &jobs.Jobs[0]
			break
		}

		time.Sleep(1 * time.Second)
	}
	return job
}

// func (t *Tasker) Update(ctx *common.Context, job *titan.Job) {
// 	_, err := t.api.JobIdPatch(job.Id, titan.JobWrapper{*job})
// 	if err != nil {
// 		log.Errorln("Update failed", "job", job.Id, "err", err)
// 	}
// }

func (t *Tasker) RetryTask(ctx *common.Context, job *titan.Job) error {
	panic("Not implemented Retry")
}

func (t *Tasker) IsCancelled(ctx *common.Context, job *titan.Job) bool {
	wrapper, err := t.api.JobIdGet(job.Id)
	if err != nil {
		log.Errorln("JobIdGet from Cancel", "err", err)
		return false
	}

	// FIXME(nikhil) Current branch does not capture cancellation.
	return wrapper.Job.Status == "error"
}

func (t *Tasker) Log(ctx *common.Context, job *titan.Job, r io.Reader) {
	bytes, err := ioutil.ReadAll(r)
	if err != nil {
		log.Errorln("Error reading log", "err", err)
		return
	}

	log.Infoln("Log is ", string(bytes))
	log.Errorln("Titan does not support log upload yet!")
}
<commit_msg>Use correct APIs in runner to get jobs.<commit_after>package main

import (
	"io"
	"io/ioutil"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/iron-io/go/common"
	"github.com/iron-io/titan_go"
)

type Tasker struct {
	api *titan.JobsApi
}

// Titan tasker.
func NewTasker() *Tasker {
	// FIXME(nikhil): Build API from path obtained from config.
	api := titan.NewJobsApiWithBasePath("http://localhost:8080")
	return &Tasker{api}
}

func (t *Tasker) Job(ctx *common.Context) *titan.Job {
	var job *titan.Job
	for {
		jobs, err := t.api.JobsConsumeGet(1)
		if err != nil {
			log.Errorln("Tasker JobsConsumeGet", "err", err)
		} else if len(jobs.Jobs) > 0 {
			job = &jobs.Jobs[0]
			break
		}

		time.Sleep(1 * time.Second)
	}
	return job
}

// func (t *Tasker) Update(ctx *common.Context, job *titan.Job) {
// 	_, err := t.api.JobIdPatch(job.Id, titan.JobWrapper{*job})
// 	if err != nil {
// 		log.Errorln("Update failed", "job", job.Id, "err", err)
// 	}
// }

func (t *Tasker) RetryTask(ctx *common.Context, job *titan.Job) error {
	panic("Not implemented Retry")
}

func (t *Tasker) IsCancelled(ctx *common.Context, job *titan.Job) bool {
	wrapper, err := t.api.JobIdGet(job.Id)
	if err != nil {
		log.Errorln("JobIdGet from Cancel", "err", err)
		return false
	}

	// FIXME(nikhil) Current branch does not capture cancellation.
	return wrapper.Job.Status == "error"
}

func (t *Tasker) Log(ctx *common.Context, job *titan.Job, r io.Reader) {
	bytes, err := ioutil.ReadAll(r)
	if err != nil {
		log.Errorln("Error reading log", "err", err)
		return
	}

	log.Infoln("Log is ", string(bytes))
	log.Errorln("Titan does not support log upload yet!")
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
)

type Template struct {
	source     string
	attributes map[string]interface{}
}

func (t *Template) initialize(source string) {
	t.source = source
}

func (t *Template) render() string {
	// ERB.new(t.source).result(binding)
	return t.source
}

type Entries struct {
	Servers    map[string][]string    `json:"servers"`
	Attributes map[string]interface{} `json:"attributes"`
}

func layout() map[string]Entries {
	input, err := ioutil.ReadFile(path.Join(home, "layout.json"))

	if err != nil {
		fmt.Println(err)
	}

	var e map[string]Entries

	json.Unmarshal(input, &e)

	return e
}

func run(server, recipe, command string, attributes map[string]interface{}) (string, int) {
	template_path := path.Join(home, "recipe", recipe, command)

	if _, err := os.Stat(template_path); os.IsNotExist(err) {
		return "unable to locate: " + template_path, 1
	}

	source, err := ioutil.ReadFile(template_path)

	if err != nil {
		return "unable to read file: " + template_path, 1
	}

	template := &Template{source: string(source), attributes: attributes}

	out, status := ssh(server, template.render())

	return out, status
}

func ssh(server, script string) (string, int) {
	cmd := exec.Command(script[:len(script)-1]) // TODO

	var stderrb bytes.Buffer
	var stdoutb bytes.Buffer

	cmd.Stdout = &stdoutb
	cmd.Stderr = &stderrb

	err := cmd.Run()

	if err != nil {
		fmt.Print(err)
		return stdoutb.String() + stderrb.String(), 1
	}

	// out, status = Open3.capture2e("ssh -T -F #{path("ssh_config")} #{server}", :stdin_data => script)

	return stdoutb.String() + stderrb.String(), 0
}

var home string

func init() {
	home, err := os.Getwd()

	if err != nil {
		fmt.Println("unable to get pwd", home, err)
		os.Exit(1)
	}
}

func main() {
	fmt.Print("\033[01;33mMISSING\033[00m")
	fmt.Print("\033[01;32mDONE\033[00m")

	var command = flag.String("c", "", "command")
	var directory = flag.String("d", ".", "directory") // recipe / layout.json root
	var environment = flag.String("e", "", "environment")
	var quiet = flag.Bool("q", false, "quiet mode")
	var server = flag.String("s", "", "server")
	var verbose = flag.Bool("v", false, "verbose mode")

	flag.Parse()

	fmt.Println(home, *directory, *quiet, *verbose)

	e := layout()

	layout := e[*environment]

	var servers []string

	if len(strings.Split(*server, ",")) > 1 {
		for _, v := range strings.Split(*server, ",") {
			servers = append(servers, v)
		}
	} else {
		for k, _ := range layout.Servers {
			servers = append(servers, k)
		}
	}

	exit_status := 0

	for _, v := range servers {
		recipes := layout.Servers[v]

		fmt.Print(v)

		for _, recipe := range recipes {
			fmt.Printf("  %s: ", recipe)

			filename := path.Join(home, "recipe", recipe)

			if _, err := os.Stat(filename); os.IsNotExist(err) {
				fmt.Printf("unable to locate: %s\n", filename)
				os.Exit(1)
			}

			stdout, status := run(v, recipe, *command, layout.Attributes)

			switch status {
			case -1: // nil is better? negative error codes?
				fmt.Print("?")
			case 0:
				fmt.Print("\033[01;32mOK\033[00m")
			default:
				fmt.Print("\033[01;31mERROR\033[00m")
				exit_status = 1
				break
			}

			if len(stdout) > 0 {
				fmt.Fprintf(os.Stderr, " %s\n", stdout)
			}
		}
	}

	os.Exit(exit_status)
}
<commit_msg>more less<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
)

type Template struct {
	attributes map[string]interface{}
	source     string
}

func (t *Template) initialize(source string) {
	t.source = source
}

func (t *Template) render() string {
	// ERB.new(t.source).result(binding)
	return t.source
}

type Entries struct {
	Attributes map[string]interface{} `json:"attributes"`
	Servers    map[string][]string    `json:"servers"`
}

func layout() map[string]Entries {
	input, err := ioutil.ReadFile(path.Join(home, "layout.json"))

	if err != nil {
		fmt.Println(err)
	}

	var e map[string]Entries

	json.Unmarshal(input, &e)

	return e
}

func run(server, recipe, command string, attributes map[string]interface{}) (string, int) {
	template_path := path.Join(home, "recipe", recipe, command)

	if _, err := os.Stat(template_path); os.IsNotExist(err) {
		return "unable to locate: " + template_path, 1
	}

	source, err := ioutil.ReadFile(template_path)

	if err != nil {
		return "unable to read file: " + template_path, 1
	}

	template := &Template{source: string(source), attributes: attributes}

	out, status := ssh(server, template.render())

	return out, status
}

func ssh(server, script string) (string, int) {
	cmd := exec.Command(script[:len(script)-1]) // TODO

	var stderrb bytes.Buffer
	var stdoutb bytes.Buffer

	cmd.Stdout = &stdoutb
	cmd.Stderr = &stderrb

	err := cmd.Run()

	if err != nil {
		fmt.Print(err)
		return stdoutb.String() + stderrb.String(), 1
	}

	// out, status = Open3.capture2e("ssh -T -F #{path("ssh_config")} #{server}", :stdin_data => script)

	return stdoutb.String() + stderrb.String(), 0
}

var home string

func init() {
	home, err := os.Getwd()

	if err != nil {
		fmt.Println("unable to get pwd", home, err)
		os.Exit(1)
	}
}

func main() {
	fmt.Print("\033[01;33mMISSING\033[00m")
	fmt.Print("\033[01;32mDONE\033[00m")

	var command = flag.String("c", "", "command")
	var directory = flag.String("d", ".", "directory") // recipe / layout.json root
	var environment = flag.String("e", "", "environment")
	var quiet = flag.Bool("q", false, "quiet mode")
	var server = flag.String("s", "", "server")
	var verbose = flag.Bool("v", false, "verbose mode")

	flag.Parse()

	fmt.Println(home, *directory, *quiet, *verbose)

	e := layout()

	layout := e[*environment]

	var servers []string

	if len(strings.Split(*server, ",")) > 1 {
		for _, v := range strings.Split(*server, ",") {
			servers = append(servers, v)
		}
	} else {
		for k, _ := range layout.Servers {
			servers = append(servers, k)
		}
	}

	exit_status := 0

	for _, v := range servers {
		recipes := layout.Servers[v]

		fmt.Print(v)

		for _, recipe := range recipes {
			fmt.Printf("  %s: ", recipe)

			filename := path.Join(home, "recipe", recipe)

			if _, err := os.Stat(filename); os.IsNotExist(err) {
				fmt.Printf("unable to locate: %s\n", filename)
				os.Exit(1)
			}

			stdout, status := run(v, recipe, *command, layout.Attributes)

			switch status {
			case -1: // nil is better? negative error codes?
				fmt.Print("?")
			case 0:
				fmt.Print("\033[01;32mOK\033[00m")
			default:
				fmt.Print("\033[01;31mERROR\033[00m")
				exit_status = 1
				break
			}

			if len(stdout) > 0 {
				fmt.Fprintf(os.Stderr, " %s\n", stdout)
			}
		}
	}

	os.Exit(exit_status)
}
<|endoftext|>