<commit_before>/*
Copyright (c) 2014 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package importx

import (
	"encoding/xml"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"path"

	"github.com/vmware/govmomi"
	"github.com/vmware/govmomi/govc/cli"
	"github.com/vmware/govmomi/govc/flags"
	"github.com/vmware/govmomi/vim25/progress"
	"github.com/vmware/govmomi/vim25/soap"
	"github.com/vmware/govmomi/vim25/types"
)

type ovf struct {
	*flags.DatastoreFlag
	*flags.ResourcePoolFlag
	*flags.HostSystemFlag
	*flags.OutputFlag

	Client       *govmomi.Client
	Datacenter   *govmomi.Datacenter
	Datastore    *govmomi.Datastore
	ResourcePool *govmomi.ResourcePool

	Archive
}

func init() {
	cli.Register("import.ovf", &ovf{})
}

func (cmd *ovf) Register(f *flag.FlagSet) {}

func (cmd *ovf) Process() error { return nil }

func (cmd *ovf) Run(f *flag.FlagSet) error {
	file, err := cmd.Prepare(f)

	if err != nil {
		return err
	}

	cmd.Archive = &FileArchive{file}

	return cmd.Import(file)
}

func (cmd *ovf) Prepare(f *flag.FlagSet) (string, error) {
	var err error

	args := f.Args()
	if len(args) != 1 {
		return "", errors.New("no file to import")
	}

	cmd.Client, err = cmd.DatastoreFlag.Client()
	if err != nil {
		return "", err
	}

	cmd.Datacenter, err = cmd.DatastoreFlag.Datacenter()
	if err != nil {
		return "", err
	}

	cmd.Datastore, err = cmd.DatastoreFlag.Datastore()
	if err != nil {
		return "", err
	}

	cmd.ResourcePool, err = cmd.ResourcePoolFlag.ResourcePool()
	if err != nil {
		return "", err
	}

	return f.Arg(0), nil
}

func (cmd *ovf) ReadAll(name string) ([]byte, error) {
	f, _, err := cmd.Open(name)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	return ioutil.ReadAll(f)
}

func (cmd *ovf) Import(fpath string) error {
	c := cmd.Client

	desc, err := cmd.ReadAll(fpath)
	if err != nil {
		return err
	}

	// extract name from .ovf for use as VM name
	ovf := struct {
		VirtualSystem struct {
			Name string
		}
	}{}

	if err := xml.Unmarshal(desc, &ovf); err != nil {
		return fmt.Errorf("failed to parse ovf: %s", err.Error())
	}

	cisp := types.OvfCreateImportSpecParams{
		EntityName: ovf.VirtualSystem.Name,
		OvfManagerCommonParams: types.OvfManagerCommonParams{
			Locale: "US",
		},
	}

	spec, err := c.OvfManager().CreateImportSpec(string(desc), cmd.ResourcePool, cmd.Datastore, cisp)
	if err != nil {
		return err
	}

	if spec.Error != nil {
		return errors.New(spec.Error[0].LocalizedMessage)
	}

	if spec.Warning != nil {
		for _, w := range spec.Warning {
			_, _ = cmd.Log(fmt.Sprintf("Warning: %s\n", w.LocalizedMessage))
		}
	}

	// TODO: ImportSpec may have unitNumber==0, but this field is optional in the wsdl
	// and hence omitempty in the struct tag; but unitNumber is required for certain devices.
	s := &spec.ImportSpec.(*types.VirtualMachineImportSpec).ConfigSpec
	for _, d := range s.DeviceChange {
		n := &d.GetVirtualDeviceConfigSpec().Device.GetVirtualDevice().UnitNumber
		if *n == 0 {
			*n = -1
		}
	}

	var host *govmomi.HostSystem
	if cmd.SearchFlag.IsSet() {
		if host, err = cmd.HostSystem(); err != nil {
			return err
		}
	}

	// TODO: need a folder option
	folders, err := cmd.Datacenter.Folders()
	if err != nil {
		return err
	}

	lease, err := cmd.ResourcePool.ImportVApp(spec.ImportSpec, folders.VmFolder, host)
	if err != nil {
		return err
	}

	info, err := lease.Wait()
	if err != nil {
		return err
	}

	// Build slice of items and URLs first, so that the lease updater can know
	// about every item that needs to be uploaded, and thereby infer progress.
	var items []ovfFileItem

	for _, device := range info.DeviceUrl {
		for _, item := range spec.FileItem {
			if device.ImportKey != item.DeviceId {
				continue
			}

			u, err := c.Client.ParseURL(device.Url)
			if err != nil {
				return err
			}

			i := ovfFileItem{
				url:  u,
				item: item,
				ch:   make(chan progress.Report),
			}

			items = append(items, i)
		}
	}

	u := newLeaseUpdater(cmd.Client, lease, items)
	defer u.Done()

	for _, i := range items {
		err = cmd.Upload(lease, i)
		if err != nil {
			return err
		}
	}

	return lease.HttpNfcLeaseComplete()
}

func (cmd *ovf) Upload(lease *govmomi.HttpNfcLease, ofi ovfFileItem) error {
	item := ofi.item
	file := item.Path

	f, size, err := cmd.Open(file)
	if err != nil {
		return err
	}
	defer f.Close()

	logger := cmd.ProgressLogger(fmt.Sprintf("Uploading %s... ", path.Base(file)))
	defer logger.Wait()

	opts := soap.Upload{
		Type:          "application/x-vnd.vmware-streamVmdk",
		Method:        "POST",
		ContentLength: size,
		Progress:      progress.Tee(ofi, logger),
	}

	if item.Create {
		opts.Method = "PUT"
	}

	return cmd.Client.Client.Upload(f, ofi.url, &opts)
}
<commit_msg>Support non-disk files in import.ovf<commit_after>/*
Copyright (c) 2014 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package importx

import (
	"encoding/xml"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"path"

	"github.com/vmware/govmomi"
	"github.com/vmware/govmomi/govc/cli"
	"github.com/vmware/govmomi/govc/flags"
	"github.com/vmware/govmomi/vim25/progress"
	"github.com/vmware/govmomi/vim25/soap"
	"github.com/vmware/govmomi/vim25/types"
)

type ovf struct {
	*flags.DatastoreFlag
	*flags.ResourcePoolFlag
	*flags.HostSystemFlag
	*flags.OutputFlag

	Client       *govmomi.Client
	Datacenter   *govmomi.Datacenter
	Datastore    *govmomi.Datastore
	ResourcePool *govmomi.ResourcePool

	Archive
}

func init() {
	cli.Register("import.ovf", &ovf{})
}

func (cmd *ovf) Register(f *flag.FlagSet) {}

func (cmd *ovf) Process() error { return nil }

func (cmd *ovf) Run(f *flag.FlagSet) error {
	file, err := cmd.Prepare(f)

	if err != nil {
		return err
	}

	cmd.Archive = &FileArchive{file}

	return cmd.Import(file)
}

func (cmd *ovf) Prepare(f *flag.FlagSet) (string, error) {
	var err error

	args := f.Args()
	if len(args) != 1 {
		return "", errors.New("no file to import")
	}

	cmd.Client, err = cmd.DatastoreFlag.Client()
	if err != nil {
		return "", err
	}

	cmd.Datacenter, err = cmd.DatastoreFlag.Datacenter()
	if err != nil {
		return "", err
	}

	cmd.Datastore, err = cmd.DatastoreFlag.Datastore()
	if err != nil {
		return "", err
	}

	cmd.ResourcePool, err = cmd.ResourcePoolFlag.ResourcePool()
	if err != nil {
		return "", err
	}

	return f.Arg(0), nil
}

func (cmd *ovf) ReadAll(name string) ([]byte, error) {
	f, _, err := cmd.Open(name)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	return ioutil.ReadAll(f)
}

func (cmd *ovf) Import(fpath string) error {
	c := cmd.Client

	desc, err := cmd.ReadAll(fpath)
	if err != nil {
		return err
	}

	// extract name from .ovf for use as VM name
	ovf := struct {
		VirtualSystem struct {
			Name string
		}
	}{}

	if err := xml.Unmarshal(desc, &ovf); err != nil {
		return fmt.Errorf("failed to parse ovf: %s", err.Error())
	}

	cisp := types.OvfCreateImportSpecParams{
		EntityName: ovf.VirtualSystem.Name,
		OvfManagerCommonParams: types.OvfManagerCommonParams{
			Locale: "US",
		},
	}

	spec, err := c.OvfManager().CreateImportSpec(string(desc), cmd.ResourcePool, cmd.Datastore, cisp)
	if err != nil {
		return err
	}

	if spec.Error != nil {
		return errors.New(spec.Error[0].LocalizedMessage)
	}

	if spec.Warning != nil {
		for _, w := range spec.Warning {
			_, _ = cmd.Log(fmt.Sprintf("Warning: %s\n", w.LocalizedMessage))
		}
	}

	// TODO: ImportSpec may have unitNumber==0, but this field is optional in the wsdl
	// and hence omitempty in the struct tag; but unitNumber is required for certain devices.
	s := &spec.ImportSpec.(*types.VirtualMachineImportSpec).ConfigSpec
	for _, d := range s.DeviceChange {
		n := &d.GetVirtualDeviceConfigSpec().Device.GetVirtualDevice().UnitNumber
		if *n == 0 {
			*n = -1
		}
	}

	var host *govmomi.HostSystem
	if cmd.SearchFlag.IsSet() {
		if host, err = cmd.HostSystem(); err != nil {
			return err
		}
	}

	// TODO: need a folder option
	folders, err := cmd.Datacenter.Folders()
	if err != nil {
		return err
	}

	lease, err := cmd.ResourcePool.ImportVApp(spec.ImportSpec, folders.VmFolder, host)
	if err != nil {
		return err
	}

	info, err := lease.Wait()
	if err != nil {
		return err
	}

	// Build slice of items and URLs first, so that the lease updater can know
	// about every item that needs to be uploaded, and thereby infer progress.
	var items []ovfFileItem

	for _, device := range info.DeviceUrl {
		for _, item := range spec.FileItem {
			if device.ImportKey != item.DeviceId {
				continue
			}

			u, err := c.Client.ParseURL(device.Url)
			if err != nil {
				return err
			}

			i := ovfFileItem{
				url:  u,
				item: item,
				ch:   make(chan progress.Report),
			}

			items = append(items, i)
		}
	}

	u := newLeaseUpdater(cmd.Client, lease, items)
	defer u.Done()

	for _, i := range items {
		err = cmd.Upload(lease, i)
		if err != nil {
			return err
		}
	}

	return lease.HttpNfcLeaseComplete()
}

func (cmd *ovf) Upload(lease *govmomi.HttpNfcLease, ofi ovfFileItem) error {
	item := ofi.item
	file := item.Path

	f, size, err := cmd.Open(file)
	if err != nil {
		return err
	}
	defer f.Close()

	logger := cmd.ProgressLogger(fmt.Sprintf("Uploading %s... ", path.Base(file)))
	defer logger.Wait()

	opts := soap.Upload{
		ContentLength: size,
		Progress:      progress.Tee(ofi, logger),
	}

	// Non-disk files (such as .iso) use the PUT method.
	// Overwrite: t header is also required in this case (ovftool does the same)
	if item.Create {
		opts.Method = "PUT"
		opts.Headers = map[string]string{
			"Overwrite": "t",
		}
	} else {
		opts.Method = "POST"
		opts.Type = "application/x-vnd.vmware-streamVmdk"
	}

	return cmd.Client.Client.Upload(f, ofi.url, &opts)
}
<|endoftext|>


<commit_before>package main

import (
	. "github.com/gorilla/feeds"
	"github.com/gorilla/mux"
	"github.com/sourcegraph/sitemap"
	"database/sql"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"
)

const (
	itemsPerPage  = 10
	statusEnabled = 1
	levelRoot     = iota
	levelList
	levelVote
)

type Listboard struct {
	config *Config
	m      *Model
	tp     *TransPool
}

type ValidationErrors []string

type appHandler func(http.ResponseWriter, *http.Request) error

func NewListboard() *Listboard {
	return &Listboard{}
}

func (l *Listboard) Run(args []string) {
	var err error

	l.config = NewConfig()
	if err = l.config.Load(args); err != nil {
		panic(err)
	}

	l.m = NewModel(l.config)
	if err = l.m.Init(l.config); err != nil {
		panic(err)
	}

	l.tp = NewTransPool(l.config.Translations)

	r := mux.NewRouter()
	r.HandleFunc("/", appHandler(l.indexHandler).ServeHTTP).Methods("GET")
	r.HandleFunc("/feed.xml", appHandler(l.feedHandler).ServeHTTP).Methods("GET")
	r.HandleFunc("/all.xml", appHandler(l.feedAlllHandler).ServeHTTP).Methods("GET")
	r.HandleFunc("/sitemap.xml", appHandler(l.sitemapHandler).ServeHTTP).Methods("GET")

	r.HandleFunc("/add.html", appHandler(l.addFormHandler).ServeHTTP).Methods("GET", "POST")
	r.HandleFunc("/list/{listId}/{slug}", appHandler(l.listHandler).ServeHTTP).Methods("GET", "POST")
	r.HandleFunc("/vote/{itemId}/{slug}", appHandler(l.voteHandler).ServeHTTP).Methods("GET", "POST")

	// Static assets
	r.PathPrefix("/").Handler(http.FileServer(http.Dir("./public_html")))

	http.Handle("/", r)

	log.Printf("Starting server at %s", l.config.Server)
	if err := http.ListenAndServe(l.config.Server, nil); err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}

func (fn appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if err := fn(w, r); err != nil {
		if err == sql.ErrNoRows {
			http.Error(w, "Not found", http.StatusNotFound)
			return
		}
		httpError, ok := err.(HTTPError)
		if ok {
			http.Error(w, httpError.Message, httpError.Code)
			return
		}
		// Default to 500 Internal Server Error
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func (l *Listboard) indexHandler(w http.ResponseWriter, r *http.Request) error {
	pageStr := r.URL.Query().Get("hostname")
	page := 0
	var err error
	if len(pageStr) != 0 {
		page, err = strconv.Atoi(pageStr)
		if err != nil {
			log.Printf("%s is not a valid page number", pageStr)
			page = 0
		}
	}
	sc := l.config.getSiteConfig(l.getToken(r))
	s := NewSession(sc, l.tp.Get(sc.Language))
	s.AddPath("", s.Lang("Home"))
	s.Set("Lists", l.m.mustGetChildNodes(sc.DomainId, 0, itemsPerPage, page, "updated DESC"))
	return s.render(w, r, sc.templatePath("layout.html"), sc.templatePath("index.html"))
}

func (l *Listboard) addFormHandler(w http.ResponseWriter, r *http.Request) error {
	sc := l.config.getSiteConfig(l.getToken(r))

	var errors ValidationErrors
	var node Node
	tr := l.tp.Get(sc.Language)
	if r.Method == "POST" {
		if !inHoneypot(r.FormValue("name")) {
			node, errors = l.validateForm(r, sc.DomainId, 0, levelRoot, tr)
			if len(errors) == 0 {
				// save and redirect
				id, err := l.m.addNode(&node)
				if err != nil {
					return &HTTPError{Err: err, Code: http.StatusInternalServerError}
				}
				url := "/list/" + strconv.Itoa(id) + "/" + hfSlug(node.Title)
				http.Redirect(w, r, url, http.StatusFound)
			}
		}
	}
	s := NewSession(sc, tr)
	s.Set("Errors", errors)
	s.Set("Form", node)
	s.AddPath("/", s.Lang("Home"))
	s.AddPath("", s.Lang("New list"))
	s.Set("Subtitle", s.Lang("New list"))
	return s.render(w, r, sc.templatePath("layout.html"), sc.templatePath("add.html"), sc.templatePath("form.html"))
}

func (l *Listboard) listHandler(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	listId, err := strconv.Atoi(vars["listId"])
	if err != nil {
		log.Printf("%d is not a valid list number", listId)
		return err
	}
	sc := l.config.getSiteConfig(l.getToken(r))
	tr := l.tp.Get(sc.Language)

	var errors ValidationErrors
	var node Node

	if r.Method == "POST" {
		if !inHoneypot(r.FormValue("name")) {
			node, errors = l.validateForm(r, sc.DomainId, listId, levelList, tr)
			if len(errors) == 0 {
				// save and redirect
				id, err := l.m.addNode(&node)
				if err != nil {
					return &HTTPError{
						Err: err,
						Message: err.Error(),
						Code: http.StatusInternalServerError,
					}
				}
				url := "/list/" + strconv.Itoa(listId) + "/" + hfSlug(node.Title) + "#I" + strconv.Itoa(id)
				http.Redirect(w, r, url, http.StatusFound)
			}
		}
	}
	s := NewSession(sc, tr)

	s.Set("Errors", errors)
	s.Set("Form", node)
	list, err := l.m.getNode(sc.DomainId, listId)
	if err != nil {
		return err
	}
	s.Set("List", list)
	s.Set("Items", l.m.mustGetChildNodes(sc.DomainId, listId, itemsPerPage, 0, "vote DESC, created"))
	s.Set("FormTitle", s.Lang("New suggestion"))
	s.Set("Subtitle", list.Title)
	s.Set("Description", list.Title)
	s.AddPath("/", s.Lang("Home"))
	s.AddPath("", list.Title)
	return s.render(w, r, sc.templatePath("layout.html"), sc.templatePath("list.html"), sc.templatePath("form.html"))
}

func (l *Listboard) voteHandler(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	itemId, err := strconv.Atoi(vars["itemId"])
	if err != nil {
		log.Printf("%d is not a valid item number", itemId)
		return err
	}
	sc := l.config.getSiteConfig(l.getToken(r))
	tr := l.tp.Get(sc.Language)
	var errors ValidationErrors
	var node Node
	item, err := l.m.getNode(sc.DomainId, itemId)
	if err != nil {
		return err
	}
	if r.Method == "POST" {
		if !inHoneypot(r.FormValue("name")) {
			node, errors = l.validateForm(r, sc.DomainId, itemId, levelVote, tr)
			if len(errors) == 0 {
				id, err := l.m.addNode(&node)
				if err != nil {
					return &HTTPError{
						Err: err,
						Message: err.Error(),
						Code: http.StatusInternalServerError,
					}
				}
				if err := l.m.Vote(sc.DomainId, node.Vote, id, itemId, item.ParentId); err != nil {
					return &HTTPError{
						Err: err,
						Message: err.Error(),
						Code: http.StatusInternalServerError,
					}
				}
				http.Redirect(w, r, r.URL.String()+"#I"+strconv.Itoa(id), http.StatusFound)
			}
		}
	}
	s := NewSession(sc, tr)
	s.Set("Subtitle", item.Title)
	s.Set("Description", item.Title)
	s.Set("ShowVote", true)
	s.Set("Errors", errors)
	list, err := l.m.getNode(sc.DomainId, item.ParentId)
	if err != nil {
		return err
	}
	s.Set("List", list)
	s.Set("Item", item)
	if len(node.Title) == 0 {
		node.Title = s.Lang("Re") + ": " + item.Title
	}
	s.Set("Form", node)
	s.Set("Items", l.m.mustGetChildNodes(sc.DomainId, itemId, itemsPerPage, 0, "created"))
	s.Set("FormTitle", s.Lang("New vote"))
	s.AddPath("/", s.Lang("Home"))
	s.AddPath("/list/"+strconv.Itoa(list.Id)+"/"+hfSlug(list.Title), list.Title)
	s.AddPath("", item.Title)
	return s.render(w, r, sc.templatePath("layout.html"), sc.templatePath("vote.html"), sc.templatePath("form.html"))
}

func (l *Listboard) feed(w http.ResponseWriter, sc *SiteConfig, baseURL string, nodes *NodeList) error {
	feed := &Feed{
		Title:       sc.Title,
		Link:        &Link{Href: baseURL},
		Description: sc.Description,
		Author:      &Author{sc.AuthorName, sc.AuthorEmail},
		Created:     time.Now(),
	}
	for _, node := range *nodes {
		feed.Items = append(feed.Items, &Item{
			Title:       node.Title,
			Link:        &Link{Href: baseURL + "/list/" + strconv.Itoa(node.Id) + "/" + hfSlug(node.Title)},
			Description: string(node.Rendered),
			Created:     node.Created,
		})
	}
	w.Header().Set("Content-Type", "application/rss+xml")
	return feed.WriteRss(w)
}

func (l *Listboard) feedHandler(w http.ResponseWriter, r *http.Request) error {
	sc := l.config.getSiteConfig(l.getToken(r))
	nodes := l.m.mustGetChildNodes(sc.DomainId, 0, 20, 0, "created")
	baseUrl := "http://" + r.Host
	return l.feed(w, sc, baseUrl, nodes)
}

func (l *Listboard) feedAlllHandler(w http.ResponseWriter, r *http.Request) error {
	sc := l.config.getSiteConfig(l.getToken(r))
	nodes := l.m.mustGetAllNodes(sc.DomainId, 20, 0, "created")
	baseUrl := "http://" + r.Host
	return l.feed(w, sc, baseUrl, nodes)
}

func (l *Listboard) sitemapHandler(w http.ResponseWriter, r *http.Request) error {
	sc := l.config.getSiteConfig(l.getToken(r))
	nodes := l.m.mustGetChildNodes(sc.DomainId, 0, 1000, 0, "created")
	var urlSet sitemap.URLSet
	for _, node := range *nodes {
		urlSet.URLs = append(urlSet.URLs, sitemap.URL{
			Loc:        "http://" + r.Host + "/list/" + strconv.Itoa(node.Id) + "/" + hfSlug(node.Title),
			LastMod:    &node.Created,
			ChangeFreq: sitemap.Daily,
			Priority:   0.7,
		})
	}
	xml, err := sitemap.Marshal(&urlSet)
	if err != nil {
		return err
	}
	w.Header().Set("Content-Type", "application/xml")
	_, err = w.Write(xml)
	return err;
}

func (l *Listboard) validateForm(r *http.Request, domainId, parentId, level int, ln *Language) (Node, ValidationErrors) {
	node := Node{
		ParentId: parentId,
		DomainId: domainId,
		Title:    strings.TrimSpace(r.FormValue("title")),
		Vote:     getVote(r.FormValue("vote")),
		Tripcode: getTripcode(r.FormValue("password")),
		Body:     r.FormValue("body"),
		Status:   statusEnabled,
		Level:    level,
	}
	errors := ValidationErrors{}
	if len(node.Title) < 3 {
		errors = append(errors, ln.Lang("Title must be at least 3 characters long"))
	}
	if len(node.Body) < 10 {
		errors = append(errors, ln.Lang("Please, write something"))
	} else {
		node.Rendered = renderText(node.Body)
		// Check again after the rendering
		if len(node.Rendered) < 10 {
			errors = append(errors, ln.Lang("Please, write something"))
		}
	}
	return node, errors
}

func (l *Listboard) getToken(r *http.Request) string {
	return r.Header.Get(l.config.Token)
}
<commit_msg>Feed order by fixed<commit_after>package main

import (
	. "github.com/gorilla/feeds"
	"github.com/gorilla/mux"
	"github.com/sourcegraph/sitemap"
	"database/sql"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"
)

const (
	itemsPerPage  = 10
	statusEnabled = 1
	levelRoot     = iota
	levelList
	levelVote
)

type Listboard struct {
	config *Config
	m      *Model
	tp     *TransPool
}

type ValidationErrors []string

type appHandler func(http.ResponseWriter, *http.Request) error

func NewListboard() *Listboard {
	return &Listboard{}
}

func (l *Listboard) Run(args []string) {
	var err error

	l.config = NewConfig()
	if err = l.config.Load(args); err != nil {
		panic(err)
	}

	l.m = NewModel(l.config)
	if err = l.m.Init(l.config); err != nil {
		panic(err)
	}

	l.tp = NewTransPool(l.config.Translations)

	r := mux.NewRouter()
	r.HandleFunc("/", appHandler(l.indexHandler).ServeHTTP).Methods("GET")
	r.HandleFunc("/feed.xml", appHandler(l.feedHandler).ServeHTTP).Methods("GET")
	r.HandleFunc("/all.xml", appHandler(l.feedAlllHandler).ServeHTTP).Methods("GET")
	r.HandleFunc("/sitemap.xml", appHandler(l.sitemapHandler).ServeHTTP).Methods("GET")

	r.HandleFunc("/add.html", appHandler(l.addFormHandler).ServeHTTP).Methods("GET", "POST")
	r.HandleFunc("/list/{listId}/{slug}", appHandler(l.listHandler).ServeHTTP).Methods("GET", "POST")
	r.HandleFunc("/vote/{itemId}/{slug}", appHandler(l.voteHandler).ServeHTTP).Methods("GET", "POST")

	// Static assets
	r.PathPrefix("/").Handler(http.FileServer(http.Dir("./public_html")))

	http.Handle("/", r)

	log.Printf("Starting server at %s", l.config.Server)
	if err := http.ListenAndServe(l.config.Server, nil); err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}

func (fn appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if err := fn(w, r); err != nil {
		if err == sql.ErrNoRows {
			http.Error(w, "Not found", http.StatusNotFound)
			return
		}
		httpError, ok := err.(HTTPError)
		if ok {
			http.Error(w, httpError.Message, httpError.Code)
			return
		}
		// Default to 500 Internal Server Error
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func (l *Listboard) indexHandler(w http.ResponseWriter, r *http.Request) error {
	pageStr := r.URL.Query().Get("hostname")
	page := 0
	var err error
	if len(pageStr) != 0 {
		page, err = strconv.Atoi(pageStr)
		if err != nil {
			log.Printf("%s is not a valid page number", pageStr)
			page = 0
		}
	}
	sc := l.config.getSiteConfig(l.getToken(r))
	s := NewSession(sc, l.tp.Get(sc.Language))
	s.AddPath("", s.Lang("Home"))
	s.Set("Lists", l.m.mustGetChildNodes(sc.DomainId, 0, itemsPerPage, page, "updated DESC"))
	return s.render(w, r, sc.templatePath("layout.html"), sc.templatePath("index.html"))
}

func (l *Listboard) addFormHandler(w http.ResponseWriter, r *http.Request) error {
	sc := l.config.getSiteConfig(l.getToken(r))

	var errors ValidationErrors
	var node Node
	tr := l.tp.Get(sc.Language)
	if r.Method == "POST" {
		if !inHoneypot(r.FormValue("name")) {
			node, errors = l.validateForm(r, sc.DomainId, 0, levelRoot, tr)
			if len(errors) == 0 {
				// save and redirect
				id, err := l.m.addNode(&node)
				if err != nil {
					return &HTTPError{Err: err, Code: http.StatusInternalServerError}
				}
				url := "/list/" + strconv.Itoa(id) + "/" + hfSlug(node.Title)
				http.Redirect(w, r, url, http.StatusFound)
			}
		}
	}
	s := NewSession(sc, tr)
	s.Set("Errors", errors)
	s.Set("Form", node)
	s.AddPath("/", s.Lang("Home"))
	s.AddPath("", s.Lang("New list"))
	s.Set("Subtitle", s.Lang("New list"))
	return s.render(w, r, sc.templatePath("layout.html"), sc.templatePath("add.html"), sc.templatePath("form.html"))
}

func (l *Listboard) listHandler(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	listId, err := strconv.Atoi(vars["listId"])
	if err != nil {
		log.Printf("%d is not a valid list number", listId)
		return err
	}
	sc := l.config.getSiteConfig(l.getToken(r))
	tr := l.tp.Get(sc.Language)

	var errors ValidationErrors
	var node Node

	if r.Method == "POST" {
		if !inHoneypot(r.FormValue("name")) {
			node, errors = l.validateForm(r, sc.DomainId, listId, levelList, tr)
			if len(errors) == 0 {
				// save and redirect
				id, err := l.m.addNode(&node)
				if err != nil {
					return &HTTPError{
						Err: err,
						Message: err.Error(),
						Code: http.StatusInternalServerError,
					}
				}
				url := "/list/" + strconv.Itoa(listId) + "/" + hfSlug(node.Title) + "#I" + strconv.Itoa(id)
				http.Redirect(w, r, url, http.StatusFound)
			}
		}
	}
	s := NewSession(sc, tr)

	s.Set("Errors", errors)
	s.Set("Form", node)
	list, err := l.m.getNode(sc.DomainId, listId)
	if err != nil {
		return err
	}
	s.Set("List", list)
	s.Set("Items", l.m.mustGetChildNodes(sc.DomainId, listId, itemsPerPage, 0, "vote DESC, created"))
	s.Set("FormTitle", s.Lang("New suggestion"))
	s.Set("Subtitle", list.Title)
	s.Set("Description", list.Title)
	s.AddPath("/", s.Lang("Home"))
	s.AddPath("", list.Title)
	return s.render(w, r, sc.templatePath("layout.html"), sc.templatePath("list.html"), sc.templatePath("form.html"))
}

func (l *Listboard) voteHandler(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	itemId, err := strconv.Atoi(vars["itemId"])
	if err != nil {
		log.Printf("%d is not a valid item number", itemId)
		return err
	}
	sc := l.config.getSiteConfig(l.getToken(r))
	tr := l.tp.Get(sc.Language)
	var errors ValidationErrors
	var node Node
	item, err := l.m.getNode(sc.DomainId, itemId)
	if err != nil {
		return err
	}
	if r.Method == "POST" {
		if !inHoneypot(r.FormValue("name")) {
			node, errors = l.validateForm(r, sc.DomainId, itemId, levelVote, tr)
			if len(errors) == 0 {
				id, err := l.m.addNode(&node)
				if err != nil {
					return &HTTPError{
						Err: err,
						Message: err.Error(),
						Code: http.StatusInternalServerError,
					}
				}
				if err := l.m.Vote(sc.DomainId, node.Vote, id, itemId, item.ParentId); err != nil {
					return &HTTPError{
						Err: err,
						Message: err.Error(),
						Code: http.StatusInternalServerError,
					}
				}
				http.Redirect(w, r, r.URL.String()+"#I"+strconv.Itoa(id), http.StatusFound)
			}
		}
	}
	s := NewSession(sc, tr)
	s.Set("Subtitle", item.Title)
	s.Set("Description", item.Title)
	s.Set("ShowVote", true)
	s.Set("Errors", errors)
	list, err := l.m.getNode(sc.DomainId, item.ParentId)
	if err != nil {
		return err
	}
	s.Set("List", list)
	s.Set("Item", item)
	if len(node.Title) == 0 {
		node.Title = s.Lang("Re") + ": " + item.Title
	}
	s.Set("Form", node)
	s.Set("Items", l.m.mustGetChildNodes(sc.DomainId, itemId, itemsPerPage, 0, "created"))
	s.Set("FormTitle", s.Lang("New vote"))
	s.AddPath("/", s.Lang("Home"))
	s.AddPath("/list/"+strconv.Itoa(list.Id)+"/"+hfSlug(list.Title), list.Title)
	s.AddPath("", item.Title)
	return s.render(w, r, sc.templatePath("layout.html"), sc.templatePath("vote.html"), sc.templatePath("form.html"))
}

func (l *Listboard) feed(w http.ResponseWriter, sc *SiteConfig, baseURL string, nodes *NodeList) error {
	feed := &Feed{
		Title:       sc.Title,
		Link:        &Link{Href: baseURL},
		Description: sc.Description,
		Author:      &Author{sc.AuthorName, sc.AuthorEmail},
		Created:     time.Now(),
	}
	for _, node := range *nodes {
		feed.Items = append(feed.Items, &Item{
			Title:       node.Title,
			Link:        &Link{Href: baseURL + "/list/" + strconv.Itoa(node.Id) + "/" + hfSlug(node.Title)},
			Description: string(node.Rendered),
			Created:     node.Created,
		})
	}
	w.Header().Set("Content-Type", "application/rss+xml")
	return feed.WriteRss(w)
}

func (l *Listboard) feedHandler(w http.ResponseWriter, r *http.Request) error {
	sc := l.config.getSiteConfig(l.getToken(r))
	nodes := l.m.mustGetChildNodes(sc.DomainId, 0, 20, 0, "created DESC")
	baseUrl := "http://" + r.Host
	return l.feed(w, sc, baseUrl, nodes)
}

func (l *Listboard) feedAlllHandler(w http.ResponseWriter, r *http.Request) error {
	sc := l.config.getSiteConfig(l.getToken(r))
	nodes := l.m.mustGetAllNodes(sc.DomainId, 20, 0, "created DESC")
	baseUrl := "http://" + r.Host
	return l.feed(w, sc, baseUrl, nodes)
}

func (l *Listboard) sitemapHandler(w http.ResponseWriter, r *http.Request) error {
	sc := l.config.getSiteConfig(l.getToken(r))
	nodes := l.m.mustGetChildNodes(sc.DomainId, 0, 1000, 0, "created")
	var urlSet sitemap.URLSet
	for _, node := range *nodes {
		urlSet.URLs = append(urlSet.URLs, sitemap.URL{
			Loc:        "http://" + r.Host + "/list/" + strconv.Itoa(node.Id) + "/" + hfSlug(node.Title),
			LastMod:    &node.Created,
			ChangeFreq: sitemap.Daily,
			Priority:   0.7,
		})
	}
	xml, err := sitemap.Marshal(&urlSet)
	if err != nil {
		return err
	}
	w.Header().Set("Content-Type", "application/xml")
	_, err = w.Write(xml)
	return err;
}

func (l *Listboard) validateForm(r *http.Request, domainId, parentId, level int, ln *Language) (Node, ValidationErrors) {
	node := Node{
		ParentId: parentId,
		DomainId: domainId,
		Title:    strings.TrimSpace(r.FormValue("title")),
		Vote:     getVote(r.FormValue("vote")),
		Tripcode: getTripcode(r.FormValue("password")),
		Body:     r.FormValue("body"),
		Status:   statusEnabled,
		Level:    level,
	}
	errors := ValidationErrors{}
	if len(node.Title) < 3 {
		errors = append(errors, ln.Lang("Title must be at least 3 characters long"))
	}
	if len(node.Body) < 10 {
		errors = append(errors, ln.Lang("Please, write something"))
	} else {
		node.Rendered = renderText(node.Body)
		// Check again after the rendering
		if len(node.Rendered) < 10 {
			errors = append(errors, ln.Lang("Please, write something"))
		}
	}
	return node, errors
}

func (l *Listboard) getToken(r *http.Request) string {
	return r.Header.Get(l.config.Token)
}
<|endoftext|>


<commit_before>/*-
 * Copyright 2016 Square Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package sqmetrics

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"runtime"
	"time"

	"github.com/rcrowley/go-metrics"
)

// SquareMetrics posts metrics to an HTTP/JSON bridge endpoint
type SquareMetrics struct {
	registry metrics.Registry
	url      string
	prefix   string
	hostname string
}

// NewMetrics is the entry point for this code
func NewMetrics(metricsURL, metricsPrefix string, registry metrics.Registry) *SquareMetrics {
	hostname, err := os.Hostname()
	if err != nil {
		panic(err)
	}

	metrics := &SquareMetrics{
		registry: registry,
		url:      metricsURL,
		prefix:   metricsPrefix,
		hostname: hostname,
	}

	if metricsURL != "" {
		go metrics.publishMetrics()
	}

	go metrics.collectSystemMetrics()
	return metrics
}

func (mb *SquareMetrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	metrics := mb.SerializeMetrics()
	raw, err := json.Marshal(metrics)
	if err != nil {
		panic(err)
	}
	w.Write(raw)
}

// Publish metrics to bridge
func (mb *SquareMetrics) publishMetrics() {
	for range time.Tick(1 * time.Second) {
		mb.postMetrics()
	}
}

// Collect memory usage metrics
func (mb *SquareMetrics) collectSystemMetrics() {
	var mem runtime.MemStats

	update := func(name string, value uint64) {
		metrics.GetOrRegisterGauge(name, mb.registry).Update(int64(value))
	}

	updateFloat := func(name string, value float64) {
		metrics.GetOrRegisterGaugeFloat64(name, mb.registry).Update(value)
	}

	sample := metrics.NewExpDecaySample(1028, 0.015)
	gcHistogram := metrics.GetOrRegisterHistogram("runtime.mem.gc.duration", mb.registry, sample)

	var observedPauses uint32 = 0
	for range time.Tick(1 * time.Second) {
		runtime.ReadMemStats(&mem)

		update("runtime.mem.alloc", mem.Alloc)
		update("runtime.mem.total-alloc", mem.TotalAlloc)
		update("runtime.mem.sys", mem.Sys)
		update("runtime.mem.lookups", mem.Lookups)
		update("runtime.mem.mallocs", mem.Mallocs)
		update("runtime.mem.frees", mem.Frees)

		update("runtime.mem.heap.alloc", mem.HeapAlloc)
		update("runtime.mem.heap.sys", mem.HeapSys)
		update("runtime.mem.heap.idle", mem.HeapIdle)
		update("runtime.mem.heap.inuse", mem.HeapInuse)
		update("runtime.mem.heap.released", mem.HeapReleased)
		update("runtime.mem.heap.objects", mem.HeapObjects)

		update("runtime.mem.stack.inuse", mem.StackInuse)
		update("runtime.mem.stack.sys", mem.StackSys)
		update("runtime.mem.stack.sys", mem.StackSys)

		update("runtime.goroutines", uint64(runtime.NumGoroutine()))
		update("runtime.cgo-calls", uint64(runtime.NumCgoCall()))

		update("runtime.mem.gc.num-gc", uint64(mem.NumGC))
		updateFloat("runtime.mem.gc.cpu-fraction", mem.GCCPUFraction)

		// Update histogram of GC pauses
		for ; observedPauses < mem.NumGC; observedPauses++ {
			gcHistogram.Update(int64(mem.PauseNs[(observedPauses+1)%256]))
		}
	}
}

func (mb *SquareMetrics) postMetrics() {
	metrics := mb.SerializeMetrics()
	raw, err := json.Marshal(metrics)
	if err != nil {
		panic(err)
	}
	resp, err := http.Post(mb.url, "application/json", bytes.NewReader(raw))
	if err == nil {
		resp.Body.Close()
	}
}

func (mb *SquareMetrics) serializeMetric(now int64, metric tuple) map[string]interface{} {
	return map[string]interface{}{
		"timestamp": now,
		"metric":    fmt.Sprintf("%s.%s", mb.prefix, metric.name),
		"value":     metric.value,
		"hostname":  mb.hostname,
	}
}

type tuple struct {
	name  string
	value interface{}
}

// SerializeMetrics returns a map of the collected metrics, suitable for JSON marshalling
func (mb *SquareMetrics) SerializeMetrics() []map[string]interface{} {
	nvs := []tuple{}

	mb.registry.Each(func(name string, i interface{}) {
		switch metric := i.(type) {
		case metrics.Counter:
			nvs = append(nvs, tuple{name, metric.Count()})
		case metrics.Gauge:
			nvs = append(nvs, tuple{name, metric.Value()})
		case metrics.GaugeFloat64:
			nvs = append(nvs, tuple{name, metric.Value()})
		case metrics.Timer:
			timer := metric.Snapshot()
			nvs = append(nvs, []tuple{
				{fmt.Sprintf("%s.count", name), timer.Count()},
				{fmt.Sprintf("%s.min", name), timer.Min()},
				{fmt.Sprintf("%s.max", name), timer.Max()},
				{fmt.Sprintf("%s.mean", name), timer.Mean()},
				{fmt.Sprintf("%s.50-percentile", name), timer.Percentile(0.5)},
				{fmt.Sprintf("%s.75-percentile", name), timer.Percentile(0.75)},
				{fmt.Sprintf("%s.95-percentile", name), timer.Percentile(0.95)},
				{fmt.Sprintf("%s.99-percentile", name), timer.Percentile(0.99)},
			}...)
		}
	})

	now := time.Now().Unix()
	out := []map[string]interface{}{}
	for _, nv := range nvs {
		out = append(out, mb.serializeMetric(now, nv))
	}

	return out
}
<commit_msg>Support serializing histogram<commit_after>/*-
 * Copyright 2016 Square Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package sqmetrics

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"runtime"
	"time"

	"github.com/rcrowley/go-metrics"
)

// SquareMetrics posts metrics to an HTTP/JSON bridge endpoint
type SquareMetrics struct {
	registry metrics.Registry
	url      string
	prefix   string
	hostname string
}

// NewMetrics is the entry point for this code
func NewMetrics(metricsURL, metricsPrefix string, registry metrics.Registry) *SquareMetrics {
	hostname, err := os.Hostname()
	if err != nil {
		panic(err)
	}

	metrics := &SquareMetrics{
		registry: registry,
		url:      metricsURL,
		prefix:   metricsPrefix,
		hostname: hostname,
	}

	if metricsURL != "" {
		go metrics.publishMetrics()
	}

	go metrics.collectSystemMetrics()
	return metrics
}

func (mb *SquareMetrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	metrics := mb.SerializeMetrics()
	raw, err := json.Marshal(metrics)
	if err != nil {
		panic(err)
	}
	w.Write(raw)
}

// Publish metrics to bridge
func (mb *SquareMetrics) publishMetrics() {
	for range time.Tick(1 * time.Second) {
		mb.postMetrics()
	}
}

// Collect memory usage metrics
func (mb *SquareMetrics) collectSystemMetrics() {
	var mem runtime.MemStats

	update := func(name string, value uint64) {
		metrics.GetOrRegisterGauge(name, mb.registry).Update(int64(value))
	}

	updateFloat := func(name string, value float64) {
		metrics.GetOrRegisterGaugeFloat64(name, mb.registry).Update(value)
	}

	sample := metrics.NewExpDecaySample(1028, 0.015)
	gcHistogram := metrics.GetOrRegisterHistogram("runtime.mem.gc.duration", mb.registry, sample)

	var observedPauses uint32 = 0
	for range time.Tick(1 * time.Second) {
		runtime.ReadMemStats(&mem)

		update("runtime.mem.alloc", mem.Alloc)
		update("runtime.mem.total-alloc", mem.TotalAlloc)
		update("runtime.mem.sys", mem.Sys)
		update("runtime.mem.lookups", mem.Lookups)
		update("runtime.mem.mallocs", mem.Mallocs)
		update("runtime.mem.frees", mem.Frees)

		update("runtime.mem.heap.alloc", mem.HeapAlloc)
		update("runtime.mem.heap.sys", mem.HeapSys)
		update("runtime.mem.heap.idle", mem.HeapIdle)
		update("runtime.mem.heap.inuse", mem.HeapInuse)
		update("runtime.mem.heap.released", mem.HeapReleased)
		update("runtime.mem.heap.objects", mem.HeapObjects)

		update("runtime.mem.stack.inuse", mem.StackInuse)
		update("runtime.mem.stack.sys", mem.StackSys)
		update("runtime.mem.stack.sys", mem.StackSys)

		update("runtime.goroutines", uint64(runtime.NumGoroutine()))
		update("runtime.cgo-calls", uint64(runtime.NumCgoCall()))

		update("runtime.mem.gc.num-gc", uint64(mem.NumGC))
		updateFloat("runtime.mem.gc.cpu-fraction", mem.GCCPUFraction)

		// Update histogram of GC pauses
		for ; observedPauses < mem.NumGC; observedPauses++ {
			gcHistogram.Update(int64(mem.PauseNs[(observedPauses+1)%256]))
		}
	}
}

func (mb *SquareMetrics) postMetrics() {
	metrics := mb.SerializeMetrics()
	raw, err := json.Marshal(metrics)
	if err != nil {
		panic(err)
	}
	resp, err := http.Post(mb.url, "application/json", bytes.NewReader(raw))
	if err == nil {
		resp.Body.Close()
	}
}

func (mb *SquareMetrics) serializeMetric(now int64, metric tuple) map[string]interface{} {
	return map[string]interface{}{
		"timestamp": now,
		"metric":    fmt.Sprintf("%s.%s", mb.prefix, metric.name),
		"value":     metric.value,
		"hostname":  mb.hostname,
	}
}

type tuple struct {
	name  string
	value interface{}
}

// SerializeMetrics returns a map of the collected metrics, suitable for JSON marshalling
func (mb *SquareMetrics) SerializeMetrics() []map[string]interface{} {
	nvs := []tuple{}

	mb.registry.Each(func(name string, i interface{}) {
		switch metric := i.(type) {
		case metrics.Counter:
			nvs = append(nvs, tuple{name, metric.Count()})
		case metrics.Gauge:
			nvs = append(nvs, tuple{name, metric.Value()})
		case metrics.GaugeFloat64:
			nvs = append(nvs, tuple{name, metric.Value()})
		case metrics.Histogram:
			histogram := metric.Snapshot()
			nvs = append(nvs, []tuple{
				{fmt.Sprintf("%s.count", name), histogram.Count()},
				{fmt.Sprintf("%s.min", name), histogram.Min()},
				{fmt.Sprintf("%s.max", name), histogram.Max()},
				{fmt.Sprintf("%s.mean", name), histogram.Mean()},
				{fmt.Sprintf("%s.50-percentile", name), histogram.Percentile(0.5)},
				{fmt.Sprintf("%s.75-percentile", name), histogram.Percentile(0.75)},
				{fmt.Sprintf("%s.95-percentile", name), histogram.Percentile(0.95)},
				{fmt.Sprintf("%s.99-percentile", name), histogram.Percentile(0.99)},
			}...)
		case metrics.Timer:
			timer := metric.Snapshot()
			nvs = append(nvs, []tuple{
				{fmt.Sprintf("%s.count", name), timer.Count()},
				{fmt.Sprintf("%s.min", name), timer.Min()},
				{fmt.Sprintf("%s.max", name), timer.Max()},
				{fmt.Sprintf("%s.mean", name), timer.Mean()},
				{fmt.Sprintf("%s.50-percentile", name), timer.Percentile(0.5)},
				{fmt.Sprintf("%s.75-percentile", name), timer.Percentile(0.75)},
				{fmt.Sprintf("%s.95-percentile", name), timer.Percentile(0.95)},
				{fmt.Sprintf("%s.99-percentile", name), timer.Percentile(0.99)},
			}...)
		}
	})

	now := time.Now().Unix()
	out := []map[string]interface{}{}
	for _, nv := range nvs {
		out = append(out, mb.serializeMetric(now, nv))
	}

	return out
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/hashicorp/aws-sdk-go/aws"
	"github.com/hashicorp/aws-sdk-go/gen/route53"
)

func resourceAwsRoute53Record() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsRoute53RecordCreate,
		Read:   resourceAwsRoute53RecordRead,
		Update: resourceAwsRoute53RecordUpdate,
		Delete: resourceAwsRoute53RecordDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"type": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"zone_id": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"ttl": &schema.Schema{
				Type:     schema.TypeInt,
				Required: true,
			},

			"records": &schema.Schema{
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Required: true,
				Set: func(v interface{}) int {
					return hashcode.String(v.(string))
				},
			},
		},
	}
}

func resourceAwsRoute53RecordUpdate(d *schema.ResourceData, meta interface{}) error {
	// Route 53 supports CREATE, DELETE, and UPSERT actions. We use UPSERT, and
	// AWS dynamically determins if a record should be created or updated.
	// Amazon Route 53 can update an existing resource record set only when all
	// of the following values match: Name, Type
	// (and SetIdentifier, which we don't use yet).
	// See http://docs.aws.amazon.com/Route53/latest/APIReference/API_ChangeResourceRecordSets_Requests.html#change-rrsets-request-action
	//
	// Because we use UPSERT, for resouce update here we simply fall through to
	// our resource create function.
	return resourceAwsRoute53RecordCreate(d, meta)
}

func resourceAwsRoute53RecordCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).r53conn
	zone := d.Get("zone_id").(string)

	zoneRecord, err := conn.GetHostedZone(&route53.GetHostedZoneRequest{ID: aws.String(zone)})
	if err != nil {
		return err
	}

	// Get the record
	rec, err := resourceAwsRoute53RecordBuildSet(d, *zoneRecord.HostedZone.Name)
	if err != nil {
		return err
	}

	// Create the new records. We abuse StateChangeConf for this to
	// retry for us since Route53 sometimes returns errors about another
	// operation happening at the same time.
	changeBatch := &route53.ChangeBatch{
		Comment: aws.String("Managed by Terraform"),
		Changes: []route53.Change{
			route53.Change{
				Action:            aws.String("UPSERT"),
				ResourceRecordSet: rec,
			},
		},
	}

	req := &route53.ChangeResourceRecordSetsRequest{
		HostedZoneID: aws.String(cleanZoneID(*zoneRecord.HostedZone.ID)),
		ChangeBatch:  changeBatch,
	}

	log.Printf("[DEBUG] Creating resource records for zone: %s, name: %s",
		zone, *rec.Name)

	wait := resource.StateChangeConf{
		Pending:    []string{"rejected"},
		Target:     "accepted",
		Timeout:    5 * time.Minute,
		MinTimeout: 1 * time.Second,
		Refresh: func() (interface{}, string, error) {
			resp, err := conn.ChangeResourceRecordSets(req)
			if err != nil {
				if r53err, ok := err.(aws.APIError); ok {
					if r53err.Code == "PriorRequestNotComplete" {
						// There is some pending operation, so just retry
						// in a bit.
						return nil, "rejected", nil
					}
				}

				return nil, "failure", err
			}

			return resp, "accepted", nil
		},
	}

	respRaw, err := wait.WaitForState()
	if err != nil {
		return err
	}
	changeInfo := respRaw.(*route53.ChangeResourceRecordSetsResponse).ChangeInfo

	// Generate an ID
	d.SetId(fmt.Sprintf("%s_%s_%s", zone, d.Get("name").(string), d.Get("type").(string)))

	// Wait until we are done
	wait = resource.StateChangeConf{
		Delay:      30 * time.Second,
		Pending:    []string{"PENDING"},
		Target:     "INSYNC",
		Timeout:    30 * time.Minute,
		MinTimeout: 5 * time.Second,
		Refresh: func() (result interface{}, state string, err error) {
			changeRequest := &route53.GetChangeRequest{
				ID: aws.String(cleanChangeID(*changeInfo.ID)),
			}
			return resourceAwsGoRoute53Wait(conn, changeRequest)
		},
	}
	_, err = wait.WaitForState()
	if err != nil {
		return err
	}

	return nil
}

func resourceAwsRoute53RecordRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).r53conn

	zone := d.Get("zone_id").(string)

	// get expanded name
	zoneRecord, err := conn.GetHostedZone(&route53.GetHostedZoneRequest{ID: aws.String(zone)})
	if err != nil {
		return err
	}
	en := expandRecordName(d.Get("name").(string), *zoneRecord.HostedZone.Name)

	lopts := &route53.ListResourceRecordSetsRequest{
		HostedZoneID:    aws.String(cleanZoneID(zone)),
		StartRecordName: aws.String(en),
		StartRecordType: aws.String(d.Get("type").(string)),
	}

	resp, err := conn.ListResourceRecordSets(lopts)
	if err != nil {
		return err
	}

	// Scan for a matching record
	found := false
	for _, record := range resp.ResourceRecordSets {
		name := cleanRecordName(*record.Name)
		if FQDN(name) != FQDN(*lopts.StartRecordName) {
			continue
		}
		if strings.ToUpper(*record.Type) != strings.ToUpper(*lopts.StartRecordType) {
			continue
		}

		found = true

		d.Set("records", record.ResourceRecords)
		d.Set("ttl", record.TTL)

		break
	}

	if !found {
		d.SetId("")
	}

	return nil
}

func resourceAwsRoute53RecordDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).r53conn

	zone := d.Get("zone_id").(string)
	log.Printf("[DEBUG] Deleting resource records for zone: %s, name: %s",
		zone, d.Get("name").(string))
	zoneRecord, err := conn.GetHostedZone(&route53.GetHostedZoneRequest{ID: aws.String(zone)})
	if err != nil {
		return err
	}
	// Get the records
	rec, err := resourceAwsRoute53RecordBuildSet(d, *zoneRecord.HostedZone.Name)
	if err != nil {
		return err
	}

	// Create the new records
	changeBatch := &route53.ChangeBatch{
		Comment: aws.String("Deleted by Terraform"),
		Changes: []route53.Change{
			route53.Change{
				Action:            aws.String("DELETE"),
				ResourceRecordSet: rec,
			},
		},
	}

	req := &route53.ChangeResourceRecordSetsRequest{
		HostedZoneID: aws.String(cleanZoneID(zone)),
		ChangeBatch:  changeBatch,
	}

	wait := resource.StateChangeConf{
		Pending:    []string{"rejected"},
		Target:     "accepted",
		Timeout:    5 * time.Minute,
		MinTimeout: 1 * time.Second,
		Refresh: func() (interface{}, string, error) {
			_, err := conn.ChangeResourceRecordSets(req)
			if err != nil {
				if r53err, ok := err.(aws.APIError); ok {
					if r53err.Code == "PriorRequestNotComplete" {
						// There is some pending operation, so just retry
						// in a bit.
						return 42, "rejected", nil
					}

					if r53err.Code == "InvalidChangeBatch" {
						// This means that the record is already gone.
						return 42, "accepted", nil
					}
				}

				return 42, "failure", err
			}

			return 42, "accepted", nil
		},
	}

	if _, err := wait.WaitForState(); err != nil {
		return err
	}

	return nil
}

func resourceAwsRoute53RecordBuildSet(d *schema.ResourceData, zoneName string) (*route53.ResourceRecordSet, error) {
	recs := d.Get("records").(*schema.Set).List()
	records := make([]route53.ResourceRecord, 0, len(recs))

	typeStr := d.Get("type").(string)
	for _, r := range recs {
		switch typeStr {
		case "TXT":
			str := fmt.Sprintf("\"%s\"", r.(string))
			records = append(records, route53.ResourceRecord{Value: aws.String(str)})
		default:
			records = append(records, route53.ResourceRecord{Value: aws.String(r.(string))})
		}
	}

	// get expanded name
	en := expandRecordName(d.Get("name").(string), zoneName)

	// Create the RecordSet request with the fully expanded name, e.g.
	// sub.domain.com. Route 53 requires a fully qualified domain name, but does
	// not require the trailing ".", which it will itself, so we don't call FQDN
	// here.
	rec := &route53.ResourceRecordSet{
		Name:            aws.String(en),
		Type:            aws.String(d.Get("type").(string)),
		TTL:             aws.Long(int64(d.Get("ttl").(int))),
		ResourceRecords: records,
	}
	return rec, nil
}

func FQDN(name string) string {
	n := len(name)
	if n == 0 || name[n-1] == '.' {
		return name
	} else {
		return name + "."
	}
}

// Route 53 stores the "*" wildcard indicator as ASCII 42 and returns the
// octal equivalent, "\\052". Here we look for that, and convert back to "*"
// as needed.
func cleanRecordName(name string) string {
	str := name
	if strings.HasPrefix(name, "\\052") {
		str = strings.Replace(name, "\\052", "*", 1)
		log.Printf("[DEBUG] Replacing octal \\052 for * in: %s", name)
	}
	return str
}

// Check if the current record name contains the zone suffix.
// If it does not, add the zone name to form a fully qualified name
// and keep AWS happy.
func expandRecordName(name, zone string) string {
	rn := strings.TrimSuffix(name, ".")
	zone = strings.TrimSuffix(zone, ".")
	if !strings.HasSuffix(rn, zone) {
		rn = strings.Join([]string{name, zone}, ".")
	}
	return rn
}
<commit_msg>Fix comment typo<commit_after>package aws

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/hashicorp/aws-sdk-go/aws"
	"github.com/hashicorp/aws-sdk-go/gen/route53"
)

func resourceAwsRoute53Record() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsRoute53RecordCreate,
		Read:   resourceAwsRoute53RecordRead,
		Update: resourceAwsRoute53RecordUpdate,
		Delete: resourceAwsRoute53RecordDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"type": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"zone_id": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"ttl": &schema.Schema{
				Type:     schema.TypeInt,
				Required: true,
			},

			"records": &schema.Schema{
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Required: true,
				Set: func(v interface{}) int {
					return hashcode.String(v.(string))
				},
			},
		},
	}
}

func resourceAwsRoute53RecordUpdate(d *schema.ResourceData, meta interface{}) error {
	// Route 53 supports CREATE, DELETE, and UPSERT actions. We use UPSERT, and
	// AWS dynamically determines if a record should be created or updated.
	// Amazon Route 53 can update an existing resource record set only when all
	// of the following values match: Name, Type
	// (and SetIdentifier, which we don't use yet).
	// See http://docs.aws.amazon.com/Route53/latest/APIReference/API_ChangeResourceRecordSets_Requests.html#change-rrsets-request-action
	//
	// Because we use UPSERT, for resouce update here we simply fall through to
	// our resource create function.
	return resourceAwsRoute53RecordCreate(d, meta)
}

func resourceAwsRoute53RecordCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).r53conn
	zone := d.Get("zone_id").(string)

	zoneRecord, err := conn.GetHostedZone(&route53.GetHostedZoneRequest{ID: aws.String(zone)})
	if err != nil {
		return err
	}

	// Get the record
	rec, err := resourceAwsRoute53RecordBuildSet(d, *zoneRecord.HostedZone.Name)
	if err != nil {
		return err
	}

	// Create the new records. We abuse StateChangeConf for this to
	// retry for us since Route53 sometimes returns errors about another
	// operation happening at the same time.
	changeBatch := &route53.ChangeBatch{
		Comment: aws.String("Managed by Terraform"),
		Changes: []route53.Change{
			route53.Change{
				Action:            aws.String("UPSERT"),
				ResourceRecordSet: rec,
			},
		},
	}

	req := &route53.ChangeResourceRecordSetsRequest{
		HostedZoneID: aws.String(cleanZoneID(*zoneRecord.HostedZone.ID)),
		ChangeBatch:  changeBatch,
	}

	log.Printf("[DEBUG] Creating resource records for zone: %s, name: %s",
		zone, *rec.Name)

	wait := resource.StateChangeConf{
		Pending:    []string{"rejected"},
		Target:     "accepted",
		Timeout:    5 * time.Minute,
		MinTimeout: 1 * time.Second,
		Refresh: func() (interface{}, string, error) {
			resp, err := conn.ChangeResourceRecordSets(req)
			if err != nil {
				if r53err, ok := err.(aws.APIError); ok {
					if r53err.Code == "PriorRequestNotComplete" {
						// There is some pending operation, so just retry
						// in a bit.
						return nil, "rejected", nil
					}
				}

				return nil, "failure", err
			}

			return resp, "accepted", nil
		},
	}

	respRaw, err := wait.WaitForState()
	if err != nil {
		return err
	}
	changeInfo := respRaw.(*route53.ChangeResourceRecordSetsResponse).ChangeInfo

	// Generate an ID
	d.SetId(fmt.Sprintf("%s_%s_%s", zone, d.Get("name").(string), d.Get("type").(string)))

	// Wait until we are done
	wait = resource.StateChangeConf{
		Delay:      30 * time.Second,
		Pending:    []string{"PENDING"},
		Target:     "INSYNC",
		Timeout:    30 * time.Minute,
		MinTimeout: 5 * time.Second,
		Refresh: func() (result interface{}, state string, err error) {
			changeRequest := &route53.GetChangeRequest{
				ID: aws.String(cleanChangeID(*changeInfo.ID)),
			}
			return resourceAwsGoRoute53Wait(conn, changeRequest)
		},
	}
	_, err = wait.WaitForState()
	if err != nil {
		return err
	}

	return nil
}

func resourceAwsRoute53RecordRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).r53conn

	zone := d.Get("zone_id").(string)

	// get expanded name
	zoneRecord, err := conn.GetHostedZone(&route53.GetHostedZoneRequest{ID: aws.String(zone)})
	if err != nil {
		return err
	}
	en := expandRecordName(d.Get("name").(string), *zoneRecord.HostedZone.Name)

	lopts := &route53.ListResourceRecordSetsRequest{
		HostedZoneID:    aws.String(cleanZoneID(zone)),
		StartRecordName: aws.String(en),
		StartRecordType: aws.String(d.Get("type").(string)),
	}

	resp, err := conn.ListResourceRecordSets(lopts)
	if err != nil {
		return err
	}

	// Scan for a matching record
	found := false
	for _, record := range resp.ResourceRecordSets {
		name := cleanRecordName(*record.Name)
		if FQDN(name) != FQDN(*lopts.StartRecordName) {
			continue
		}
		if strings.ToUpper(*record.Type) != strings.ToUpper(*lopts.StartRecordType) {
			continue
		}

		found = true

		d.Set("records", record.ResourceRecords)
		d.Set("ttl", record.TTL)

		break
	}

	if !found {
		d.SetId("")
	}

	return nil
}

func resourceAwsRoute53RecordDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).r53conn

	zone := d.Get("zone_id").(string)
	log.Printf("[DEBUG] Deleting resource records for zone: %s, name: %s",
		zone, d.Get("name").(string))
	zoneRecord, err := conn.GetHostedZone(&route53.GetHostedZoneRequest{ID: aws.String(zone)})
	if err != nil {
		return err
	}
	// Get the records
	rec, err := resourceAwsRoute53RecordBuildSet(d, *zoneRecord.HostedZone.Name)
	if err != nil {
		return err
	}

	// Create the new records
	changeBatch := &route53.ChangeBatch{
		Comment: aws.String("Deleted by Terraform"),
		Changes: []route53.Change{
			route53.Change{
				Action:            aws.String("DELETE"),
				ResourceRecordSet: rec,
			},
		},
	}

	req := &route53.ChangeResourceRecordSetsRequest{
		HostedZoneID: aws.String(cleanZoneID(zone)),
		ChangeBatch:  changeBatch,
	}

	wait := resource.StateChangeConf{
		Pending:    []string{"rejected"},
		Target:     "accepted",
		Timeout:    5 * time.Minute,
		MinTimeout: 1 * time.Second,
		Refresh: func() (interface{}, string, error) {
			_, err := conn.ChangeResourceRecordSets(req)
			if err != nil {
				if r53err, ok := err.(aws.APIError); ok {
					if r53err.Code == "PriorRequestNotComplete" {
						// There is some pending operation, so just retry
						// in a bit.
						return 42, "rejected", nil
					}

					if r53err.Code == "InvalidChangeBatch" {
						// This means that the record is already gone.
						return 42, "accepted", nil
					}
				}

				return 42, "failure", err
			}

			return 42, "accepted", nil
		},
	}

	if _, err := wait.WaitForState(); err != nil {
		return err
	}

	return nil
}

func resourceAwsRoute53RecordBuildSet(d *schema.ResourceData, zoneName string) (*route53.ResourceRecordSet, error) {
	recs := d.Get("records").(*schema.Set).List()
	records := make([]route53.ResourceRecord, 0, len(recs))

	typeStr := d.Get("type").(string)
	for _, r := range recs {
		switch typeStr {
		case "TXT":
			str := fmt.Sprintf("\"%s\"", r.(string))
			records = append(records, route53.ResourceRecord{Value: aws.String(str)})
		default:
			records = append(records, route53.ResourceRecord{Value: aws.String(r.(string))})
		}
	}

	// get expanded name
	en := expandRecordName(d.Get("name").(string), zoneName)

	// Create the RecordSet request with the fully expanded name, e.g.
	// sub.domain.com. Route 53 requires a fully qualified domain name, but does
	// not require the trailing ".", which it will itself, so we don't call FQDN
	// here.
	rec := &route53.ResourceRecordSet{
		Name:            aws.String(en),
		Type:            aws.String(d.Get("type").(string)),
		TTL:             aws.Long(int64(d.Get("ttl").(int))),
		ResourceRecords: records,
	}
	return rec, nil
}

func FQDN(name string) string {
	n := len(name)
	if n == 0 || name[n-1] == '.' {
		return name
	} else {
		return name + "."
	}
}

// Route 53 stores the "*" wildcard indicator as ASCII 42 and returns the
// octal equivalent, "\\052". Here we look for that, and convert back to "*"
// as needed.
func cleanRecordName(name string) string {
	str := name
	if strings.HasPrefix(name, "\\052") {
		str = strings.Replace(name, "\\052", "*", 1)
		log.Printf("[DEBUG] Replacing octal \\052 for * in: %s", name)
	}
	return str
}

// Check if the current record name contains the zone suffix.
// If it does not, add the zone name to form a fully qualified name
// and keep AWS happy.
func expandRecordName(name, zone string) string {
	rn := strings.TrimSuffix(name, ".")
	zone = strings.TrimSuffix(zone, ".")
	if !strings.HasSuffix(rn, zone) {
		rn = strings.Join([]string{name, zone}, ".")
	}
	return rn
}
<|endoftext|>


<commit_before>// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/luci/luci-go/common/logging"
	"github.com/luci/luci-go/common/logging/gologger"
	gol "github.com/op/go-logging"
	"golang.org/x/net/context"
)

var (
	verbose    = flag.Bool("verbose", false, "print debug messages to stderr")
	withGithub = flag.Bool(
		"with-github", true,
		"include $GOPATH/src/github.com in proto search path")
	withGoogleProtobuf = flag.Bool(
		"with-google-protobuf", true,
		"map .proto files in gitub.com/luci/luci-go/common/proto/google to google/protobuf/*.proto")
)

func resolveGoogleProtobufPackages(c context.Context) (map[string]string, error) {
	const (
		protoPrefix = "google/protobuf/"
		pkgPath     = "github.com/luci/luci-go/common/proto/google"
	)
	var result = map[string]string{
		protoPrefix + "descriptor.proto": pkgPath + "/descriptor", //snowflake
	}

	pkg, err := build.Import(pkgPath, "", build.FindOnly)
	if err != nil {
		return nil, err
	}
	protoFiles, err := findProtoFiles(pkg.Dir)
	if err != nil {
		return nil, err
	}
	if len(protoFiles) == 0 {
		logging.Warningf(c, "no .proto files in %s", pkg.Dir)
		return nil, nil
	}
	for _, protoFile := range protoFiles {
		result[protoPrefix+filepath.Base(protoFile)] = pkgPath
	}
	return result, nil
}

func protoc(c context.Context, protoFiles []string, dir, descSetOut string) error {
	var params []string

	if *withGoogleProtobuf {
		googlePackages, err := resolveGoogleProtobufPackages(c)
		if err != nil {
			return err
		}
		for k, v := range googlePackages {
			params = append(params, fmt.Sprintf("M%s=%s", k, v))
		}
	}

	params = append(params, "plugins=grpc")
	goOut := fmt.Sprintf("%s:%s", strings.Join(params, ","), dir)

	args := []string{
		"--proto_path=" + dir,
		"--go_out=" + goOut,
		"--descriptor_set_out=" + descSetOut,
		"--include_imports",
		"--include_source_info",
	}

	if *withGithub {
		for _, p := range strings.Split(os.Getenv("GOPATH"), string(filepath.ListSeparator)) {
			path := filepath.Join(p, "src", "github.com")
			if info, err := os.Stat(path); os.IsNotExist(err) || !info.IsDir() {
				continue
			} else if err != nil {
				return err
			}
			args = append(args, "--proto_path="+path)
		}
	}

	args = append(args, protoFiles...)
	logging.Infof(c, "protoc %s", strings.Join(args, " "))
	protoc := exec.Command("protoc", args...)
	protoc.Stdout = os.Stdout
	protoc.Stderr = os.Stderr
	return protoc.Run()
}

func run(c context.Context, dir string) error {
	if s, err := os.Stat(dir); os.IsNotExist(err) {
		return fmt.Errorf("%s does not exist", dir)
	} else if err != nil {
		return err
	} else if !s.IsDir() {
		return fmt.Errorf("%s is not a directory", dir)
	}

	// Find .proto files
	protoFiles, err := findProtoFiles(dir)
	if err != nil {
		return err
	}
	if len(protoFiles) == 0 {
		return fmt.Errorf(".proto files not found")
	}

	// Compile all .proto files.
	tmpDir, err := ioutil.TempDir("", "")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpDir)
	descPath := filepath.Join(tmpDir, "package.desc")
	if err := protoc(c, protoFiles, dir, descPath); err != nil {
		return err
	}

	// Transform .go files
	var packageName string
	for _, p := range protoFiles {
		goFile := strings.TrimSuffix(p, ".proto") + ".pb.go"
		var t transformer
		if err := t.transformGoFile(goFile); err != nil {
			return fmt.Errorf("could not transform %s: %s", goFile, err)
		}
		if packageName == "" {
			packageName = t.PackageName
		}
	}

	return genDiscoveryFile(c, filepath.Join(dir, "pb.discovery.go"), descPath, packageName)
}

func setupLogging(c context.Context) context.Context {
	logCfg := gologger.LoggerConfig{
		Format: gologger.StandardFormat,
		Out:    os.Stderr,
		Level:  gol.WARNING,
	}
	if *verbose {
		logCfg.Level = gol.DEBUG
	}
	return logCfg.Use(c)
}

func usage() {
	fmt.Fprintln(os.Stderr,
		`Compiles all .proto files in a directory to .go with grpc+prpc support.
usage: cproto [flags] [dir]

Flags:`)
	flag.PrintDefaults()
}

func main() {
	flag.Usage = usage
	flag.Parse()

	if flag.NArg() > 1 {
		flag.Usage()
		os.Exit(1)
	}
	dir := "."
	if flag.NArg() == 1 {
		dir = flag.Arg(0)
	}

	c := setupLogging(context.Background())
	if err := run(c, dir); err != nil {
		exitCode := 1
		if exit, ok := err.(*exec.ExitError); ok {
			if wait, ok := exit.Sys().(syscall.WaitStatus); ok {
				exitCode = wait.ExitStatus()
			}
		} else {
			fmt.Fprintln(os.Stderr, err.Error())
		}
		os.Exit(exitCode)
	}
}

func findProtoFiles(dir string) ([]string, error) {
	return filepath.Glob(filepath.Join(dir, "*.proto"))
}

// isInPackage returns true if the filename is a part of the package.
func isInPackage(fileName string, pkg string) (bool, error) {
	dir, err := filepath.Abs(filepath.Dir(fileName))
	if err != nil {
		return false, err
	}
	dir = path.Clean(dir)
	pkg = path.Clean(pkg)
	if !strings.HasSuffix(dir, pkg) {
		return false, nil
	}

	src := strings.TrimSuffix(dir, pkg)
	src = path.Clean(src)
	goPaths := strings.Split(os.Getenv("GOPATH"), string(filepath.ListSeparator))
	for _, goPath := range goPaths {
		if filepath.Join(goPath, "src") == src {
			return true, nil
		}
	}
	return false, nil
}
<commit_msg>tools/cmd/prpcgen: add -test option<commit_after>// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/luci/luci-go/common/logging"
	"github.com/luci/luci-go/common/logging/gologger"
	gol "github.com/op/go-logging"
	"golang.org/x/net/context"
)

var (
	verbose    = flag.Bool("verbose", false, "print debug messages to stderr")
	withGithub = flag.Bool(
		"with-github", true,
		"include $GOPATH/src/github.com in proto search path")
	withGoogleProtobuf = flag.Bool(
		"with-google-protobuf", true,
		"map .proto files in gitub.com/luci/luci-go/common/proto/google to google/protobuf/*.proto")
	renameToTestGo = flag.Bool(
		"test", false,
		"rename generated files from *.go to *_test.go")
)

func resolveGoogleProtobufPackages(c context.Context) (map[string]string, error) {
	const (
		protoPrefix = "google/protobuf/"
		pkgPath     = "github.com/luci/luci-go/common/proto/google"
	)
	var result = map[string]string{
		protoPrefix + "descriptor.proto": pkgPath + "/descriptor", //snowflake
	}

	pkg, err := build.Import(pkgPath, "", build.FindOnly)
	if err != nil {
		return nil, err
	}
	protoFiles, err := findProtoFiles(pkg.Dir)
	if err != nil {
		return nil, err
	}
	if len(protoFiles) == 0 {
		logging.Warningf(c, "no .proto files in %s", pkg.Dir)
		return nil, nil
	}
	for _, protoFile := range protoFiles {
		result[protoPrefix+filepath.Base(protoFile)] = pkgPath
	}
	return result, nil
}

func protoc(c context.Context, protoFiles []string, dir, descSetOut string) error {
	var params []string

	if *withGoogleProtobuf {
		googlePackages, err := resolveGoogleProtobufPackages(c)
		if err != nil {
			return err
		}
		for k, v := range googlePackages {
			params = append(params, fmt.Sprintf("M%s=%s", k, v))
		}
	}

	params = append(params, "plugins=grpc")
	goOut := fmt.Sprintf("%s:%s", strings.Join(params, ","), dir)

	args := []string{
		"--proto_path=" + dir,
		"--go_out=" + goOut,
		"--descriptor_set_out=" + descSetOut,
		"--include_imports",
		"--include_source_info",
	}

	if *withGithub {
		for _, p := range strings.Split(os.Getenv("GOPATH"), string(filepath.ListSeparator)) {
			path := filepath.Join(p, "src", "github.com")
			if info, err := os.Stat(path); os.IsNotExist(err) || !info.IsDir() {
				continue
			} else if err != nil {
				return err
			}
			args = append(args, "--proto_path="+path)
		}
	}

	args = append(args, protoFiles...)
	logging.Infof(c, "protoc %s", strings.Join(args, " "))
	protoc := exec.Command("protoc", args...)
	protoc.Stdout = os.Stdout
	protoc.Stderr = os.Stderr
	return protoc.Run()
}

func run(c context.Context, dir string) error {
	if s, err := os.Stat(dir); os.IsNotExist(err) {
		return fmt.Errorf("%s does not exist", dir)
	} else if err != nil {
		return err
	} else if !s.IsDir() {
		return fmt.Errorf("%s is not a directory", dir)
	}

	// Find .proto files
	protoFiles, err := findProtoFiles(dir)
	if err != nil {
		return err
	}
	if len(protoFiles) == 0 {
		return fmt.Errorf(".proto files not found")
	}

	// Compile all .proto files.
	tmpDir, err := ioutil.TempDir("", "")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpDir)
	descPath := filepath.Join(tmpDir, "package.desc")
	if err := protoc(c, protoFiles, dir, descPath); err != nil {
		return err
	}

	// Transform .go files
	var packageName string
	for _, p := range protoFiles {
		goFile := strings.TrimSuffix(p, ".proto") + ".pb.go"
		var t transformer
		if err := t.transformGoFile(goFile); err != nil {
			return fmt.Errorf("could not transform %s: %s", goFile, err)
		}

		if packageName == "" {
			packageName = t.PackageName
		}

		if *renameToTestGo {
			newName := strings.TrimSuffix(goFile, ".go") + "_test.go"
			if err := os.Rename(goFile, newName); err != nil {
				return err
			}
		}
	}

	// Generate pb.prpc.go
	discoveryFile := "pb.discovery.go"
	if *renameToTestGo {
		discoveryFile = "pb.discovery_test.go"
	}
	return genDiscoveryFile(c, filepath.Join(dir, discoveryFile), descPath, packageName)
}

func setupLogging(c context.Context) context.Context {
	logCfg := gologger.LoggerConfig{
		Format: gologger.StandardFormat,
		Out:    os.Stderr,
		Level:  gol.WARNING,
	}
	if *verbose {
		logCfg.Level = gol.DEBUG
	}
	return logCfg.Use(c)
}

func usage() {
	fmt.Fprintln(os.Stderr,
		`Compiles all .proto files in a directory to .go with grpc+prpc support.
usage: cproto [flags] [dir]

Flags:`)
	flag.PrintDefaults()
}

func main() {
	flag.Usage = usage
	flag.Parse()

	if flag.NArg() > 1 {
		flag.Usage()
		os.Exit(1)
	}
	dir := "."
	if flag.NArg() == 1 {
		dir = flag.Arg(0)
	}

	c := setupLogging(context.Background())
	if err := run(c, dir); err != nil {
		exitCode := 1
		if exit, ok := err.(*exec.ExitError); ok {
			if wait, ok := exit.Sys().(syscall.WaitStatus); ok {
				exitCode = wait.ExitStatus()
			}
		} else {
			fmt.Fprintln(os.Stderr, err.Error())
		}
		os.Exit(exitCode)
	}
}

func findProtoFiles(dir string) ([]string, error) {
	return filepath.Glob(filepath.Join(dir, "*.proto"))
}

// isInPackage returns true if the filename is a part of the package.
func isInPackage(fileName string, pkg string) (bool, error) {
	dir, err := filepath.Abs(filepath.Dir(fileName))
	if err != nil {
		return false, err
	}
	dir = path.Clean(dir)
	pkg = path.Clean(pkg)
	if !strings.HasSuffix(dir, pkg) {
		return false, nil
	}

	src := strings.TrimSuffix(dir, pkg)
	src = path.Clean(src)
	goPaths := strings.Split(os.Getenv("GOPATH"), string(filepath.ListSeparator))
	for _, goPath := range goPaths {
		if filepath.Join(goPath, "src") == src {
			return true, nil
		}
	}
	return false, nil
}
<|endoftext|>


<commit_before>package main

import "fmt"

import "path/filepath"
import "log"

import (
	"os"
	"os/exec"
)
import "strings"

type marker struct {
	name string
	path string
	link *volume
}

type volume struct {
	dir  string
	mark *marker
}

type container struct {
	name    string
	id      string
	stopped bool
	volumes []volume
}

func mustcmd(acmd string) string {
	fmt.Println(acmd)
	out, err := cmd(acmd)
	if err != nil {
		log.Fatal(fmt.Sprintf("out='%s', err='%s'", out, err))
	}
	return string(out)
}

func cmd(cmd string) (string, error) {
	out, err := exec.Command("sh", "-c", cmd).Output()
	return strings.TrimSpace(string(out)), err
}

func readVolumes() {
	out := mustcmd("sudo ls -1 /mnt/sda1/var/lib/docker/vfs/dir")
	vollines := strings.Split(out, "\n")
	fmt.Println(vollines)
	for _, volline := range vollines {
		dir := volline
		dirlink, err := cmd(fmt.Sprintf("sudo readlink /mnt/sda1/var/lib/docker/vfs/dir/%s", dir))
		fmt.Printf("---\ndir: '%s'\ndlk: '%s'\nerr='%v'", dir, dirlink, err)
		if err != nil {
			fmt.Printf("Invalid dir detected: '%s'\n", dir)
			mustcmd(fmt.Sprintf("sudo rm /mnt/sda1/var/lib/docker/vfs/dir/%s", dir))
		}
	}
}

// docker run --rm -i -t -v `pwd`:`pwd` -w `pwd` --entrypoint="/bin/bash" go -c 'go build gcl.go'
func main() {
	readVolumes()
	os.Exit(0)
	out := mustcmd("docker ps -qa --no-trunc")
	containers := strings.Split(out, "\n")
	containers = containers[:len(containers)-1]
	for _, id := range containers {
		fmt.Printf("container id: %s\n", id)
		out = mustcmd(fmt.Sprintf("docker inspect -f '{{ .Name }}' %s", id))
		name := strings.TrimSpace(out)[1:]
		fmt.Printf("  name: '%s'\n", name)
		out = mustcmd(fmt.Sprintf("docker inspect -f '{{ range $key, $value := .Volumes }}{{ $key }},{{ $value }}##~#{{ end }}' %s", name))
		volumes := strings.Split(out, "##~#")
		volumes = volumes[:len(volumes)-1]
		fmt.Printf("    volumes: '%v': %d\n", volumes, len(volumes))
		for i, volume := range volumes {
			vols := strings.Split(volume, ",")
			path := vols[0]
			vfs := vols[1]
			fmt.Printf("      volume %d: '%v'=>'%v'\n", i, path, vfs)
			if strings.Contains(vfs, "/var/lib/docker/vfs/dir/") {
				vfs2, _ := cmd(fmt.Sprintf("sudo readlink %s 2> /dev/null", vfs))
				vfs2 = strings.TrimSpace(vfs2)
				fmt.Printf("      vfs2 '%s' from sudo readlink %s\n", vfs2, vfs)
				link := "." + name + "###" + strings.Replace(path, "/", ",#,", -1)
				fmt.Printf("      link '%s'\n", link)
				ls, _ := cmd(fmt.Sprintf("sudo readlink /var/lib/docker/vfs/dir/%s 2> /dev/null", link))
				ls = strings.TrimSpace(ls)
				fmt.Printf("      ls link '%s'\n", ls)
				dir := filepath.Base(vfs)
				if vfs2 != "" {
					dir = filepath.Base(vfs2)
				}
				if ls == "" {
					fmt.Printf("      ln '%s' to '%s'\n", link, dir)
					mustcmd(fmt.Sprintf("sudo ln -s %s /var/lib/docker/vfs/dir/%s", dir, link))
				}
				if ls != dir && ls != "" {
					apathdir := fmt.Sprintf("/var/lib/docker/vfs/dir/%s", dir)
					mustcmd(fmt.Sprintf("sudo rm -Rf %s", apathdir))
					acmd := fmt.Sprintf("sudo ln -s %s %s", ls, apathdir)
					fmt.Printf("      must redirect '%s' to '%s'\n%s\n", dir, ls, acmd)
					mustcmd(acmd)
				}
			}
		}
	}
}
<commit_msg>gcl.go: uses os.Stat() for reading vfs folders<commit_after>package main

import "fmt"

import "path/filepath"
import "log"

import (
	"os"
	"os/exec"
)
import "strings"

type marker struct {
	name string
	path string
	link *volume
}

type volume struct {
	dir  string
	mark *marker
}

type container struct {
	name    string
	id      string
	stopped bool
	volumes []volume
}

func mustcmd(acmd string) string {
	fmt.Println(acmd)
	out, err := cmd(acmd)
	if err != nil {
		log.Fatal(fmt.Sprintf("out='%s', err='%s'", out, err))
	}
	return string(out)
}

func cmd(cmd string) (string, error) {
	out, err := exec.Command("sh", "-c", cmd).Output()
	return strings.TrimSpace(string(out)), err
}

func readVolumes() {
	out := mustcmd("sudo ls -1 /mnt/sda1/var/lib/docker/vfs/dir")
	vollines := strings.Split(out, "\n")
	fmt.Println(vollines)
	for _, volline := range vollines {
		dir := volline
		fdir := fmt.Sprintf("/mnt/sda1/var/lib/docker/vfs/dir/", dir)
		dirlink, err := cmd(fdir)
		fmt.Printf("---\ndir: '%s'\ndlk: '%s'\nerr='%v'", dir, dirlink, err)
		if err != nil {
			finfo, err := os.Stat(fdir)
			if err != nil {
				fmt.Printf("Invalid dir detected: '%s' (%s)\n", dir, err)
			}
			if finfo.IsDir() {
				fmt.Printf("dir detected: '%s'\n", dir)
			} else {
				fmt.Printf("Invalid dir (file) detected: '%s'\n", dir)
			}
			// mustcmd(fmt.Sprintf("sudo rm /mnt/sda1/var/lib/docker/vfs/dir/%s", dir))
		}
	}
}

// docker run --rm -i -t -v `pwd`:`pwd` -w `pwd` --entrypoint="/bin/bash" go -c 'go build gcl.go'
func main() {
	readVolumes()
	os.Exit(0)
	out := mustcmd("docker ps -qa --no-trunc")
	containers := strings.Split(out, "\n")
	containers = containers[:len(containers)-1]
	for _, id := range containers {
		fmt.Printf("container id: %s\n", id)
		out = mustcmd(fmt.Sprintf("docker inspect -f '{{ .Name }}' %s", id))
		name := strings.TrimSpace(out)[1:]
		fmt.Printf("  name: '%s'\n", name)
		out = mustcmd(fmt.Sprintf("docker inspect -f '{{ range $key, $value := .Volumes }}{{ $key }},{{ $value }}##~#{{ end }}' %s", name))
		volumes := strings.Split(out, "##~#")
		volumes = volumes[:len(volumes)-1]
		fmt.Printf("    volumes: '%v': %d\n", volumes, len(volumes))
		for i, volume := range volumes {
			vols := strings.Split(volume, ",")
			path := vols[0]
			vfs := vols[1]
			fmt.Printf("      volume %d: '%v'=>'%v'\n", i, path, vfs)
			if strings.Contains(vfs, "/var/lib/docker/vfs/dir/") {
				vfs2, _ := cmd(fmt.Sprintf("sudo readlink %s 2> /dev/null", vfs))
				vfs2 = strings.TrimSpace(vfs2)
				fmt.Printf("      vfs2 '%s' from sudo readlink %s\n", vfs2, vfs)
				link := "." + name + "###" + strings.Replace(path, "/", ",#,", -1)
				fmt.Printf("      link '%s'\n", link)
				ls, _ := cmd(fmt.Sprintf("sudo readlink /var/lib/docker/vfs/dir/%s 2> /dev/null", link))
				ls = strings.TrimSpace(ls)
				fmt.Printf("      ls link '%s'\n", ls)
				dir := filepath.Base(vfs)
				if vfs2 != "" {
					dir = filepath.Base(vfs2)
				}
				if ls == "" {
					fmt.Printf("      ln '%s' to '%s'\n", link, dir)
					mustcmd(fmt.Sprintf("sudo ln -s %s /var/lib/docker/vfs/dir/%s", dir, link))
				}
				if ls != dir && ls != "" {
					apathdir := fmt.Sprintf("/var/lib/docker/vfs/dir/%s", dir)
					mustcmd(fmt.Sprintf("sudo rm -Rf %s", apathdir))
					acmd := fmt.Sprintf("sudo ln -s %s %s", ls, apathdir)
					fmt.Printf("      must redirect '%s' to '%s'\n%s\n", dir, ls, acmd)
					mustcmd(acmd)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type mpath struct {
	name string
	path string
}

type vdir string

func (vd vdir) trunc() string {
	s := string(vd)
	if len(s) > 0 {
		return s[:7]
	}
	return ""
}

var mpathre = regexp.MustCompile(`^\.(.*)###(.*)$`)
var vdirre = regexp.MustCompile(`^[a-f0-9]{64}$`)

func newmpath(mp string) *mpath {
	res := mpathre.FindAllStringSubmatch(mp, -1)
	var mres *mpath
	if res != nil && len(res) == 1 && len(res[0]) == 3 {
		name := res[0][1]
		path := res[0][2]
		path = strings.Replace(path, ",#,", "/", -1)
		mres = &mpath{name, path}
	}
	return mres
}

func (mp *mpath) String() string {
	return "." + mp.name + "###" + strings.Replace(mp.path, "/", ",#,", -1)
}

func newvdir(vd string) vdir {
	res := vdirre.FindString(vd)
	vres := vdir("")
	if res != "" && res == vd {
		vres = vdir(vd)
	}
	return vres
}

type marker struct {
	mp  *mpath
	dir vdir
}

func (mk *marker) String() string {
	return fmt.Sprintf("marker '%s'<%s->%s>", mk.dir.trunc(), mk.mp.name, mk.mp.path)
}

type markers []*marker

var allmarkers = markers{}

func (marks markers) getMarker(name string, path string, dir vdir) *marker {
	var mark *marker
	for _, marker := range marks {
		if marker.mp.name == name {
			if marker.mp.path == path {
				mark = marker
				break
			} else {
				fmt.Printf("Invalid marker path detected: '%s' (vs. container volume path '%s')\n", marker.mp.path, path)
				return nil
			}
		}
	}
	if mark == nil {
		mark = &marker{mp: &mpath{name: name, path: path}, dir: dir}
	}
	if mark.dir != dir {
		// TODO: move dir and ln -s mark.dir dir
		fmt.Printf("Mark container named '%s' for path '%s' as link to '%s' (from '%s')\n", name, path, mark.dir, dir)
	}
	// TODO: check that link exists
	return mark
}

type volume struct {
	dir  vdir
	mark *marker
}

type volumes []*volume

var allvolumes = volumes{}

func (v *volume) String() string {
	return fmt.Sprintf("vol '%s'<%v>", v.dir.trunc(), v.mark)
}

func (vols volumes) getVolume(vd vdir, path string, name string) *volume {
	var vol *volume
	for _, volume := range vols {
		if string(volume.dir) == string(vd) {
			vol = volume
			if vol.mark == nil {
				vol.mark = allmarkers.getMarker(name, path, vol.dir)
			}
			if vol.mark == nil {
				return nil
			}
			break
		}
	}
	if vol == nil {
		vol = &volume{dir: vd}
		vol.mark = allmarkers.getMarker(name, path, vol.dir)
		if vol.mark == nil {
			return nil
		}
	}
	return vol
}

type container struct {
	name    string
	id      string
	stopped bool
	volumes []*volume
}

func (c *container) trunc() string {
	if len(c.id) > 0 {
		return c.id[:7]
	}
	return ""
}

func (c *container) String() string {
	return "cnt '" + c.name + "' (" + c.trunc() + ")" + fmt.Sprintf("[%v] - %d vol", c.stopped, len(c.volumes))
}

var containers = []*container{}

func mustcmd(acmd string) string {
	out, err := cmd(acmd)
	if err != nil {
		log.Fatal(fmt.Sprintf("out='%s', err='%s'", out, err))
	}
	return string(out)
}

func cmd(cmd string) (string, error) {
	fmt.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	return strings.TrimSpace(string(out)), err
}

func readVolumes() {
	out := mustcmd("sudo ls -a1F /mnt/sda1/var/lib/docker/vfs/dir")
	vollines := strings.Split(out, "\n")
	// fmt.Println(vollines)
	for _, volline := range vollines {
		dir := volline
		if dir == "./" || dir == "../" {
			continue
		}
		if strings.HasSuffix(dir, "@") {
			dir = dir[:len(dir)-1]
			fdir := fmt.Sprintf("/mnt/sda1/var/lib/docker/vfs/dir/%s", dir)
			mp := newmpath(dir)
			if mp == nil {
				fmt.Printf("Invalid marker detected: '%s'\n", dir)
				mustcmd("sudo rm " + fdir)
			} else {
				dirlink, err := cmd("sudo readlink " + fdir)
				fmt.Printf("---\ndir: '%s'\ndlk: '%s'\nerr='%v'\n", dir, dirlink, err)
				if err != nil {
					fmt.Printf("Invalid marker (no readlink) detected: '%s'\n", dir)
					mustcmd("sudo rm " + fdir)
				} else {
					_, err := cmd("sudo ls /mnt/sda1/var/lib/docker/vfs/dir/" + dirlink)
					if err != nil {
						fmt.Printf("Invalid marker (readlink no ls) detected: '%s'\n", dir)
						mustcmd("sudo rm " + fdir)
					} else {
						vd := newvdir(dirlink)
						if vd == "" {
							fmt.Printf("Invalid marker (readlink no vdir) detected: '%s'\n", dir)
							mustcmd("sudo rm " + fdir)
						} else {
							allmarkers = append(allmarkers, &marker{mp, vd})
						}
					}
				}
			}
		} else if strings.HasSuffix(dir, "/") {

			dir = dir[:len(dir)-1]
			fdir := fmt.Sprintf("/mnt/sda1/var/lib/docker/vfs/dir/%s", dir)
			vd := newvdir(dir)
			if vd == "" {
				fmt.Printf("Invalid volume folder detected: '%s'\n", dir)
				mustcmd("sudo rm " + fdir)
			} else {
				allvolumes = append(allvolumes, &volume{dir: vd})
			}
		} else {
			fdir := fmt.Sprintf("/mnt/sda1/var/lib/docker/vfs/dir/%s", dir)
			fmt.Printf("Invalid file detected: '%s'\n", dir)
			mustcmd("sudo rm " + fdir)
		}
	}
	fmt.Printf("volumes: %v\nmarkers: %v\n", allvolumes, allmarkers)
}

func readContainer() {

	out := mustcmd("docker ps -aq --no-trunc")
	contlines := strings.Split(out, "\n")
	// fmt.Println(contlines)
	for _, contline := range contlines {
		id := contline
		res := mustcmd("docker inspect -f '{{ .Name }},{{ range $key, $value := .Volumes }}{{ $key }},{{ $value }}##~#{{ end }}' " + id)
		// fmt.Println("res1: '" + res + "'")
		name := res[1:strings.Index(res, ",")]
		cont := &container{name: name, id: id}
		res = res[strings.Index(res, ",")+1:]
		// fmt.Println("res2: '" + res + "'")
		vols := strings.Split(res, "##~#")
		// fmt.Println(vols)
		for _, vol := range vols {
			elts := strings.Split(vol, ",")
			if len(elts) == 2 {
				// fmt.Printf("elts: '%v'\n", elts)
				path := elts[0]
				vfs := elts[1]
				if strings.Contains(vfs, "/var/lib/docker/vfs/dir/") {
					vd := newvdir(filepath.Base(vfs))
					if vd == "" {
						fmt.Printf("Invalid volume folder detected: '%s'\n", vfs)
						break
					}
					newvol := allvolumes.getVolume(vd, path, name)
					cont.volumes = append(cont.volumes, newvol)
				}
			}
		}
		containers = append(containers, cont)
	}
	fmt.Printf("containers: %v\n", containers)
}

// TODO check if check steps are still necessary
func (v *volume) accept(m *marker) bool {
	// TODO
	return false
}

func checkContainers() {
	for _, container := range containers {
		for _, volume := range container.volumes {
			if volume.mark == nil {
				for _, mark := range allmarkers {
					if volume.accept(mark) {
						fmt.Printf("Set mark '%v' to volume '%v' of container '%v'\n", mark, volume, container)
						volume.mark = mark
						// TODO check if ln is needed
					}
				}
			}
			if volume.mark == nil {
				// TODO check if vfs folder exist.
				// If it does, make the marker
			}
		}
	}
}

func (c *container) accept(v *volume) bool {
	// TODO
	return false
}

func checkVolumes() {
	for _, volume := range allvolumes {
		orphan := true
		for _, container := range containers {
			if container.accept(volume) {
				orphan = false
				break
			}
		}
		if orphan {
			fmt.Printf("Orphan detected, volume '%v'\n", volume)
			// TODO rm if necessary or at least mv _xxx
		}
	}
}

// docker run --rm -i -t -v `pwd`:`pwd` -w `pwd` --entrypoint="/bin/bash" go -c 'go build gcl.go'
func main() {
	fmt.Println("### 1/4 readVolumes     ###")
	readVolumes()
	fmt.Println("### 2/4 readContainer   ###")
	readContainer()
	fmt.Println("### 3/4 checkContainers ###")
	checkContainers()
	fmt.Println("### 4/4 checkVolumes    ###")
	checkVolumes()
	fmt.Println("---      All done       ---")
	os.Exit(0)
}
<commit_msg>gcl.go: removes checkContainers step<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type mpath struct {
	name string
	path string
}

type vdir string

func (vd vdir) trunc() string {
	s := string(vd)
	if len(s) > 0 {
		return s[:7]
	}
	return ""
}

var mpathre = regexp.MustCompile(`^\.(.*)###(.*)$`)
var vdirre = regexp.MustCompile(`^[a-f0-9]{64}$`)

func newmpath(mp string) *mpath {
	res := mpathre.FindAllStringSubmatch(mp, -1)
	var mres *mpath
	if res != nil && len(res) == 1 && len(res[0]) == 3 {
		name := res[0][1]
		path := res[0][2]
		path = strings.Replace(path, ",#,", "/", -1)
		mres = &mpath{name, path}
	}
	return mres
}

func (mp *mpath) String() string {
	return "." + mp.name + "###" + strings.Replace(mp.path, "/", ",#,", -1)
}

func newvdir(vd string) vdir {
	res := vdirre.FindString(vd)
	vres := vdir("")
	if res != "" && res == vd {
		vres = vdir(vd)
	}
	return vres
}

type marker struct {
	mp  *mpath
	dir vdir
}

func (mk *marker) String() string {
	return fmt.Sprintf("marker '%s'<%s->%s>", mk.dir.trunc(), mk.mp.name, mk.mp.path)
}

type markers []*marker

var allmarkers = markers{}

func (marks markers) getMarker(name string, path string, dir vdir) *marker {
	var mark *marker
	for _, marker := range marks {
		if marker.mp.name == name {
			if marker.mp.path == path {
				mark = marker
				break
			} else {
				fmt.Printf("Invalid marker path detected: '%s' (vs. container volume path '%s')\n", marker.mp.path, path)
				return nil
			}
		}
	}
	if mark == nil {
		mark = &marker{mp: &mpath{name: name, path: path}, dir: dir}
	}
	if mark.dir != dir {
		// TODO: move dir and ln -s mark.dir dir
		fmt.Printf("Mark container named '%s' for path '%s' as link to '%s' (from '%s')\n", name, path, mark.dir, dir)
	}
	// TODO: check that link exists
	return mark
}

type volume struct {
	dir  vdir
	mark *marker
}

type volumes []*volume

var allvolumes = volumes{}

func (v *volume) String() string {
	return fmt.Sprintf("vol '%s'<%v>", v.dir.trunc(), v.mark)
}

func (vols volumes) getVolume(vd vdir, path string, name string) *volume {
	var vol *volume
	for _, volume := range vols {
		if string(volume.dir) == string(vd) {
			vol = volume
			if vol.mark == nil {
				vol.mark = allmarkers.getMarker(name, path, vol.dir)
			}
			if vol.mark == nil {
				return nil
			}
			break
		}
	}
	if vol == nil {
		vol = &volume{dir: vd}
		vol.mark = allmarkers.getMarker(name, path, vol.dir)
		if vol.mark == nil {
			return nil
		}
	}
	return vol
}

type container struct {
	name    string
	id      string
	stopped bool
	volumes []*volume
}

func (c *container) trunc() string {
	if len(c.id) > 0 {
		return c.id[:7]
	}
	return ""
}

func (c *container) String() string {
	return "cnt '" + c.name + "' (" + c.trunc() + ")" + fmt.Sprintf("[%v] - %d vol", c.stopped, len(c.volumes))
}

var containers = []*container{}

func mustcmd(acmd string) string {
	out, err := cmd(acmd)
	if err != nil {
		log.Fatal(fmt.Sprintf("out='%s', err='%s'", out, err))
	}
	return string(out)
}

func cmd(cmd string) (string, error) {
	fmt.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	return strings.TrimSpace(string(out)), err
}

func readVolumes() {
	out := mustcmd("sudo ls -a1F /mnt/sda1/var/lib/docker/vfs/dir")
	vollines := strings.Split(out, "\n")
	// fmt.Println(vollines)
	for _, volline := range vollines {
		dir := volline
		if dir == "./" || dir == "../" {
			continue
		}
		if strings.HasSuffix(dir, "@") {
			dir = dir[:len(dir)-1]
			fdir := fmt.Sprintf("/mnt/sda1/var/lib/docker/vfs/dir/%s", dir)
			mp := newmpath(dir)
			if mp == nil {
				fmt.Printf("Invalid marker detected: '%s'\n", dir)
				mustcmd("sudo rm " + fdir)
			} else {
				dirlink, err := cmd("sudo readlink " + fdir)
				fmt.Printf("---\ndir: '%s'\ndlk: '%s'\nerr='%v'\n", dir, dirlink, err)
				if err != nil {
					fmt.Printf("Invalid marker (no readlink) detected: '%s'\n", dir)
					mustcmd("sudo rm " + fdir)
				} else {
					_, err := cmd("sudo ls /mnt/sda1/var/lib/docker/vfs/dir/" + dirlink)
					if err != nil {
						fmt.Printf("Invalid marker (readlink no ls) detected: '%s'\n", dir)
						mustcmd("sudo rm " + fdir)
					} else {
						vd := newvdir(dirlink)
						if vd == "" {
							fmt.Printf("Invalid marker (readlink no vdir) detected: '%s'\n", dir)
							mustcmd("sudo rm " + fdir)
						} else {
							allmarkers = append(allmarkers, &marker{mp, vd})
						}
					}
				}
			}
		} else if strings.HasSuffix(dir, "/") {

			dir = dir[:len(dir)-1]
			fdir := fmt.Sprintf("/mnt/sda1/var/lib/docker/vfs/dir/%s", dir)
			vd := newvdir(dir)
			if vd == "" {
				fmt.Printf("Invalid volume folder detected: '%s'\n", dir)
				mustcmd("sudo rm " + fdir)
			} else {
				allvolumes = append(allvolumes, &volume{dir: vd})
			}
		} else {
			fdir := fmt.Sprintf("/mnt/sda1/var/lib/docker/vfs/dir/%s", dir)
			fmt.Printf("Invalid file detected: '%s'\n", dir)
			mustcmd("sudo rm " + fdir)
		}
	}
	fmt.Printf("volumes: %v\nmarkers: %v\n", allvolumes, allmarkers)
}

func readContainer() {

	out := mustcmd("docker ps -aq --no-trunc")
	contlines := strings.Split(out, "\n")
	// fmt.Println(contlines)
	for _, contline := range contlines {
		id := contline
		res := mustcmd("docker inspect -f '{{ .Name }},{{ range $key, $value := .Volumes }}{{ $key }},{{ $value }}##~#{{ end }}' " + id)
		// fmt.Println("res1: '" + res + "'")
		name := res[1:strings.Index(res, ",")]
		cont := &container{name: name, id: id}
		res = res[strings.Index(res, ",")+1:]
		// fmt.Println("res2: '" + res + "'")
		vols := strings.Split(res, "##~#")
		// fmt.Println(vols)
		for _, vol := range vols {
			elts := strings.Split(vol, ",")
			if len(elts) == 2 {
				// fmt.Printf("elts: '%v'\n", elts)
				path := elts[0]
				vfs := elts[1]
				if strings.Contains(vfs, "/var/lib/docker/vfs/dir/") {
					vd := newvdir(filepath.Base(vfs))
					if vd == "" {
						fmt.Printf("Invalid volume folder detected: '%s'\n", vfs)
						break
					}
					newvol := allvolumes.getVolume(vd, path, name)
					cont.volumes = append(cont.volumes, newvol)
				}
			}
		}
		containers = append(containers, cont)
	}
	fmt.Printf("containers: %v\n", containers)
}

// TODO check if check steps are still necessary
func (v *volume) accept(m *marker) bool {
	// TODO
	return false
}

func (c *container) accept(v *volume) bool {
	// TODO
	return false
}

func checkVolumes() {
	for _, volume := range allvolumes {
		orphan := true
		for _, container := range containers {
			if container.accept(volume) {
				orphan = false
				break
			}
		}
		if orphan {
			fmt.Printf("Orphan detected, volume '%v'\n", volume)
			// TODO rm if necessary or at least mv _xxx
		}
	}
}

// docker run --rm -i -t -v `pwd`:`pwd` -w `pwd` --entrypoint="/bin/bash" go -c 'go build gcl.go'
func main() {
	fmt.Println("### 1/3 readVolumes     ###")
	readVolumes()
	fmt.Println("### 2/3 readContainer   ###")
	readContainer()
	fmt.Println("### 3/3 checkVolumes    ###")
	checkVolumes()
	fmt.Println("---      All done       ---")
	os.Exit(0)
}
<|endoftext|>


<commit_before>// Package deploy builds & deploys function zips.
package deploy

import (
	"fmt"

	"github.com/tj/cobra"

	"github.com/apex/apex/cmd/apex/root"
	"github.com/apex/apex/stats"
	"github.com/apex/apex/utils"
)

// env vars.
var env []string

// env file.
var envFile string

// concurrency of deploys.
var concurrency int

// alias.
var alias string

// zip path.
var zip string

// example output.
const example = `
    Deploy all functions
    $ apex deploy

    Deploy specific functions
    $ apex deploy foo bar

    Deploy canary alias
    $ apex deploy foo --alias canary

    Deploy functions in a different project
    $ apex deploy -C ~/dev/myapp

		Deploy function with existing zip
		$ apex build > out.zip && apex deploy foo --zip out.zip

    Deploy all functions starting with "auth"
    $ apex deploy auth*`

// Command config.
var Command = &cobra.Command{
	Use:     "deploy [<name>...]",
	Short:   "Deploy functions and config",
	Example: example,
	RunE:    run,
}

// Initialize.
func init() {
	root.Register(Command)

	f := Command.Flags()
	f.StringSliceVarP(&env, "set", "s", nil, "Set environment variable")
	f.StringVarP(&envFile, "env-file", "E", "", "Set environment variables from JSON file")
	f.StringVarP(&alias, "alias", "a", "current", "Function alias")
	f.StringVarP(&zip, "zip", "z", "", "Zip path")
	f.IntVarP(&concurrency, "concurrency", "c", 5, "Concurrent deploys")
}

// Run command.
func run(c *cobra.Command, args []string) error {
	stats.Track("Deploy", map[string]interface{}{
		"concurrency": concurrency,
		"has_alias":   alias != "",
		"has_zip":     zip != "",
		"env":         len(env),
		"args":        len(args),
	})

	root.Project.Concurrency = concurrency
	root.Project.Alias = alias
	root.Project.Zip = zip

	if err := root.Project.LoadFunctions(args...); err != nil {
		return err
	}

	for _, fn := range root.Project.Functions {
		stats.Track("Deploy Function", map[string]interface{}{
			"runtime":      fn.Runtime,
			"has_alias":    alias != "",
			"has_zip":      zip != "",
			"has_env_file": envFile != "",
			"env":          len(env),
		})
	}

	if envFile != "" {
		if err := root.Project.LoadEnvFromFile(envFile); err != nil {
			return fmt.Errorf("reading env file %q: %s", envFile, err)
		}
	}

	vars, err := utils.ParseEnv(env)
	if err != nil {
		return err
	}

	for k, v := range vars {
		root.Project.Setenv(k, v)
	}

	return root.Project.DeployAndClean()
}
<commit_msg>fix tabs in deploy example<commit_after>// Package deploy builds & deploys function zips.
package deploy

import (
	"fmt"

	"github.com/tj/cobra"

	"github.com/apex/apex/cmd/apex/root"
	"github.com/apex/apex/stats"
	"github.com/apex/apex/utils"
)

// env vars.
var env []string

// env file.
var envFile string

// concurrency of deploys.
var concurrency int

// alias.
var alias string

// zip path.
var zip string

// example output.
const example = `
    Deploy all functions
    $ apex deploy

    Deploy specific functions
    $ apex deploy foo bar

    Deploy canary alias
    $ apex deploy foo --alias canary

    Deploy functions in a different project
    $ apex deploy -C ~/dev/myapp

    Deploy function with existing zip
    $ apex build > out.zip && apex deploy foo --zip out.zip

    Deploy all functions starting with "auth"
    $ apex deploy auth*`

// Command config.
var Command = &cobra.Command{
	Use:     "deploy [<name>...]",
	Short:   "Deploy functions and config",
	Example: example,
	RunE:    run,
}

// Initialize.
func init() {
	root.Register(Command)

	f := Command.Flags()
	f.StringSliceVarP(&env, "set", "s", nil, "Set environment variable")
	f.StringVarP(&envFile, "env-file", "E", "", "Set environment variables from JSON file")
	f.StringVarP(&alias, "alias", "a", "current", "Function alias")
	f.StringVarP(&zip, "zip", "z", "", "Zip path")
	f.IntVarP(&concurrency, "concurrency", "c", 5, "Concurrent deploys")
}

// Run command.
func run(c *cobra.Command, args []string) error {
	stats.Track("Deploy", map[string]interface{}{
		"concurrency": concurrency,
		"has_alias":   alias != "",
		"has_zip":     zip != "",
		"env":         len(env),
		"args":        len(args),
	})

	root.Project.Concurrency = concurrency
	root.Project.Alias = alias
	root.Project.Zip = zip

	if err := root.Project.LoadFunctions(args...); err != nil {
		return err
	}

	for _, fn := range root.Project.Functions {
		stats.Track("Deploy Function", map[string]interface{}{
			"runtime":      fn.Runtime,
			"has_alias":    alias != "",
			"has_zip":      zip != "",
			"has_env_file": envFile != "",
			"env":          len(env),
		})
	}

	if envFile != "" {
		if err := root.Project.LoadEnvFromFile(envFile); err != nil {
			return fmt.Errorf("reading env file %q: %s", envFile, err)
		}
	}

	vars, err := utils.ParseEnv(env)
	if err != nil {
		return err
	}

	for k, v := range vars {
		root.Project.Setenv(k, v)
	}

	return root.Project.DeployAndClean()
}
<|endoftext|>