<commit_before>package atlas

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
)

var (
	allTypes = map[string]bool{
		"dns":        true,
		"http":       true,
		"ntp":        true,
		"ping":       true,
		"sslcert":    true,
		"traceroute": true,
		"wifi":       true,
	}
)

// -- private

// checkType verify that the type is valid
func checkType(d Definition) (valid bool) {
	_, ok := allTypes[d.Type]
	return ok
}

// checkTypeAs is a shortcut
func checkTypeAs(d Definition, t string) bool {
	valid := checkType(d)
	return valid && d.Type == t
}

// checkAllTypesAs is a generalization of checkTypeAs
func checkAllTypesAs(dl []Definition, t string) (valid bool) {
	valid = true
	for _, d := range dl {
		if d.Type != t {
			valid = false
			break
		}
	}
	return
}

// measurementList is our main answer
type measurementList struct {
	Count    int
	Next     string
	Previous string
	Results  []Measurement
}

// fetch the given resource
func fetchOneMeasurementPage(opts map[string]string) (raw *measurementList, err error) {
	req := prepareRequest("GET", "measurements", opts)

	//log.Printf("req=%s qp=%#v", MeasurementEP, opts)
	r, err := ctx.client.Do(req)
	err = handleAPIResponse(r)
	if err != nil {
		return
	}

	raw = &measurementList{}
	body, _ := ioutil.ReadAll(r.Body)
	defer r.Body.Close()

	err = json.Unmarshal(body, raw)
	//log.Printf("Count=%d raw=%v", raw.Count, r)
	//log.Printf(">> rawlist=%+v r=%+v Next=|%s|", rawlist, r, rawlist.Next)
	return
}

// -- public

// GetMeasurement gets info for a single one
func GetMeasurement(id int) (m *Measurement, err error) {
	opts := make(map[string]string)

	req := prepareRequest("GET", fmt.Sprintf("measurements/%d", id), opts)

	//log.Printf("req: %#v", req)
	r, err := ctx.client.Do(req)
	err = handleAPIResponse(r)
	if err != nil {
		return
	}

	m = &Measurement{}
	body, _ := ioutil.ReadAll(r.Body)
	defer r.Body.Close()

	err = json.Unmarshal(body, m)
	//log.Printf("json: %#v\n", m)
	return
}

// DeleteMeasurement stops (not really deletes) a given measurement
func DeleteMeasurement(id int) (err error) {
	opts := make(map[string]string)

	req := prepareRequest("DELETE", fmt.Sprintf("measurements/%d", id), opts)

	//log.Printf("req: %#v", req)
	r, err := ctx.client.Do(req)
	err = handleAPIResponse(r)
	return
}

// GetMeasurements gets info for a set
func GetMeasurements(opts map[string]string) (m []Measurement, err error) {
	// First call
	rawlist, err := fetchOneMeasurementPage(opts)

	// Empty answer
	if rawlist.Count == 0 {
		return []Measurement{}, nil
	}

	var res []Measurement

	res = append(res, rawlist.Results...)
	if rawlist.Next != "" {
		// We have pagination
		for pn := getPageNum(rawlist.Next); rawlist.Next != ""; pn = getPageNum(rawlist.Next) {
			opts["page"] = pn

			rawlist, err = fetchOneMeasurementPage(opts)
			if err != nil {
				return
			}

			res = append(res, rawlist.Results...)
		}
	}
	m = res
	return
}

// Measurement-related methods

// Start is for starting a given measurement
func (m *Measurement) Start(id int) (err error) {
	return nil
}

// Stop is an alias for delete
func (m *Measurement) Stop() (err error) {
	return DeleteMeasurement(m.ID)
}
<commit_msg>Remove unused code.<commit_after>package atlas

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
)

var (
	allTypes = map[string]bool{
		"dns":        true,
		"http":       true,
		"ntp":        true,
		"ping":       true,
		"sslcert":    true,
		"traceroute": true,
		"wifi":       true,
	}
)

// -- private

// checkType verify that the type is valid
func checkType(d Definition) (valid bool) {
	_, ok := allTypes[d.Type]
	return ok
}

// checkTypeAs is a shortcut
func checkTypeAs(d Definition, t string) bool {
	valid := checkType(d)
	return valid && d.Type == t
}

// checkAllTypesAs is a generalization of checkTypeAs
func checkAllTypesAs(dl []Definition, t string) (valid bool) {
	valid = true
	for _, d := range dl {
		if d.Type != t {
			valid = false
			break
		}
	}
	return
}

// measurementList is our main answer
type measurementList struct {
	Count    int
	Next     string
	Previous string
	Results  []Measurement
}

// fetch the given resource
func fetchOneMeasurementPage(opts map[string]string) (raw *measurementList, err error) {
	req := prepareRequest("GET", "measurements", opts)

	//log.Printf("req=%s qp=%#v", MeasurementEP, opts)
	r, err := ctx.client.Do(req)
	err = handleAPIResponse(r)
	if err != nil {
		return
	}

	raw = &measurementList{}
	body, _ := ioutil.ReadAll(r.Body)
	defer r.Body.Close()

	err = json.Unmarshal(body, raw)
	//log.Printf("Count=%d raw=%v", raw.Count, r)
	//log.Printf(">> rawlist=%+v r=%+v Next=|%s|", rawlist, r, rawlist.Next)
	return
}

// -- public

// GetMeasurement gets info for a single one
func GetMeasurement(id int) (m *Measurement, err error) {
	opts := make(map[string]string)

	req := prepareRequest("GET", fmt.Sprintf("measurements/%d", id), opts)

	//log.Printf("req: %#v", req)
	r, err := ctx.client.Do(req)
	err = handleAPIResponse(r)
	if err != nil {
		return
	}

	m = &Measurement{}
	body, _ := ioutil.ReadAll(r.Body)
	defer r.Body.Close()

	err = json.Unmarshal(body, m)
	//log.Printf("json: %#v\n", m)
	return
}

// DeleteMeasurement stops (not really deletes) a given measurement
func DeleteMeasurement(id int) (err error) {
	opts := make(map[string]string)

	req := prepareRequest("DELETE", fmt.Sprintf("measurements/%d", id), opts)

	//log.Printf("req: %#v", req)
	r, err := ctx.client.Do(req)
	err = handleAPIResponse(r)
	return
}

// GetMeasurements gets info for a set
func GetMeasurements(opts map[string]string) (m []Measurement, err error) {
	// First call
	rawlist, err := fetchOneMeasurementPage(opts)

	// Empty answer
	if rawlist.Count == 0 {
		return []Measurement{}, nil
	}

	var res []Measurement

	res = append(res, rawlist.Results...)
	if rawlist.Next != "" {
		// We have pagination
		for pn := getPageNum(rawlist.Next); rawlist.Next != ""; pn = getPageNum(rawlist.Next) {
			opts["page"] = pn

			rawlist, err = fetchOneMeasurementPage(opts)
			if err != nil {
				return
			}

			res = append(res, rawlist.Results...)
		}
	}
	m = res
	return
}

<|endoftext|>


<commit_before>package net

import (
	"bytes"
	"context"
	"encoding/json"
	"encoding/xml"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strings"
)

// Net 请求结构体
type Net struct {
	client  *http.Client // 可重复使用的client
	baseURL *url.URL     // 请求根地址
}

// SuperAgent 请求参数
type SuperAgent struct {
	net         *Net        // 当前请求包实例
	url         string      // 请求地址
	method      string      // 请求方式
	contentType string      // 请求类型
	body        interface{} // 发送请求的body
}

const (
	contentJSON = "application/json;charset=utf-8"
	contentXML  = "application/xml;charset=utf-8"
	contentText = "text/plain;charset=utf-8"
)

// New 初始化一个请求包对象
func New() *Net {
	return &Net{
		client: http.DefaultClient,
	}
}

// Get 发送 Get 请求
func (n *Net) Get(url string) *SuperAgent {
	return &SuperAgent{net: n, url: url, method: "GET"}
}

// Post 发送 Post 请求
func (n *Net) Post(url string) *SuperAgent {
	return &SuperAgent{net: n, url: url, method: "POST"}
}

// Put 发送 Put 请求
func (n *Net) Put(url string) *SuperAgent {
	return &SuperAgent{net: n, url: url, method: "PUT"}
}

// Delete 发送 Delete 请求
func (n *Net) Delete(url string) *SuperAgent {
	return &SuperAgent{net: n, url: url, method: "DELETE"}
}

// JSON 设置请求数据内容，默认用 Content-Type=application/json; 方式发送json数据
func (s *SuperAgent) JSON(body interface{}) *SuperAgent {
	s.body = body
	s.contentType = contentJSON
	return s
}

// XML 设置请求数据内容，默认用 Content-Type=application/json; 方式发送json数据
func (s *SuperAgent) XML(body interface{}) *SuperAgent {
	s.body = body
	s.contentType = contentXML
	return s
}

// Text 设置请求数据内容，默认用 Content-Type=text/plain; 方式发送string数据
func (s *SuperAgent) Text(body string) *SuperAgent {
	s.body = body
	s.contentType = contentText
	return s
}

// End 开始http请求
func (s *SuperAgent) End(ctx context.Context, v interface{}) (*http.Response, error) {
	if len(s.contentType) > 0 && s.body == nil {
		s.body = ""
	}
	var req *http.Request
	var err error
	buf := new(bytes.Buffer)
	switch s.contentType {
	case contentJSON:
		err = json.NewEncoder(buf).Encode(s.body)
	case contentXML:
		err = xml.NewEncoder(buf).Encode(s.body)
	case contentText:
		_, err = buf.WriteString(s.body.(string))
	}
	if err != nil {
		return nil, err
	}

	// 转换 url
	rel, err := url.Parse(s.url)
	if err != nil {
		return nil, err
	}
	u := s.net.baseURL.ResolveReference(rel)

	req, err = http.NewRequest(s.method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	if len(s.contentType) > 0 {
		req.Header.Set("Content-Type", contentText)
	}

	if ctx != nil {
		req = req.WithContext(ctx)
	}

	// 执行网络请求
	resp, err := s.net.client.Do(req)
	if err != nil {
		// If we got an error, and the context has been canceled, the context's error is probably more useful.
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
		}

		// If the error type is *url.Error, sanitize its URL before returning.
		if e, ok := err.(*url.Error); ok {
			if url, err := url.Parse(e.URL); err == nil {
				e.URL = url.String()
				return nil, e
			}
		}
		return nil, err
	}

	defer func() {
		// Drain up to 512 bytes and close the body to let the Transport reuse the connection
		io.CopyN(ioutil.Discard, resp.Body, 512)
		resp.Body.Close()
	}()

	if v != nil {
		if w, ok := v.(io.Writer); ok {
			io.Copy(w, resp.Body)
		} else {
			body, err := ioutil.ReadAll(resp.Body)
			if !strings.Contains(string(body), "ip_list") {
				log.Printf("url %s body %s", req.URL.Path, string(body))
			}

			// 默认认为 contentType 不为xml的情况下，所有返回都用json解析
			if strings.EqualFold(s.contentType, contentXML) {
				err = xml.Unmarshal(body, v)
			} else {
				err = json.Unmarshal(body, v)
			}

			if err == io.EOF {
				err = nil // ignore EOF errors caused by empty response body
			}
		}
	}

	return resp, err
}
<commit_msg>修复空指针bug<commit_after>package net

import (
	"bytes"
	"context"
	"encoding/json"
	"encoding/xml"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strings"
)

// Net 请求结构体
type Net struct {
	client  *http.Client // 可重复使用的client
	baseURL *url.URL     // 请求根地址
}

// SuperAgent 请求参数
type SuperAgent struct {
	net         *Net        // 当前请求包实例
	url         string      // 请求地址
	method      string      // 请求方式
	contentType string      // 请求类型
	body        interface{} // 发送请求的body
}

const (
	contentJSON = "application/json;charset=utf-8"
	contentXML  = "application/xml;charset=utf-8"
	contentText = "text/plain;charset=utf-8"
)

// New 初始化一个请求包对象
func New() *Net {
	return &Net{
		client: http.DefaultClient,
	}
}

// Get 发送 Get 请求
func (n *Net) Get(url string) *SuperAgent {
	return &SuperAgent{net: n, url: url, method: "GET"}
}

// Post 发送 Post 请求
func (n *Net) Post(url string) *SuperAgent {
	return &SuperAgent{net: n, url: url, method: "POST"}
}

// Put 发送 Put 请求
func (n *Net) Put(url string) *SuperAgent {
	return &SuperAgent{net: n, url: url, method: "PUT"}
}

// Delete 发送 Delete 请求
func (n *Net) Delete(url string) *SuperAgent {
	return &SuperAgent{net: n, url: url, method: "DELETE"}
}

// JSON 设置请求数据内容，默认用 Content-Type=application/json; 方式发送json数据
func (s *SuperAgent) JSON(body interface{}) *SuperAgent {
	s.body = body
	s.contentType = contentJSON
	return s
}

// XML 设置请求数据内容，默认用 Content-Type=application/json; 方式发送json数据
func (s *SuperAgent) XML(body interface{}) *SuperAgent {
	s.body = body
	s.contentType = contentXML
	return s
}

// Text 设置请求数据内容，默认用 Content-Type=text/plain; 方式发送string数据
func (s *SuperAgent) Text(body string) *SuperAgent {
	s.body = body
	s.contentType = contentText
	return s
}

// End 开始http请求
func (s *SuperAgent) End(ctx context.Context, v interface{}) (*http.Response, error) {
	if len(s.contentType) > 0 && s.body == nil {
		s.body = ""
	}
	var req *http.Request
	var err error
	buf := new(bytes.Buffer)
	switch s.contentType {
	case contentJSON:
		err = json.NewEncoder(buf).Encode(s.body)
	case contentXML:
		err = xml.NewEncoder(buf).Encode(s.body)
	case contentText:
		_, err = buf.WriteString(s.body.(string))
	}
	if err != nil {
		return nil, err
	}

	// 转换 url
	rel, err := url.Parse(s.url)
	if err != nil {
		return nil, err
	}
	u := s.net.baseURL.ResolveReference(rel)

	req, err = http.NewRequest(s.method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	if len(s.contentType) > 0 {
		req.Header.Set("Content-Type", contentText)
	}

	if ctx != nil {
		req = req.WithContext(ctx)
	}

	// 执行网络请求
	resp, err := s.net.client.Do(req)
	if err != nil {

		if ctx != nil {
			// If we got an error, and the context has been canceled, the context's error is probably more useful.
			select {
			case <-ctx.Done():
				return nil, ctx.Err()
			default:
			}
		}

		// If the error type is *url.Error, sanitize its URL before returning.
		if e, ok := err.(*url.Error); ok {
			if url, err := url.Parse(e.URL); err == nil {
				e.URL = url.String()
				return nil, e
			}
		}
		return nil, err
	}

	defer func() {
		// Drain up to 512 bytes and close the body to let the Transport reuse the connection
		io.CopyN(ioutil.Discard, resp.Body, 512)
		resp.Body.Close()
	}()

	if v != nil {
		if w, ok := v.(io.Writer); ok {
			io.Copy(w, resp.Body)
		} else {
			body, err := ioutil.ReadAll(resp.Body)
			if !strings.Contains(string(body), "ip_list") {
				log.Printf("url %s body %s", req.URL.Path, string(body))
			}

			// 默认认为 contentType 不为xml的情况下，所有返回都用json解析
			if strings.EqualFold(s.contentType, contentXML) {
				err = xml.Unmarshal(body, v)
			} else {
				err = json.Unmarshal(body, v)
			}

			if err == io.EOF {
				err = nil // ignore EOF errors caused by empty response body
			}
		}
	}

	return resp, err
}
<|endoftext|>


<commit_before>// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package procfs

type (
	// NetUDP represents the contents of /proc/net/udp{,6} file without the header.
	NetUDP []*netIPSocketLine

	// NetUDPSummary provides already computed values like the total queue lengths or
	// the total number of used sockets. In contrast to NetUDP it does not collect
	// the parsed lines into a slice.
	NetUDPSummary NetIPSocketSummary
)

// NetUDP returns the IPv4 kernel/networking statistics for UDP datagrams
// read from /proc/net/udp.
func (fs FS) NetUDP() (NetUDP, error) {
	return newNetUDP(fs.proc.Path("net/udp"))
}

// NetUDP6 returns the IPv6 kernel/networking statistics for UDP datagrams
// read from /proc/net/udp6.
func (fs FS) NetUDP6() (NetUDP, error) {
	return newNetUDP(fs.proc.Path("net/udp6"))
}

// NetUDPSummary returns already computed statistics like the total queue lengths
// for UDP datagrams read from /proc/net/udp.
func (fs FS) NetUDPSummary() (*NetUDPSummary, error) {
	n, err := newNetUDPSummary(fs.proc.Path("net/udp"))
	n1 := NetUDPSummary(*n)
	return &n1, err
}

// NetUDP6Summary returns already computed statistics like the total queue lengths
// for UDP datagrams read from /proc/net/udp6.
func (fs FS) NetUDP6Summary() (*NetUDPSummary, error) {
	n, err := newNetUDPSummary(fs.proc.Path("net/udp6"))
	n1 := NetUDPSummary(*n)
	return &n1, err
}

// newNetUDP creates a new NetUDP{,6} from the contents of the given file.
func newNetUDP(file string) (NetUDP, error) {
	n, err := newNetIPSocket(file)
	n1 := NetUDP(n)
	return n1, err
}

func newNetUDPSummary(file string) (*NetUDPSummary, error) {
	n, err := newNetIPSocketSummary(file)
	if n == nil {
		return nil, err
	}
	n1 := NetUDPSummary(*n)
	return &n1, err
}
<commit_msg>Fix null pointer return in net_udp (#355)<commit_after>// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package procfs

type (
	// NetUDP represents the contents of /proc/net/udp{,6} file without the header.
	NetUDP []*netIPSocketLine

	// NetUDPSummary provides already computed values like the total queue lengths or
	// the total number of used sockets. In contrast to NetUDP it does not collect
	// the parsed lines into a slice.
	NetUDPSummary NetIPSocketSummary
)

// NetUDP returns the IPv4 kernel/networking statistics for UDP datagrams
// read from /proc/net/udp.
func (fs FS) NetUDP() (NetUDP, error) {
	return newNetUDP(fs.proc.Path("net/udp"))
}

// NetUDP6 returns the IPv6 kernel/networking statistics for UDP datagrams
// read from /proc/net/udp6.
func (fs FS) NetUDP6() (NetUDP, error) {
	return newNetUDP(fs.proc.Path("net/udp6"))
}

// NetUDPSummary returns already computed statistics like the total queue lengths
// for UDP datagrams read from /proc/net/udp.
func (fs FS) NetUDPSummary() (*NetUDPSummary, error) {
	return newNetUDPSummary(fs.proc.Path("net/udp"))
}

// NetUDP6Summary returns already computed statistics like the total queue lengths
// for UDP datagrams read from /proc/net/udp6.
func (fs FS) NetUDP6Summary() (*NetUDPSummary, error) {
	return newNetUDPSummary(fs.proc.Path("net/udp6"))
}

// newNetUDP creates a new NetUDP{,6} from the contents of the given file.
func newNetUDP(file string) (NetUDP, error) {
	n, err := newNetIPSocket(file)
	n1 := NetUDP(n)
	return n1, err
}

func newNetUDPSummary(file string) (*NetUDPSummary, error) {
	n, err := newNetIPSocketSummary(file)
	if n == nil {
		return nil, err
	}
	n1 := NetUDPSummary(*n)
	return &n1, err
}
<|endoftext|>


<commit_before>package docker

import (
	"encoding/binary"
	"errors"
	"fmt"
	"log"
	"net"
	"os/exec"
	"strconv"
	"strings"
)

const (
	networkBridgeIface = "lxcbr0"
	portRangeStart     = 49153
	portRangeEnd       = 65535
)

// Calculates the first and last IP addresses in an IPNet
func networkRange(network *net.IPNet) (net.IP, net.IP) {
	netIP := network.IP.To4()
	firstIP := netIP.Mask(network.Mask)
	lastIP := net.IPv4(0, 0, 0, 0).To4()
	for i := 0; i < len(lastIP); i++ {
		lastIP[i] = netIP[i] | ^network.Mask[i]
	}
	return firstIP, lastIP
}

// Converts a 4 bytes IP into a 32 bit integer
func ipToInt(ip net.IP) int32 {
	return int32(binary.BigEndian.Uint32(ip.To4()))
}

// Converts 32 bit integer into a 4 bytes IP address
func intToIp(n int32) net.IP {
	b := make([]byte, 4)
	binary.BigEndian.PutUint32(b, uint32(n))
	return net.IP(b)
}

// Given a netmask, calculates the number of available hosts
func networkSize(mask net.IPMask) int32 {
	m := net.IPv4Mask(0, 0, 0, 0)
	for i := 0; i < net.IPv4len; i++ {
		m[i] = ^mask[i]
	}

	return int32(binary.BigEndian.Uint32(m)) + 1
}

// Wrapper around the iptables command
func iptables(args ...string) error {
	path, err := exec.LookPath("iptables")
	if err != nil {
		return fmt.Errorf("command not found: iptables")
	}
	if err := exec.Command(path, args...).Run(); err != nil {
		return fmt.Errorf("iptables failed: iptables %v", strings.Join(args, " "))
	}
	return nil
}

// Return the IPv4 address of a network interface
func getIfaceAddr(name string) (net.Addr, error) {
	iface, err := net.InterfaceByName(name)
	if err != nil {
		return nil, err
	}
	addrs, err := iface.Addrs()
	if err != nil {
		return nil, err
	}
	var addrs4 []net.Addr
	for _, addr := range addrs {
		ip := (addr.(*net.IPNet)).IP
		if ip4 := ip.To4(); len(ip4) == net.IPv4len {
			addrs4 = append(addrs4, addr)
		}
	}
	switch {
	case len(addrs4) == 0:
		return nil, fmt.Errorf("Interface %v has no IP addresses", name)
	case len(addrs4) > 1:
		fmt.Printf("Interface %v has more than 1 IPv4 address. Defaulting to using %v\n",
			name, (addrs4[0].(*net.IPNet)).IP)
	}
	return addrs4[0], nil
}

// Port mapper takes care of mapping external ports to containers by setting
// up iptables rules.
// It keeps track of all mappings and is able to unmap at will
type PortMapper struct {
	mapping map[int]net.TCPAddr
}

func (mapper *PortMapper) cleanup() error {
	// Ignore errors - This could mean the chains were never set up
	iptables("-t", "nat", "-D", "PREROUTING", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "DOCKER")
	iptables("-t", "nat", "-D", "OUTPUT", "-j", "DOCKER")
	iptables("-t", "nat", "-F", "DOCKER")
	iptables("-t", "nat", "-X", "DOCKER")
	mapper.mapping = make(map[int]net.TCPAddr)
	return nil
}

func (mapper *PortMapper) setup() error {
	if err := iptables("-t", "nat", "-N", "DOCKER"); err != nil {
		return fmt.Errorf("Failed to create DOCKER chain: %s", err)
	}
	if err := iptables("-t", "nat", "-A", "PREROUTING", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "DOCKER"); err != nil {
		return fmt.Errorf("Failed to inject docker in PREROUTING chain: %s", err)
	}
	if err := iptables("-t", "nat", "-A", "OUTPUT", "-j", "DOCKER"); err != nil {
		return fmt.Errorf("Failed to inject docker in OUTPUT chain: %s", err)
	}
	return nil
}

func (mapper *PortMapper) iptablesForward(rule string, port int, dest net.TCPAddr) error {
	return iptables("-t", "nat", rule, "DOCKER", "-p", "tcp", "--dport", strconv.Itoa(port),
		"-j", "DNAT", "--to-destination", net.JoinHostPort(dest.IP.String(), strconv.Itoa(dest.Port)))
}

func (mapper *PortMapper) Map(port int, dest net.TCPAddr) error {
	if err := mapper.iptablesForward("-A", port, dest); err != nil {
		return err
	}
	mapper.mapping[port] = dest
	return nil
}

func (mapper *PortMapper) Unmap(port int) error {
	dest, ok := mapper.mapping[port]
	if !ok {
		return errors.New("Port is not mapped")
	}
	if err := mapper.iptablesForward("-D", port, dest); err != nil {
		return err
	}
	delete(mapper.mapping, port)
	return nil
}

func newPortMapper() (*PortMapper, error) {
	mapper := &PortMapper{}
	if err := mapper.cleanup(); err != nil {
		return nil, err
	}
	if err := mapper.setup(); err != nil {
		return nil, err
	}
	return mapper, nil
}

// Port allocator: Atomatically allocate and release networking ports
type PortAllocator struct {
	ports chan (int)
}

func (alloc *PortAllocator) populate(start, end int) {
	alloc.ports = make(chan int, end-start)
	for port := start; port < end; port++ {
		alloc.ports <- port
	}
}

func (alloc *PortAllocator) Acquire() (int, error) {
	select {
	case port := <-alloc.ports:
		return port, nil
	default:
		return -1, errors.New("No more ports available")
	}
	return -1, nil
}

func (alloc *PortAllocator) Release(port int) error {
	select {
	case alloc.ports <- port:
		return nil
	default:
		return errors.New("Too many ports have been released")
	}
	return nil
}

func newPortAllocator(start, end int) (*PortAllocator, error) {
	allocator := &PortAllocator{}
	allocator.populate(start, end)
	return allocator, nil
}

// IP allocator: Atomatically allocate and release networking ports
type IPAllocator struct {
	network       *net.IPNet
	queueAlloc    chan allocatedIP
	queueReleased chan net.IP
	inUse         map[int32]struct{}
}

type allocatedIP struct {
	ip  net.IP
	err error
}

func (alloc *IPAllocator) run() {
	firstIP, _ := networkRange(alloc.network)
	ipNum := ipToInt(firstIP)
	ownIP := ipToInt(alloc.network.IP)
	size := networkSize(alloc.network.Mask)

	pos := int32(1)
	max := size - 2 // -1 for the broadcast address, -1 for the gateway address
	for {
		var (
			newNum int32
			inUse  bool
		)

		// Find first unused IP, give up after one whole round
		for attempt := int32(0); attempt < max; attempt++ {
			newNum = ipNum + pos

			pos = pos%max + 1

			// The network's IP is never okay to use
			if newNum == ownIP {
				continue
			}

			if _, inUse = alloc.inUse[newNum]; !inUse {
				// We found an unused IP
				break
			}
		}

		ip := allocatedIP{ip: intToIp(newNum)}
		if inUse {
			ip.err = errors.New("No unallocated IP available")
		}

		select {
		case alloc.queueAlloc <- ip:
			alloc.inUse[newNum] = struct{}{}
		case released := <-alloc.queueReleased:
			r := ipToInt(released)
			delete(alloc.inUse, r)

			if inUse {
				// If we couldn't allocate a new IP, the released one
				// will be the only free one now, so instantly use it
				// next time
				pos = r - ipNum
			} else {
				// Use same IP as last time
				if pos == 1 {
					pos = max
				} else {
					pos--
				}
			}
		}
	}
}

func (alloc *IPAllocator) Acquire() (net.IP, error) {
	ip := <-alloc.queueAlloc
	return ip.ip, ip.err
}

func (alloc *IPAllocator) Release(ip net.IP) {
	alloc.queueReleased <- ip
}

func newIPAllocator(network *net.IPNet) *IPAllocator {
	alloc := &IPAllocator{
		network:       network,
		queueAlloc:    make(chan allocatedIP),
		queueReleased: make(chan net.IP),
		inUse:         make(map[int32]struct{}),
	}

	go alloc.run()

	return alloc
}

// Network interface represents the networking stack of a container
type NetworkInterface struct {
	IPNet   net.IPNet
	Gateway net.IP

	manager  *NetworkManager
	extPorts []int
}

// Allocate an external TCP port and map it to the interface
func (iface *NetworkInterface) AllocatePort(port int) (int, error) {
	extPort, err := iface.manager.portAllocator.Acquire()
	if err != nil {
		return -1, err
	}
	if err := iface.manager.portMapper.Map(extPort, net.TCPAddr{IP: iface.IPNet.IP, Port: port}); err != nil {
		iface.manager.portAllocator.Release(extPort)
		return -1, err
	}
	iface.extPorts = append(iface.extPorts, extPort)
	return extPort, nil
}

// Release: Network cleanup - release all resources
func (iface *NetworkInterface) Release() {
	for _, port := range iface.extPorts {
		if err := iface.manager.portMapper.Unmap(port); err != nil {
			log.Printf("Unable to unmap port %v: %v", port, err)
		}
		if err := iface.manager.portAllocator.Release(port); err != nil {
			log.Printf("Unable to release port %v: %v", port, err)
		}

	}

	iface.manager.ipAllocator.Release(iface.IPNet.IP)
}

// Network Manager manages a set of network interfaces
// Only *one* manager per host machine should be used
type NetworkManager struct {
	bridgeIface   string
	bridgeNetwork *net.IPNet

	ipAllocator   *IPAllocator
	portAllocator *PortAllocator
	portMapper    *PortMapper
}

// Allocate a network interface
func (manager *NetworkManager) Allocate() (*NetworkInterface, error) {
	ip, err := manager.ipAllocator.Acquire()
	if err != nil {
		return nil, err
	}
	iface := &NetworkInterface{
		IPNet:   net.IPNet{IP: ip, Mask: manager.bridgeNetwork.Mask},
		Gateway: manager.bridgeNetwork.IP,
		manager: manager,
	}
	return iface, nil
}

func newNetworkManager(bridgeIface string) (*NetworkManager, error) {
	addr, err := getIfaceAddr(bridgeIface)
	if err != nil {
		return nil, err
	}
	network := addr.(*net.IPNet)

	ipAllocator := newIPAllocator(network)

	portAllocator, err := newPortAllocator(portRangeStart, portRangeEnd)
	if err != nil {
		return nil, err
	}

	portMapper, err := newPortMapper()
	if err != nil {
		return nil, err
	}

	manager := &NetworkManager{
		bridgeIface:   bridgeIface,
		bridgeNetwork: network,
		ipAllocator:   ipAllocator,
		portAllocator: portAllocator,
		portMapper:    portMapper,
	}
	return manager, nil
}
<commit_msg>stop looping remote:port from host to containers<commit_after>package docker

import (
	"encoding/binary"
	"errors"
	"fmt"
	"log"
	"net"
	"os/exec"
	"strconv"
	"strings"
)

const (
	networkBridgeIface = "lxcbr0"
	portRangeStart     = 49153
	portRangeEnd       = 65535
)

// Calculates the first and last IP addresses in an IPNet
func networkRange(network *net.IPNet) (net.IP, net.IP) {
	netIP := network.IP.To4()
	firstIP := netIP.Mask(network.Mask)
	lastIP := net.IPv4(0, 0, 0, 0).To4()
	for i := 0; i < len(lastIP); i++ {
		lastIP[i] = netIP[i] | ^network.Mask[i]
	}
	return firstIP, lastIP
}

// Converts a 4 bytes IP into a 32 bit integer
func ipToInt(ip net.IP) int32 {
	return int32(binary.BigEndian.Uint32(ip.To4()))
}

// Converts 32 bit integer into a 4 bytes IP address
func intToIp(n int32) net.IP {
	b := make([]byte, 4)
	binary.BigEndian.PutUint32(b, uint32(n))
	return net.IP(b)
}

// Given a netmask, calculates the number of available hosts
func networkSize(mask net.IPMask) int32 {
	m := net.IPv4Mask(0, 0, 0, 0)
	for i := 0; i < net.IPv4len; i++ {
		m[i] = ^mask[i]
	}

	return int32(binary.BigEndian.Uint32(m)) + 1
}

// Wrapper around the iptables command
func iptables(args ...string) error {
	path, err := exec.LookPath("iptables")
	if err != nil {
		return fmt.Errorf("command not found: iptables")
	}
	if err := exec.Command(path, args...).Run(); err != nil {
		return fmt.Errorf("iptables failed: iptables %v", strings.Join(args, " "))
	}
	return nil
}

// Return the IPv4 address of a network interface
func getIfaceAddr(name string) (net.Addr, error) {
	iface, err := net.InterfaceByName(name)
	if err != nil {
		return nil, err
	}
	addrs, err := iface.Addrs()
	if err != nil {
		return nil, err
	}
	var addrs4 []net.Addr
	for _, addr := range addrs {
		ip := (addr.(*net.IPNet)).IP
		if ip4 := ip.To4(); len(ip4) == net.IPv4len {
			addrs4 = append(addrs4, addr)
		}
	}
	switch {
	case len(addrs4) == 0:
		return nil, fmt.Errorf("Interface %v has no IP addresses", name)
	case len(addrs4) > 1:
		fmt.Printf("Interface %v has more than 1 IPv4 address. Defaulting to using %v\n",
			name, (addrs4[0].(*net.IPNet)).IP)
	}
	return addrs4[0], nil
}

// Port mapper takes care of mapping external ports to containers by setting
// up iptables rules.
// It keeps track of all mappings and is able to unmap at will
type PortMapper struct {
	mapping map[int]net.TCPAddr
}

func (mapper *PortMapper) cleanup() error {
	// Ignore errors - This could mean the chains were never set up
	iptables("-t", "nat", "-D", "PREROUTING", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "DOCKER")
	iptables("-t", "nat", "-D", "OUTPUT", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "DOCKER")
	iptables("-t", "nat", "-F", "DOCKER")
	iptables("-t", "nat", "-X", "DOCKER")
	mapper.mapping = make(map[int]net.TCPAddr)
	return nil
}

func (mapper *PortMapper) setup() error {
	if err := iptables("-t", "nat", "-N", "DOCKER"); err != nil {
		return fmt.Errorf("Failed to create DOCKER chain: %s", err)
	}
	if err := iptables("-t", "nat", "-A", "PREROUTING", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "DOCKER"); err != nil {
		return fmt.Errorf("Failed to inject docker in PREROUTING chain: %s", err)
	}
	if err := iptables("-t", "nat", "-A", "OUTPUT", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "DOCKER"); err != nil {
		return fmt.Errorf("Failed to inject docker in OUTPUT chain: %s", err)
	}
	return nil
}

func (mapper *PortMapper) iptablesForward(rule string, port int, dest net.TCPAddr) error {
	return iptables("-t", "nat", rule, "DOCKER", "-p", "tcp", "--dport", strconv.Itoa(port),
		"-j", "DNAT", "--to-destination", net.JoinHostPort(dest.IP.String(), strconv.Itoa(dest.Port)))
}

func (mapper *PortMapper) Map(port int, dest net.TCPAddr) error {
	if err := mapper.iptablesForward("-A", port, dest); err != nil {
		return err
	}
	mapper.mapping[port] = dest
	return nil
}

func (mapper *PortMapper) Unmap(port int) error {
	dest, ok := mapper.mapping[port]
	if !ok {
		return errors.New("Port is not mapped")
	}
	if err := mapper.iptablesForward("-D", port, dest); err != nil {
		return err
	}
	delete(mapper.mapping, port)
	return nil
}

func newPortMapper() (*PortMapper, error) {
	mapper := &PortMapper{}
	if err := mapper.cleanup(); err != nil {
		return nil, err
	}
	if err := mapper.setup(); err != nil {
		return nil, err
	}
	return mapper, nil
}

// Port allocator: Atomatically allocate and release networking ports
type PortAllocator struct {
	ports chan (int)
}

func (alloc *PortAllocator) populate(start, end int) {
	alloc.ports = make(chan int, end-start)
	for port := start; port < end; port++ {
		alloc.ports <- port
	}
}

func (alloc *PortAllocator) Acquire() (int, error) {
	select {
	case port := <-alloc.ports:
		return port, nil
	default:
		return -1, errors.New("No more ports available")
	}
	return -1, nil
}

func (alloc *PortAllocator) Release(port int) error {
	select {
	case alloc.ports <- port:
		return nil
	default:
		return errors.New("Too many ports have been released")
	}
	return nil
}

func newPortAllocator(start, end int) (*PortAllocator, error) {
	allocator := &PortAllocator{}
	allocator.populate(start, end)
	return allocator, nil
}

// IP allocator: Atomatically allocate and release networking ports
type IPAllocator struct {
	network       *net.IPNet
	queueAlloc    chan allocatedIP
	queueReleased chan net.IP
	inUse         map[int32]struct{}
}

type allocatedIP struct {
	ip  net.IP
	err error
}

func (alloc *IPAllocator) run() {
	firstIP, _ := networkRange(alloc.network)
	ipNum := ipToInt(firstIP)
	ownIP := ipToInt(alloc.network.IP)
	size := networkSize(alloc.network.Mask)

	pos := int32(1)
	max := size - 2 // -1 for the broadcast address, -1 for the gateway address
	for {
		var (
			newNum int32
			inUse  bool
		)

		// Find first unused IP, give up after one whole round
		for attempt := int32(0); attempt < max; attempt++ {
			newNum = ipNum + pos

			pos = pos%max + 1

			// The network's IP is never okay to use
			if newNum == ownIP {
				continue
			}

			if _, inUse = alloc.inUse[newNum]; !inUse {
				// We found an unused IP
				break
			}
		}

		ip := allocatedIP{ip: intToIp(newNum)}
		if inUse {
			ip.err = errors.New("No unallocated IP available")
		}

		select {
		case alloc.queueAlloc <- ip:
			alloc.inUse[newNum] = struct{}{}
		case released := <-alloc.queueReleased:
			r := ipToInt(released)
			delete(alloc.inUse, r)

			if inUse {
				// If we couldn't allocate a new IP, the released one
				// will be the only free one now, so instantly use it
				// next time
				pos = r - ipNum
			} else {
				// Use same IP as last time
				if pos == 1 {
					pos = max
				} else {
					pos--
				}
			}
		}
	}
}

func (alloc *IPAllocator) Acquire() (net.IP, error) {
	ip := <-alloc.queueAlloc
	return ip.ip, ip.err
}

func (alloc *IPAllocator) Release(ip net.IP) {
	alloc.queueReleased <- ip
}

func newIPAllocator(network *net.IPNet) *IPAllocator {
	alloc := &IPAllocator{
		network:       network,
		queueAlloc:    make(chan allocatedIP),
		queueReleased: make(chan net.IP),
		inUse:         make(map[int32]struct{}),
	}

	go alloc.run()

	return alloc
}

// Network interface represents the networking stack of a container
type NetworkInterface struct {
	IPNet   net.IPNet
	Gateway net.IP

	manager  *NetworkManager
	extPorts []int
}

// Allocate an external TCP port and map it to the interface
func (iface *NetworkInterface) AllocatePort(port int) (int, error) {
	extPort, err := iface.manager.portAllocator.Acquire()
	if err != nil {
		return -1, err
	}
	if err := iface.manager.portMapper.Map(extPort, net.TCPAddr{IP: iface.IPNet.IP, Port: port}); err != nil {
		iface.manager.portAllocator.Release(extPort)
		return -1, err
	}
	iface.extPorts = append(iface.extPorts, extPort)
	return extPort, nil
}

// Release: Network cleanup - release all resources
func (iface *NetworkInterface) Release() {
	for _, port := range iface.extPorts {
		if err := iface.manager.portMapper.Unmap(port); err != nil {
			log.Printf("Unable to unmap port %v: %v", port, err)
		}
		if err := iface.manager.portAllocator.Release(port); err != nil {
			log.Printf("Unable to release port %v: %v", port, err)
		}

	}

	iface.manager.ipAllocator.Release(iface.IPNet.IP)
}

// Network Manager manages a set of network interfaces
// Only *one* manager per host machine should be used
type NetworkManager struct {
	bridgeIface   string
	bridgeNetwork *net.IPNet

	ipAllocator   *IPAllocator
	portAllocator *PortAllocator
	portMapper    *PortMapper
}

// Allocate a network interface
func (manager *NetworkManager) Allocate() (*NetworkInterface, error) {
	ip, err := manager.ipAllocator.Acquire()
	if err != nil {
		return nil, err
	}
	iface := &NetworkInterface{
		IPNet:   net.IPNet{IP: ip, Mask: manager.bridgeNetwork.Mask},
		Gateway: manager.bridgeNetwork.IP,
		manager: manager,
	}
	return iface, nil
}

func newNetworkManager(bridgeIface string) (*NetworkManager, error) {
	addr, err := getIfaceAddr(bridgeIface)
	if err != nil {
		return nil, err
	}
	network := addr.(*net.IPNet)

	ipAllocator := newIPAllocator(network)

	portAllocator, err := newPortAllocator(portRangeStart, portRangeEnd)
	if err != nil {
		return nil, err
	}

	portMapper, err := newPortMapper()
	if err != nil {
		return nil, err
	}

	manager := &NetworkManager{
		bridgeIface:   bridgeIface,
		bridgeNetwork: network,
		ipAllocator:   ipAllocator,
		portAllocator: portAllocator,
		portMapper:    portMapper,
	}
	return manager, nil
}
<|endoftext|>


<commit_before>package frame_test

import (
	"bytes"
	"crypto/md5"
	"io"
	"testing"

	"gopkg.in/mewkiz/flac.v1"
)

var golden = []struct {
	name string
}{
	{name: "../testdata/love.flac"},   // i=0
	{name: "../testdata/19875.flac"},  // i=1
	{name: "../testdata/44127.flac"},  // i=2
	{name: "../testdata/59996.flac"},  // i=3
	{name: "../testdata/80574.flac"},  // i=4
	{name: "../testdata/172960.flac"}, // i=5
	{name: "../testdata/189983.flac"}, // i=6
	{name: "../testdata/191885.flac"}, // i=7
	{name: "../testdata/212768.flac"}, // i=8
	{name: "../testdata/220014.flac"}, // i=9
	{name: "../testdata/243749.flac"}, // i=10
	{name: "../testdata/256529.flac"}, // i=11
	{name: "../testdata/257344.flac"}, // i=12
}

func TestFrameHash(t *testing.T) {
	for i, g := range golden {
		stream, err := flac.Open(g.name)
		if err != nil {
			t.Fatal(err)
		}
		defer stream.Close()

		md5sum := md5.New()
		for frameNum := 0; ; frameNum++ {
			frame, err := stream.ParseNext()
			if err != nil {
				if err == io.EOF {
					break
				}
				t.Errorf("i=%d, frameNum=%d: error while parsing frame; %v", i, frameNum, err)
				continue
			}
			frame.Hash(md5sum)
		}
		want := stream.Info.MD5sum[:]
		got := md5sum.Sum(nil)
		// Verify the decoded audio samples by comparing the MD5 checksum that is
		// stored in StreamInfo with the computed one.
		if !bytes.Equal(got, want) {
			t.Errorf("i=%d: MD5 checksum mismatch for decoded audio samples; expected %32x, got %32x", i, want, got)
		}
	}
}

func BenchmarkFrameParse(b *testing.B) {
	// The file 151185.flac is a 119.5 MB public domain FLAC file used to
	// benchmark the flac library. Because of its size, it has not been included
	// in the repository, but is available for download at
	//
	//    http://freesound.org/people/jarfil/sounds/151185/
	stream, err := flac.Open("../testdata/benchmark/151185.flac")
	if err != nil {
		b.Fatal(err)
	}
	for i := 0; i < b.N; i++ {
		_, err := stream.ParseNext()
		if err != nil {
			if err == io.EOF {
				break
			}
			b.Fatal(err)
		}
	}
}
<commit_msg>frame: Fix Frame.Parse benchmark and add Frame.Hash benchmark.<commit_after>package frame_test

import (
	"bytes"
	"crypto/md5"
	"io"
	"testing"

	"gopkg.in/mewkiz/flac.v1"
)

var golden = []struct {
	name string
}{
	{name: "../testdata/love.flac"},   // i=0
	{name: "../testdata/19875.flac"},  // i=1
	{name: "../testdata/44127.flac"},  // i=2
	{name: "../testdata/59996.flac"},  // i=3
	{name: "../testdata/80574.flac"},  // i=4
	{name: "../testdata/172960.flac"}, // i=5
	{name: "../testdata/189983.flac"}, // i=6
	{name: "../testdata/191885.flac"}, // i=7
	{name: "../testdata/212768.flac"}, // i=8
	{name: "../testdata/220014.flac"}, // i=9
	{name: "../testdata/243749.flac"}, // i=10
	{name: "../testdata/256529.flac"}, // i=11
	{name: "../testdata/257344.flac"}, // i=12
}

func TestFrameHash(t *testing.T) {
	for i, g := range golden {
		stream, err := flac.Open(g.name)
		if err != nil {
			t.Fatal(err)
		}
		defer stream.Close()

		md5sum := md5.New()
		for frameNum := 0; ; frameNum++ {
			frame, err := stream.ParseNext()
			if err != nil {
				if err == io.EOF {
					break
				}
				t.Errorf("i=%d, frameNum=%d: error while parsing frame; %v", i, frameNum, err)
				continue
			}
			frame.Hash(md5sum)
		}
		want := stream.Info.MD5sum[:]
		got := md5sum.Sum(nil)
		// Verify the decoded audio samples by comparing the MD5 checksum that is
		// stored in StreamInfo with the computed one.
		if !bytes.Equal(got, want) {
			t.Errorf("i=%d: MD5 checksum mismatch for decoded audio samples; expected %32x, got %32x", i, want, got)
		}
	}
}

func BenchmarkFrameParse(b *testing.B) {
	// The file 151185.flac is a 119.5 MB public domain FLAC file used to
	// benchmark the flac library. Because of its size, it has not been included
	// in the repository, but is available for download at
	//
	//    http://freesound.org/people/jarfil/sounds/151185/
	for i := 0; i < b.N; i++ {
		stream, err := flac.Open("../testdata/benchmark/151185.flac")
		if err != nil {
			b.Fatal(err)
		}
		for {
			_, err := stream.ParseNext()
			if err != nil {
				if err == io.EOF {
					break
				}
				stream.Close()
				b.Fatal(err)
			}
		}
		stream.Close()
	}
}

func BenchmarkFrameHash(b *testing.B) {
	// The file 151185.flac is a 119.5 MB public domain FLAC file used to
	// benchmark the flac library. Because of its size, it has not been included
	// in the repository, but is available for download at
	//
	//    http://freesound.org/people/jarfil/sounds/151185/
	for i := 0; i < b.N; i++ {
		stream, err := flac.Open("../testdata/benchmark/151185.flac")
		if err != nil {
			b.Fatal(err)
		}
		md5sum := md5.New()
		for {
			frame, err := stream.ParseNext()
			if err != nil {
				if err == io.EOF {
					break
				}
				stream.Close()
				b.Fatal(err)
			}
			frame.Hash(md5sum)
		}
		stream.Close()
		want := stream.Info.MD5sum[:]
		got := md5sum.Sum(nil)
		// Verify the decoded audio samples by comparing the MD5 checksum that is
		// stored in StreamInfo with the computed one.
		if !bytes.Equal(got, want) {
			b.Fatalf("MD5 checksum mismatch for decoded audio samples; expected %32x, got %32x", want, got)
		}
	}
}
<|endoftext|>


<commit_before>package sarama

import "testing"

var (
	emptyMessage = []byte{
		167, 236, 104, 3, // CRC
		0x00,                   // magic version byte
		0x00,                   // attribute flags
		0xFF, 0xFF, 0xFF, 0xFF, // key
		0xFF, 0xFF, 0xFF, 0xFF} // value

	emptyGzipMessage = []byte{
		97, 79, 149, 90, //CRC
		0x00,                   // magic version byte
		0x01,                   // attribute flags
		0xFF, 0xFF, 0xFF, 0xFF, // key
		// value
		0x00, 0x00, 0x00, 0x17,
		0x1f, 0x8b,
		0x08,
		0, 0, 9, 110, 136, 0, 255, 1, 0, 0, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0}

	emptyBulkSnappyMessage = []byte{
		180, 47, 53, 209, //CRC
		0x00,                   // magic version byte
		0x02,                   // attribute flags
		0xFF, 0xFF, 0xFF, 0xFF, // key
		0, 0, 0, 42,
		130, 83, 78, 65, 80, 80, 89, 0, // SNAPPY magic
		0, 0, 0, 1, // min version
		0, 0, 0, 1, // default version
		0, 0, 0, 22, 52, 0, 0, 25, 1, 16, 14, 227, 138, 104, 118, 25, 15, 13, 1, 8, 1, 0, 0, 62, 26, 0}

	emptyBulkGzipMessage = []byte{
		139, 160, 63, 141, //CRC
		0x00,                   // magic version byte
		0x01,                   // attribute flags
		0xFF, 0xFF, 0xFF, 0xFF, // key
		0x00, 0x00, 0x00, 0x27, // len
		0x1f, 0x8b, // Gzip Magic
		0x08, // deflate compressed
		0, 0, 0, 0, 0, 0, 0, 99, 96, 128, 3, 190, 202, 112, 143, 7, 12, 12, 255, 129, 0, 33, 200, 192, 136, 41, 3, 0, 199, 226, 155, 70, 52, 0, 0, 0}
)

func TestMessageEncoding(t *testing.T) {
	message := Message{}
	testEncodable(t, "empty", &message, emptyMessage)

	message.Value = []byte{}
	message.Codec = CompressionGZIP
	testEncodable(t, "empty gzip", &message, emptyGzipMessage)
}

func TestMessageDecoding(t *testing.T) {
	message := Message{}
	testDecodable(t, "empty", &message, emptyMessage)
	if message.Codec != CompressionNone {
		t.Error("Decoding produced compression codec where there was none.")
	}
	if message.Key != nil {
		t.Error("Decoding produced key where there was none.")
	}
	if message.Value != nil {
		t.Error("Decoding produced value where there was none.")
	}
	if message.Set != nil {
		t.Error("Decoding produced set where there was none.")
	}

	testDecodable(t, "empty gzip", &message, emptyGzipMessage)
	if message.Codec != CompressionGZIP {
		t.Error("Decoding produced incorrect compression codec (was gzip).")
	}
	if message.Key != nil {
		t.Error("Decoding produced key where there was none.")
	}
	if message.Value == nil || len(message.Value) != 0 {
		t.Error("Decoding produced nil or content-ful value where there was an empty array.")
	}
}

func TestMessageDecodingBulkSnappy(t *testing.T) {
	message := Message{}
	testDecodable(t, "bulk snappy", &message, emptyBulkSnappyMessage)
	if message.Codec != CompressionSnappy {
		t.Errorf("Decoding produced codec %d, but expected %d.", message.Codec, CompressionSnappy)
	}
	if message.Key != nil {
		t.Errorf("Decoding produced key %+v, but none was expected.", message.Key)
	}
	if message.Set == nil {
		t.Error("Decoding produced no set, but one was expected.")
	} else if len(message.Set.Messages) != 2 {
		t.Errorf("Decoding produced a set with %d messages, but 2 were expected.", len(message.Set.Messages))
	}
}

func TestMessageDecodingBulkGzip(t *testing.T) {
	message := Message{}
	testDecodable(t, "bulk gzip", &message, emptyBulkGzipMessage)
	if message.Codec != CompressionGZIP {
		t.Errorf("Decoding produced codec %d, but expected %d.", message.Codec, CompressionGZIP)
	}
	if message.Key != nil {
		t.Errorf("Decoding produced key %+v, but none was expected.", message.Key)
	}
	if message.Set == nil {
		t.Error("Decoding produced no set, but one was expected.")
	} else if len(message.Set.Messages) != 2 {
		t.Errorf("Decoding produced a set with %d messages, but 2 were expected.", len(message.Set.Messages))
	}
}
<commit_msg>Add lz4 decode test<commit_after>package sarama

import "testing"

var (
	emptyMessage = []byte{
		167, 236, 104, 3, // CRC
		0x00,                   // magic version byte
		0x00,                   // attribute flags
		0xFF, 0xFF, 0xFF, 0xFF, // key
		0xFF, 0xFF, 0xFF, 0xFF} // value

	emptyGzipMessage = []byte{
		97, 79, 149, 90, //CRC
		0x00,                   // magic version byte
		0x01,                   // attribute flags
		0xFF, 0xFF, 0xFF, 0xFF, // key
		// value
		0x00, 0x00, 0x00, 0x17,
		0x1f, 0x8b,
		0x08,
		0, 0, 9, 110, 136, 0, 255, 1, 0, 0, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0}

	emptyBulkSnappyMessage = []byte{
		180, 47, 53, 209, //CRC
		0x00,                   // magic version byte
		0x02,                   // attribute flags
		0xFF, 0xFF, 0xFF, 0xFF, // key
		0, 0, 0, 42,
		130, 83, 78, 65, 80, 80, 89, 0, // SNAPPY magic
		0, 0, 0, 1, // min version
		0, 0, 0, 1, // default version
		0, 0, 0, 22, 52, 0, 0, 25, 1, 16, 14, 227, 138, 104, 118, 25, 15, 13, 1, 8, 1, 0, 0, 62, 26, 0}

	emptyBulkGzipMessage = []byte{
		139, 160, 63, 141, //CRC
		0x00,                   // magic version byte
		0x01,                   // attribute flags
		0xFF, 0xFF, 0xFF, 0xFF, // key
		0x00, 0x00, 0x00, 0x27, // len
		0x1f, 0x8b, // Gzip Magic
		0x08, // deflate compressed
		0, 0, 0, 0, 0, 0, 0, 99, 96, 128, 3, 190, 202, 112, 143, 7, 12, 12, 255, 129, 0, 33, 200, 192, 136, 41, 3, 0, 199, 226, 155, 70, 52, 0, 0, 0}

	emptyBulkLZ4Message = []byte{
		246, 12, 188, 129, // CRC
		0x01,                                  // Version
		0x03,                                  // attribute flags (LZ4)
		255, 255, 249, 209, 212, 181, 73, 201, // timestamp
		0xFF, 0xFF, 0xFF, 0xFF, // key
		0x00, 0x00, 0x00, 0x47, // len
		0x04, 0x22, 0x4D, 0x18, // magic number lz4
		100, // lz4 flags 01100100
		// version: 01, block indep: 1, block checksum: 0, content size: 0, content checksum: 1, reserved: 00
		112, 185, 52, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 121, 87, 72, 224, 0, 0, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 14, 121, 87, 72, 224, 0, 0, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0,
		71, 129, 23, 111, // LZ4 checksum
	}
)

func TestMessageEncoding(t *testing.T) {
	message := Message{}
	testEncodable(t, "empty", &message, emptyMessage)

	message.Value = []byte{}
	message.Codec = CompressionGZIP
	testEncodable(t, "empty gzip", &message, emptyGzipMessage)
}

func TestMessageDecoding(t *testing.T) {
	message := Message{}
	testDecodable(t, "empty", &message, emptyMessage)
	if message.Codec != CompressionNone {
		t.Error("Decoding produced compression codec where there was none.")
	}
	if message.Key != nil {
		t.Error("Decoding produced key where there was none.")
	}
	if message.Value != nil {
		t.Error("Decoding produced value where there was none.")
	}
	if message.Set != nil {
		t.Error("Decoding produced set where there was none.")
	}

	testDecodable(t, "empty gzip", &message, emptyGzipMessage)
	if message.Codec != CompressionGZIP {
		t.Error("Decoding produced incorrect compression codec (was gzip).")
	}
	if message.Key != nil {
		t.Error("Decoding produced key where there was none.")
	}
	if message.Value == nil || len(message.Value) != 0 {
		t.Error("Decoding produced nil or content-ful value where there was an empty array.")
	}
}

func TestMessageDecodingBulkSnappy(t *testing.T) {
	message := Message{}
	testDecodable(t, "bulk snappy", &message, emptyBulkSnappyMessage)
	if message.Codec != CompressionSnappy {
		t.Errorf("Decoding produced codec %d, but expected %d.", message.Codec, CompressionSnappy)
	}
	if message.Key != nil {
		t.Errorf("Decoding produced key %+v, but none was expected.", message.Key)
	}
	if message.Set == nil {
		t.Error("Decoding produced no set, but one was expected.")
	} else if len(message.Set.Messages) != 2 {
		t.Errorf("Decoding produced a set with %d messages, but 2 were expected.", len(message.Set.Messages))
	}
}

func TestMessageDecodingBulkGzip(t *testing.T) {
	message := Message{}
	testDecodable(t, "bulk gzip", &message, emptyBulkGzipMessage)
	if message.Codec != CompressionGZIP {
		t.Errorf("Decoding produced codec %d, but expected %d.", message.Codec, CompressionGZIP)
	}
	if message.Key != nil {
		t.Errorf("Decoding produced key %+v, but none was expected.", message.Key)
	}
	if message.Set == nil {
		t.Error("Decoding produced no set, but one was expected.")
	} else if len(message.Set.Messages) != 2 {
		t.Errorf("Decoding produced a set with %d messages, but 2 were expected.", len(message.Set.Messages))
	}
}

func TestMessageDecodingBulkLZ4(t *testing.T) {
	message := Message{}
	testDecodable(t, "bulk lz4", &message, emptyBulkLZ4Message)
	if message.Codec != CompressionLZ4 {
		t.Errorf("Decoding produced codec %d, but expected %d.", message.Codec, CompressionLZ4)
	}
	if message.Key != nil {
		t.Errorf("Decoding produced key %+v, but none was expected.", message.Key)
	}
	if message.Set == nil {
		t.Error("Decoding produced no set, but one was expected.")
	} else if len(message.Set.Messages) != 2 {
		t.Errorf("Decoding produced a set with %d messages, but 2 were expected.", len(message.Set.Messages))
	}
}
<|endoftext|>


<commit_before>// Package new generates micro service templates
package new

import (
	"fmt"
	"go/build"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"text/template"
	"time"

	"github.com/micro/cli/v2"
	tmpl "github.com/micro/micro/v2/internal/template"
	"github.com/micro/micro/v2/internal/usage"
	"github.com/xlab/treeprint"
)

type config struct {
	// foo
	Alias string
	// micro new example -type
	Command string
	// go.micro
	Namespace string
	// api, srv, web, fnc
	Type string
	// go.micro.srv.foo
	FQDN string
	// github.com/micro/foo
	Dir string
	// $GOPATH/src/github.com/micro/foo
	GoDir string
	// $GOPATH
	GoPath string
	// UseGoPath
	UseGoPath bool
	// Files
	Files []file
	// Comments
	Comments []string
	// Plugins registry=etcd:broker=nats
	Plugins []string
}

type file struct {
	Path string
	Tmpl string
}

func write(c config, file, tmpl string) error {
	fn := template.FuncMap{
		"title": strings.Title,
	}

	f, err := os.Create(file)
	if err != nil {
		return err
	}
	defer f.Close()

	t, err := template.New("f").Funcs(fn).Parse(tmpl)
	if err != nil {
		return err
	}

	return t.Execute(f, c)
}

func create(c config) error {
	// check if dir exists
	if _, err := os.Stat(c.GoDir); !os.IsNotExist(err) {
		return fmt.Errorf("%s already exists", c.GoDir)
	}

	// create usage report
	u := usage.New("new")
	// a single request/service
	u.Metrics.Count["requests"] = uint64(1)
	u.Metrics.Count["services"] = uint64(1)
	// send report
	go usage.Report(u)

	// just wait
	<-time.After(time.Millisecond * 250)

	fmt.Printf("Creating service %s in %s\n\n", c.FQDN, c.GoDir)

	t := treeprint.New()

	nodes := map[string]treeprint.Tree{}
	nodes[c.GoDir] = t

	// write the files
	for _, file := range c.Files {
		f := filepath.Join(c.GoDir, file.Path)
		dir := filepath.Dir(f)

		b, ok := nodes[dir]
		if !ok {
			d, _ := filepath.Rel(c.GoDir, dir)
			b = t.AddBranch(d)
			nodes[dir] = b
		}

		if _, err := os.Stat(dir); os.IsNotExist(err) {
			if err := os.MkdirAll(dir, 0755); err != nil {
				return err
			}
		}

		p := filepath.Base(f)

		b.AddNode(p)
		if err := write(c, f, file.Tmpl); err != nil {
			return err
		}
	}

	// print tree
	fmt.Println(t.String())

	for _, comment := range c.Comments {
		fmt.Println(comment)
	}

	// just wait
	<-time.After(time.Millisecond * 250)

	return nil
}

func run(ctx *cli.Context) {
	namespace := ctx.String("namespace")
	alias := ctx.String("alias")
	fqdn := ctx.String("fqdn")
	atype := ctx.String("type")
	dir := ctx.Args().First()
	useGoPath := ctx.Bool("gopath")
	useGoModule := os.Getenv("GO111MODULE")
	var plugins []string

	if len(dir) == 0 {
		fmt.Println("specify service name")
		return
	}

	if len(namespace) == 0 {
		fmt.Println("namespace not defined")
		return
	}

	if len(atype) == 0 {
		fmt.Println("type not defined")
		return
	}

	// set the command
	command := fmt.Sprintf("micro new %s", dir)
	if len(namespace) > 0 {
		command += " --namespace=" + namespace
	}
	if len(alias) > 0 {
		command += " --alias=" + alias
	}
	if len(fqdn) > 0 {
		command += " --fqdn=" + fqdn
	}
	if len(atype) > 0 {
		command += " --type=" + atype
	}
	if plugins := ctx.StringSlice("plugin"); len(plugins) > 0 {
		command += " --plugin=" + strings.Join(plugins, ":")
	}

	// check if the path is absolute, we don't want this
	// we want to a relative path so we can install in GOPATH
	if path.IsAbs(dir) {
		fmt.Println("require relative path as service will be installed in GOPATH")
		return
	}

	var goPath string
	var goDir string

	// only set gopath if told to use it
	if useGoPath {
		goPath = build.Default.GOPATH

		// don't know GOPATH, runaway....
		if len(goPath) == 0 {
			fmt.Println("unknown GOPATH")
			return
		}

		// attempt to split path if not windows
		if runtime.GOOS == "windows" {
			goPath = strings.Split(goPath, ";")[0]
		} else {
			goPath = strings.Split(goPath, ":")[0]
		}
		goDir = filepath.Join(goPath, "src", path.Clean(dir))
	} else {
		goDir = path.Clean(dir)
	}

	if len(alias) == 0 {
		// set as last part
		alias = filepath.Base(dir)
		// strip hyphens
		parts := strings.Split(alias, "-")
		alias = parts[0]
	}

	if len(fqdn) == 0 {
		fqdn = strings.Join([]string{namespace, atype, alias}, ".")
	}

	for _, plugin := range ctx.StringSlice("plugin") {
		// registry=etcd:broker=nats
		for _, p := range strings.Split(plugin, ":") {
			// registry=etcd
			parts := strings.Split(p, "=")
			if len(parts) < 2 {
				continue
			}
			plugins = append(plugins, path.Join(parts...))
		}
	}

	var c config

	switch atype {
	case "fnc":
		// create srv config
		c = config{
			Alias:     alias,
			Command:   command,
			Namespace: namespace,
			Type:      atype,
			FQDN:      fqdn,
			Dir:       dir,
			GoDir:     goDir,
			GoPath:    goPath,
			UseGoPath: useGoPath,
			Plugins:   plugins,
			Files: []file{
				{"main.go", tmpl.MainFNC},
				{"generate.go", tmpl.GenerateFile},
				{"plugin.go", tmpl.Plugin},
				{"handler/" + alias + ".go", tmpl.HandlerFNC},
				{"subscriber/" + alias + ".go", tmpl.SubscriberFNC},
				{"proto/" + alias + "/" + alias + ".proto", tmpl.ProtoFNC},
				{"Dockerfile", tmpl.DockerFNC},
				{"Makefile", tmpl.Makefile},
				{"README.md", tmpl.ReadmeFNC},
			},
			Comments: []string{
				"\ndownload protobuf for micro:\n",
				"brew install protobuf",
				"go get -u github.com/golang/protobuf/{proto,protoc-gen-go}",
				"go get -u github.com/micro/protoc-gen-micro",
				"\ncompile the proto file " + alias + ".proto:\n",
				"cd " + goDir,
				"protoc --proto_path=.:$GOPATH/src --go_out=. --micro_out=. proto/" + alias + "/" + alias + ".proto\n",
			},
		}
	case "srv":
		// create srv config
		c = config{
			Alias:     alias,
			Command:   command,
			Namespace: namespace,
			Type:      atype,
			FQDN:      fqdn,
			Dir:       dir,
			GoDir:     goDir,
			GoPath:    goPath,
			UseGoPath: useGoPath,
			Plugins:   plugins,
			Files: []file{
				{"main.go", tmpl.MainSRV},
				{"generate.go", tmpl.GenerateFile},
				{"plugin.go", tmpl.Plugin},
				{"handler/" + alias + ".go", tmpl.HandlerSRV},
				{"subscriber/" + alias + ".go", tmpl.SubscriberSRV},
				{"proto/" + alias + "/" + alias + ".proto", tmpl.ProtoSRV},
				{"Dockerfile", tmpl.DockerSRV},
				{"Makefile", tmpl.Makefile},
				{"README.md", tmpl.Readme},
			},
			Comments: []string{
				"\ndownload protobuf for micro:\n",
				"brew install protobuf",
				"go get -u github.com/golang/protobuf/{proto,protoc-gen-go}",
				"go get -u github.com/micro/protoc-gen-micro",
				"\ncompile the proto file " + alias + ".proto:\n",
				"cd " + goDir,
				"protoc --proto_path=.:$GOPATH/src --go_out=. --micro_out=. proto/" + alias + "/" + alias + ".proto\n",
			},
		}
	case "api":
		// create api config
		c = config{
			Alias:     alias,
			Command:   command,
			Namespace: namespace,
			Type:      atype,
			FQDN:      fqdn,
			Dir:       dir,
			GoDir:     goDir,
			GoPath:    goPath,
			UseGoPath: useGoPath,
			Plugins:   plugins,
			Files: []file{
				{"main.go", tmpl.MainAPI},
				{"generate.go", tmpl.GenerateFile},
				{"plugin.go", tmpl.Plugin},
				{"client/" + alias + ".go", tmpl.WrapperAPI},
				{"handler/" + alias + ".go", tmpl.HandlerAPI},
				{"proto/" + alias + "/" + alias + ".proto", tmpl.ProtoAPI},
				{"Makefile", tmpl.Makefile},
				{"Dockerfile", tmpl.DockerSRV},
				{"README.md", tmpl.Readme},
			},
			Comments: []string{
				"\ndownload protobuf for micro:\n",
				"brew install protobuf",
				"go get -u github.com/golang/protobuf/{proto,protoc-gen-go}",
				"go get -u github.com/micro/protoc-gen-micro",
				"\ncompile the proto file " + alias + ".proto:\n",
				"cd " + goDir,
				"protoc --proto_path=.:$GOPATH/src --go_out=. --micro_out=. proto/" + alias + "/" + alias + ".proto\n",
			},
		}
	case "web":
		// create srv config
		c = config{
			Alias:     alias,
			Command:   command,
			Namespace: namespace,
			Type:      atype,
			FQDN:      fqdn,
			Dir:       dir,
			GoDir:     goDir,
			GoPath:    goPath,
			UseGoPath: useGoPath,
			Plugins:   plugins,
			Files: []file{
				{"main.go", tmpl.MainWEB},
				{"plugin.go", tmpl.Plugin},
				{"handler/handler.go", tmpl.HandlerWEB},
				{"html/index.html", tmpl.HTMLWEB},
				{"Dockerfile", tmpl.DockerWEB},
				{"Makefile", tmpl.Makefile},
				{"README.md", tmpl.Readme},
			},
			Comments: []string{},
		}
	default:
		fmt.Println("Unknown type", atype)
		return
	}

	// set gomodule
	if useGoModule != "off" {
		c.Files = append(c.Files, file{"go.mod", tmpl.Module})
	}

	if err := create(c); err != nil {
		fmt.Println(err)
		return
	}
}

func Commands() []*cli.Command {
	return []*cli.Command{
		{
			Name:  "new",
			Usage: "Create a service template",
			Flags: []cli.Flag{
				&cli.StringFlag{
					Name:  "namespace",
					Usage: "Namespace for the service e.g com.example",
					Value: "go.micro",
				},
				&cli.StringFlag{
					Name:  "type",
					Usage: "Type of service e.g api, fnc, srv, web",
					Value: "srv",
				},
				&cli.StringFlag{
					Name:  "fqdn",
					Usage: "FQDN of service e.g com.example.srv.service (defaults to namespace.type.alias)",
				},
				&cli.StringFlag{
					Name:  "alias",
					Usage: "Alias is the short name used as part of combined name if specified",
				},
				&cli.StringSliceFlag{
					Name:  "plugin",
					Usage: "Specify plugins e.g --plugin=registry=etcd:broker=nats or use flag multiple times",
				},
				&cli.BoolFlag{
					Name:  "gopath",
					Usage: "Create the service in the gopath.",
				},
			},
			Action: func(c *cli.Context) error {
				run(c)
				return nil
			},
		},
	}
}
<commit_msg>Updat micro new command to use protoc-gen-micro v2 (#515)<commit_after>// Package new generates micro service templates
package new

import (
	"fmt"
	"go/build"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"text/template"
	"time"

	"github.com/micro/cli/v2"
	tmpl "github.com/micro/micro/v2/internal/template"
	"github.com/micro/micro/v2/internal/usage"
	"github.com/xlab/treeprint"
)

type config struct {
	// foo
	Alias string
	// micro new example -type
	Command string
	// go.micro
	Namespace string
	// api, srv, web, fnc
	Type string
	// go.micro.srv.foo
	FQDN string
	// github.com/micro/foo
	Dir string
	// $GOPATH/src/github.com/micro/foo
	GoDir string
	// $GOPATH
	GoPath string
	// UseGoPath
	UseGoPath bool
	// Files
	Files []file
	// Comments
	Comments []string
	// Plugins registry=etcd:broker=nats
	Plugins []string
}

type file struct {
	Path string
	Tmpl string
}

func write(c config, file, tmpl string) error {
	fn := template.FuncMap{
		"title": strings.Title,
	}

	f, err := os.Create(file)
	if err != nil {
		return err
	}
	defer f.Close()

	t, err := template.New("f").Funcs(fn).Parse(tmpl)
	if err != nil {
		return err
	}

	return t.Execute(f, c)
}

func create(c config) error {
	// check if dir exists
	if _, err := os.Stat(c.GoDir); !os.IsNotExist(err) {
		return fmt.Errorf("%s already exists", c.GoDir)
	}

	// create usage report
	u := usage.New("new")
	// a single request/service
	u.Metrics.Count["requests"] = uint64(1)
	u.Metrics.Count["services"] = uint64(1)
	// send report
	go usage.Report(u)

	// just wait
	<-time.After(time.Millisecond * 250)

	fmt.Printf("Creating service %s in %s\n\n", c.FQDN, c.GoDir)

	t := treeprint.New()

	nodes := map[string]treeprint.Tree{}
	nodes[c.GoDir] = t

	// write the files
	for _, file := range c.Files {
		f := filepath.Join(c.GoDir, file.Path)
		dir := filepath.Dir(f)

		b, ok := nodes[dir]
		if !ok {
			d, _ := filepath.Rel(c.GoDir, dir)
			b = t.AddBranch(d)
			nodes[dir] = b
		}

		if _, err := os.Stat(dir); os.IsNotExist(err) {
			if err := os.MkdirAll(dir, 0755); err != nil {
				return err
			}
		}

		p := filepath.Base(f)

		b.AddNode(p)
		if err := write(c, f, file.Tmpl); err != nil {
			return err
		}
	}

	// print tree
	fmt.Println(t.String())

	for _, comment := range c.Comments {
		fmt.Println(comment)
	}

	// just wait
	<-time.After(time.Millisecond * 250)

	return nil
}

func run(ctx *cli.Context) {
	namespace := ctx.String("namespace")
	alias := ctx.String("alias")
	fqdn := ctx.String("fqdn")
	atype := ctx.String("type")
	dir := ctx.Args().First()
	useGoPath := ctx.Bool("gopath")
	useGoModule := os.Getenv("GO111MODULE")
	var plugins []string

	if len(dir) == 0 {
		fmt.Println("specify service name")
		return
	}

	if len(namespace) == 0 {
		fmt.Println("namespace not defined")
		return
	}

	if len(atype) == 0 {
		fmt.Println("type not defined")
		return
	}

	// set the command
	command := fmt.Sprintf("micro new %s", dir)
	if len(namespace) > 0 {
		command += " --namespace=" + namespace
	}
	if len(alias) > 0 {
		command += " --alias=" + alias
	}
	if len(fqdn) > 0 {
		command += " --fqdn=" + fqdn
	}
	if len(atype) > 0 {
		command += " --type=" + atype
	}
	if plugins := ctx.StringSlice("plugin"); len(plugins) > 0 {
		command += " --plugin=" + strings.Join(plugins, ":")
	}

	// check if the path is absolute, we don't want this
	// we want to a relative path so we can install in GOPATH
	if path.IsAbs(dir) {
		fmt.Println("require relative path as service will be installed in GOPATH")
		return
	}

	var goPath string
	var goDir string

	// only set gopath if told to use it
	if useGoPath {
		goPath = build.Default.GOPATH

		// don't know GOPATH, runaway....
		if len(goPath) == 0 {
			fmt.Println("unknown GOPATH")
			return
		}

		// attempt to split path if not windows
		if runtime.GOOS == "windows" {
			goPath = strings.Split(goPath, ";")[0]
		} else {
			goPath = strings.Split(goPath, ":")[0]
		}
		goDir = filepath.Join(goPath, "src", path.Clean(dir))
	} else {
		goDir = path.Clean(dir)
	}

	if len(alias) == 0 {
		// set as last part
		alias = filepath.Base(dir)
		// strip hyphens
		parts := strings.Split(alias, "-")
		alias = parts[0]
	}

	if len(fqdn) == 0 {
		fqdn = strings.Join([]string{namespace, atype, alias}, ".")
	}

	for _, plugin := range ctx.StringSlice("plugin") {
		// registry=etcd:broker=nats
		for _, p := range strings.Split(plugin, ":") {
			// registry=etcd
			parts := strings.Split(p, "=")
			if len(parts) < 2 {
				continue
			}
			plugins = append(plugins, path.Join(parts...))
		}
	}

	var c config

	switch atype {
	case "fnc":
		// create srv config
		c = config{
			Alias:     alias,
			Command:   command,
			Namespace: namespace,
			Type:      atype,
			FQDN:      fqdn,
			Dir:       dir,
			GoDir:     goDir,
			GoPath:    goPath,
			UseGoPath: useGoPath,
			Plugins:   plugins,
			Files: []file{
				{"main.go", tmpl.MainFNC},
				{"generate.go", tmpl.GenerateFile},
				{"plugin.go", tmpl.Plugin},
				{"handler/" + alias + ".go", tmpl.HandlerFNC},
				{"subscriber/" + alias + ".go", tmpl.SubscriberFNC},
				{"proto/" + alias + "/" + alias + ".proto", tmpl.ProtoFNC},
				{"Dockerfile", tmpl.DockerFNC},
				{"Makefile", tmpl.Makefile},
				{"README.md", tmpl.ReadmeFNC},
			},
			Comments: []string{
				"\ndownload protobuf for micro:\n",
				"brew install protobuf",
				"go get -u github.com/golang/protobuf/{proto,protoc-gen-go}",
				"go get -u github.com/micro/protoc-gen-micro/v2",
				"\ncompile the proto file " + alias + ".proto:\n",
				"cd " + goDir,
				"protoc --proto_path=.:$GOPATH/src --go_out=. --micro_out=. proto/" + alias + "/" + alias + ".proto\n",
			},
		}
	case "srv":
		// create srv config
		c = config{
			Alias:     alias,
			Command:   command,
			Namespace: namespace,
			Type:      atype,
			FQDN:      fqdn,
			Dir:       dir,
			GoDir:     goDir,
			GoPath:    goPath,
			UseGoPath: useGoPath,
			Plugins:   plugins,
			Files: []file{
				{"main.go", tmpl.MainSRV},
				{"generate.go", tmpl.GenerateFile},
				{"plugin.go", tmpl.Plugin},
				{"handler/" + alias + ".go", tmpl.HandlerSRV},
				{"subscriber/" + alias + ".go", tmpl.SubscriberSRV},
				{"proto/" + alias + "/" + alias + ".proto", tmpl.ProtoSRV},
				{"Dockerfile", tmpl.DockerSRV},
				{"Makefile", tmpl.Makefile},
				{"README.md", tmpl.Readme},
			},
			Comments: []string{
				"\ndownload protobuf for micro:\n",
				"brew install protobuf",
				"go get -u github.com/golang/protobuf/{proto,protoc-gen-go}",
				"go get -u github.com/micro/protoc-gen-micro/v2",
				"\ncompile the proto file " + alias + ".proto:\n",
				"cd " + goDir,
				"protoc --proto_path=.:$GOPATH/src --go_out=. --micro_out=. proto/" + alias + "/" + alias + ".proto\n",
			},
		}
	case "api":
		// create api config
		c = config{
			Alias:     alias,
			Command:   command,
			Namespace: namespace,
			Type:      atype,
			FQDN:      fqdn,
			Dir:       dir,
			GoDir:     goDir,
			GoPath:    goPath,
			UseGoPath: useGoPath,
			Plugins:   plugins,
			Files: []file{
				{"main.go", tmpl.MainAPI},
				{"generate.go", tmpl.GenerateFile},
				{"plugin.go", tmpl.Plugin},
				{"client/" + alias + ".go", tmpl.WrapperAPI},
				{"handler/" + alias + ".go", tmpl.HandlerAPI},
				{"proto/" + alias + "/" + alias + ".proto", tmpl.ProtoAPI},
				{"Makefile", tmpl.Makefile},
				{"Dockerfile", tmpl.DockerSRV},
				{"README.md", tmpl.Readme},
			},
			Comments: []string{
				"\ndownload protobuf for micro:\n",
				"brew install protobuf",
				"go get -u github.com/golang/protobuf/{proto,protoc-gen-go}",
				"go get -u github.com/micro/protoc-gen-micro/v2",
				"\ncompile the proto file " + alias + ".proto:\n",
				"cd " + goDir,
				"protoc --proto_path=.:$GOPATH/src --go_out=. --micro_out=. proto/" + alias + "/" + alias + ".proto\n",
			},
		}
	case "web":
		// create srv config
		c = config{
			Alias:     alias,
			Command:   command,
			Namespace: namespace,
			Type:      atype,
			FQDN:      fqdn,
			Dir:       dir,
			GoDir:     goDir,
			GoPath:    goPath,
			UseGoPath: useGoPath,
			Plugins:   plugins,
			Files: []file{
				{"main.go", tmpl.MainWEB},
				{"plugin.go", tmpl.Plugin},
				{"handler/handler.go", tmpl.HandlerWEB},
				{"html/index.html", tmpl.HTMLWEB},
				{"Dockerfile", tmpl.DockerWEB},
				{"Makefile", tmpl.Makefile},
				{"README.md", tmpl.Readme},
			},
			Comments: []string{},
		}
	default:
		fmt.Println("Unknown type", atype)
		return
	}

	// set gomodule
	if useGoModule != "off" {
		c.Files = append(c.Files, file{"go.mod", tmpl.Module})
	}

	if err := create(c); err != nil {
		fmt.Println(err)
		return
	}
}

func Commands() []*cli.Command {
	return []*cli.Command{
		{
			Name:  "new",
			Usage: "Create a service template",
			Flags: []cli.Flag{
				&cli.StringFlag{
					Name:  "namespace",
					Usage: "Namespace for the service e.g com.example",
					Value: "go.micro",
				},
				&cli.StringFlag{
					Name:  "type",
					Usage: "Type of service e.g api, fnc, srv, web",
					Value: "srv",
				},
				&cli.StringFlag{
					Name:  "fqdn",
					Usage: "FQDN of service e.g com.example.srv.service (defaults to namespace.type.alias)",
				},
				&cli.StringFlag{
					Name:  "alias",
					Usage: "Alias is the short name used as part of combined name if specified",
				},
				&cli.StringSliceFlag{
					Name:  "plugin",
					Usage: "Specify plugins e.g --plugin=registry=etcd:broker=nats or use flag multiple times",
				},
				&cli.BoolFlag{
					Name:  "gopath",
					Usage: "Create the service in the gopath.",
				},
			},
			Action: func(c *cli.Context) error {
				run(c)
				return nil
			},
		},
	}
}
<|endoftext|>


<commit_before>package buildpackrunner

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"time"

	"code.cloudfoundry.org/buildpackapplifecycle"
	"code.cloudfoundry.org/bytefmt"
	"github.com/cloudfoundry-incubator/candiedyaml"
)

const DOWNLOAD_TIMEOUT = 10 * time.Minute

type Runner interface {
	Run(config *buildpackapplifecycle.LifecycleBuilderConfig) error
}

type runner struct {
	config *buildpackapplifecycle.LifecycleBuilderConfig
}

type descriptiveError struct {
	message string
	err     error
}

func (e descriptiveError) Error() string {
	if e.err == nil {
		return e.message
	}
	return fmt.Sprintf("%s: %s", e.message, e.err.Error())
}

func newDescriptiveError(err error, message string, args ...interface{}) error {
	if len(args) == 0 {
		return descriptiveError{message: message, err: err}
	}
	return descriptiveError{message: fmt.Sprintf(message, args...), err: err}
}

type Release struct {
	DefaultProcessTypes buildpackapplifecycle.ProcessTypes `yaml:"default_process_types"`
}

func New() Runner {
	return &runner{}
}

func (runner *runner) Run(config *buildpackapplifecycle.LifecycleBuilderConfig) error {
	runner.config = config

	//set up the world
	err := runner.makeDirectories()
	if err != nil {
		return newDescriptiveError(err, "Failed to set up filesystem when generating droplet")
	}

	err = runner.downloadBuildpacks()
	if err != nil {
		return err
	}

	//detect, compile, release
	detectedBuildpack, detectedBuildpackDir, detectOutput, ok := runner.detect()
	if !ok {
		return newDescriptiveError(nil, buildpackapplifecycle.DetectFailMsg)
	}

	err = runner.compile(detectedBuildpackDir)
	if err != nil {
		return newDescriptiveError(nil, buildpackapplifecycle.CompileFailMsg)
	}

	startCommands, err := runner.readProcfile()
	if err != nil {
		return newDescriptiveError(err, "Failed to read command from Procfile")
	}

	releaseInfo, err := runner.release(detectedBuildpackDir, startCommands)
	if err != nil {
		return newDescriptiveError(err, buildpackapplifecycle.ReleaseFailMsg)
	}

	if releaseInfo.DefaultProcessTypes["web"] == "" {
		printError("No start command specified by buildpack or via Procfile.")
		printError("App will not start unless a command is provided at runtime.")
	}

	tarPath, err := exec.LookPath("tar")
	if err != nil {
		return err
	}

	//prepare the final droplet directory
	contentsDir, err := ioutil.TempDir("", "contents")
	if err != nil {
		return newDescriptiveError(err, "Failed to create droplet contents dir")
	}

	//generate staging_info.yml and result json file
	infoFilePath := path.Join(contentsDir, "staging_info.yml")
	err = runner.saveInfo(infoFilePath, detectedBuildpack, detectOutput, releaseInfo)
	if err != nil {
		return newDescriptiveError(err, "Failed to encode generated metadata")
	}

	appDir := path.Join(contentsDir, "app")
	err = runner.copyApp(runner.config.BuildDir(), appDir)
	if err != nil {
		return newDescriptiveError(err, "Failed to copy compiled droplet")
	}

	err = os.MkdirAll(path.Join(contentsDir, "tmp"), 0755)
	if err != nil {
		return newDescriptiveError(err, "Failed to set up droplet filesystem")
	}

	err = os.MkdirAll(path.Join(contentsDir, "logs"), 0755)
	if err != nil {
		return newDescriptiveError(err, "Failed to set up droplet filesystem")
	}

	err = exec.Command(tarPath, "-czf", runner.config.OutputDroplet(), "-C", contentsDir, ".").Run()
	if err != nil {
		return newDescriptiveError(err, "Failed to compress droplet")
	}

	//prepare the build artifacts cache output directory
	err = os.MkdirAll(filepath.Dir(runner.config.OutputBuildArtifactsCache()), 0755)
	if err != nil {
		return newDescriptiveError(err, "Failed to create output build artifacts cache dir")
	}

	err = exec.Command(tarPath, "-czf", runner.config.OutputBuildArtifactsCache(), "-C", runner.config.BuildArtifactsCacheDir(), ".").Run()
	if err != nil {
		return newDescriptiveError(err, "Failed to compress build artifacts")
	}

	return nil
}

func (runner *runner) makeDirectories() error {
	if err := os.MkdirAll(filepath.Dir(runner.config.OutputDroplet()), 0755); err != nil {
		return err
	}

	if err := os.MkdirAll(filepath.Dir(runner.config.OutputMetadata()), 0755); err != nil {
		return err
	}

	if err := os.MkdirAll(runner.config.BuildArtifactsCacheDir(), 0755); err != nil {
		return err
	}

	return nil
}

func (runner *runner) downloadBuildpacks() error {
	// Do we have a custom buildpack?
	for _, buildpackName := range runner.config.BuildpackOrder() {
		buildpackUrl, err := url.Parse(buildpackName)
		if err != nil {
			return fmt.Errorf("Invalid buildpack url (%s): %s", buildpackName, err.Error())
		}
		if !buildpackUrl.IsAbs() {
			continue
		}

		destination := runner.config.BuildpackPath(buildpackName)

		if IsZipFile(buildpackUrl.Path) {
			var size uint64
			zipDownloader := NewZipDownloader(runner.config.SkipCertVerify())
			size, err = zipDownloader.DownloadAndExtract(buildpackUrl, destination)
			if err == nil {
				fmt.Printf("Downloaded buildpack `%s` (%s)", buildpackUrl.String(), bytefmt.ByteSize(size))
			}
		} else {
			err = GitClone(*buildpackUrl, destination)
		}
		if err != nil {
			return err
		}
	}

	return nil
}

func (runner *runner) buildpackPath(buildpack string) (string, error) {
	buildpackPath := runner.config.BuildpackPath(buildpack)

	if runner.pathHasBinDirectory(buildpackPath) {
		return buildpackPath, nil
	}

	files, err := ioutil.ReadDir(buildpackPath)
	if err != nil {
		return "", newDescriptiveError(nil, "Failed to read buildpack directory '%s' for buildpack '%s'", buildpackPath, buildpack)
	}

	if len(files) == 1 {
		nestedPath := path.Join(buildpackPath, files[0].Name())

		if runner.pathHasBinDirectory(nestedPath) {
			return nestedPath, nil
		}
	}

	return "", newDescriptiveError(nil, "malformed buildpack does not contain a /bin dir: %s", buildpack)
}

func (runner *runner) pathHasBinDirectory(pathToTest string) bool {
	_, err := os.Stat(path.Join(pathToTest, "bin"))
	return err == nil
}

// returns buildpack name,  buildpack path, buildpack detect output, ok
func (runner *runner) detect() (string, string, string, bool) {
	for _, buildpack := range runner.config.BuildpackOrder() {

		buildpackPath, err := runner.buildpackPath(buildpack)
		if err != nil {
			printError(err.Error())
			continue
		}

		if runner.config.SkipDetect() {
			return buildpack, buildpackPath, "", true
		}

		output := new(bytes.Buffer)
		err = runner.run(exec.Command(path.Join(buildpackPath, "bin", "detect"), runner.config.BuildDir()), output)

		if err == nil {
			return buildpack, buildpackPath, strings.TrimRight(output.String(), "\n"), true
		}
	}

	return "", "", "", false
}

func (runner *runner) readProcfile() (map[string]string, error) {
	processes := map[string]string{}

	procFile, err := os.Open(filepath.Join(runner.config.BuildDir(), "Procfile"))
	if err != nil {
		if os.IsNotExist(err) {
			// Procfiles are optional
			return processes, nil
		}

		return processes, err
	}

	defer procFile.Close()

	err = candiedyaml.NewDecoder(procFile).Decode(&processes)
	if err != nil {
		// clobber candiedyaml's super low-level error
		return processes, errors.New("invalid YAML")
	}

	return processes, nil
}

func (runner *runner) compile(buildpackDir string) error {
	return runner.run(exec.Command(path.Join(buildpackDir, "bin", "compile"), runner.config.BuildDir(), runner.config.BuildArtifactsCacheDir()), os.Stdout)
}

func (runner *runner) release(buildpackDir string, startCommands map[string]string) (Release, error) {
	output := new(bytes.Buffer)

	err := runner.run(exec.Command(path.Join(buildpackDir, "bin", "release"), runner.config.BuildDir()), output)
	if err != nil {
		return Release{}, err
	}

	decoder := candiedyaml.NewDecoder(output)

	parsedRelease := Release{}

	err = decoder.Decode(&parsedRelease)
	if err != nil {
		return Release{}, newDescriptiveError(err, "buildpack's release output invalid")
	}

	if len(startCommands) > 0 {
		parsedRelease.DefaultProcessTypes = startCommands
	}

	return parsedRelease, nil
}

func (runner *runner) saveInfo(infoFilePath, buildpack, detectOutput string, releaseInfo Release) error {
	deaInfoFile, err := os.Create(infoFilePath)
	if err != nil {
		return err
	}
	defer deaInfoFile.Close()

	// JSON ⊂ YAML
	err = json.NewEncoder(deaInfoFile).Encode(DeaStagingInfo{
		DetectedBuildpack: detectOutput,
		StartCommand:      releaseInfo.DefaultProcessTypes["web"],
	})
	if err != nil {
		return err
	}

	resultFile, err := os.Create(runner.config.OutputMetadata())
	if err != nil {
		return err
	}
	defer resultFile.Close()

	err = json.NewEncoder(resultFile).Encode(buildpackapplifecycle.NewStagingResult(
		releaseInfo.DefaultProcessTypes,
		buildpackapplifecycle.LifecycleMetadata{
			BuildpackKey:      buildpack,
			DetectedBuildpack: detectOutput,
		},
	))
	if err != nil {
		return err
	}

	return nil
}

func (runner *runner) copyApp(buildDir, stageDir string) error {
	return runner.run(exec.Command("cp", "-a", buildDir, stageDir), os.Stdout)
}

func (runner *runner) run(cmd *exec.Cmd, output io.Writer) error {
	cmd.Stdout = output
	cmd.Stderr = os.Stderr

	return cmd.Run()
}

func printError(message string) {
	println(message)
}
<commit_msg>Add newline to "downloaded buildpack" output<commit_after>package buildpackrunner

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"time"

	"code.cloudfoundry.org/buildpackapplifecycle"
	"code.cloudfoundry.org/bytefmt"
	"github.com/cloudfoundry-incubator/candiedyaml"
)

const DOWNLOAD_TIMEOUT = 10 * time.Minute

type Runner interface {
	Run(config *buildpackapplifecycle.LifecycleBuilderConfig) error
}

type runner struct {
	config *buildpackapplifecycle.LifecycleBuilderConfig
}

type descriptiveError struct {
	message string
	err     error
}

func (e descriptiveError) Error() string {
	if e.err == nil {
		return e.message
	}
	return fmt.Sprintf("%s: %s", e.message, e.err.Error())
}

func newDescriptiveError(err error, message string, args ...interface{}) error {
	if len(args) == 0 {
		return descriptiveError{message: message, err: err}
	}
	return descriptiveError{message: fmt.Sprintf(message, args...), err: err}
}

type Release struct {
	DefaultProcessTypes buildpackapplifecycle.ProcessTypes `yaml:"default_process_types"`
}

func New() Runner {
	return &runner{}
}

func (runner *runner) Run(config *buildpackapplifecycle.LifecycleBuilderConfig) error {
	runner.config = config

	//set up the world
	err := runner.makeDirectories()
	if err != nil {
		return newDescriptiveError(err, "Failed to set up filesystem when generating droplet")
	}

	err = runner.downloadBuildpacks()
	if err != nil {
		return err
	}

	//detect, compile, release
	detectedBuildpack, detectedBuildpackDir, detectOutput, ok := runner.detect()
	if !ok {
		return newDescriptiveError(nil, buildpackapplifecycle.DetectFailMsg)
	}

	err = runner.compile(detectedBuildpackDir)
	if err != nil {
		return newDescriptiveError(nil, buildpackapplifecycle.CompileFailMsg)
	}

	startCommands, err := runner.readProcfile()
	if err != nil {
		return newDescriptiveError(err, "Failed to read command from Procfile")
	}

	releaseInfo, err := runner.release(detectedBuildpackDir, startCommands)
	if err != nil {
		return newDescriptiveError(err, buildpackapplifecycle.ReleaseFailMsg)
	}

	if releaseInfo.DefaultProcessTypes["web"] == "" {
		printError("No start command specified by buildpack or via Procfile.")
		printError("App will not start unless a command is provided at runtime.")
	}

	tarPath, err := exec.LookPath("tar")
	if err != nil {
		return err
	}

	//prepare the final droplet directory
	contentsDir, err := ioutil.TempDir("", "contents")
	if err != nil {
		return newDescriptiveError(err, "Failed to create droplet contents dir")
	}

	//generate staging_info.yml and result json file
	infoFilePath := path.Join(contentsDir, "staging_info.yml")
	err = runner.saveInfo(infoFilePath, detectedBuildpack, detectOutput, releaseInfo)
	if err != nil {
		return newDescriptiveError(err, "Failed to encode generated metadata")
	}

	appDir := path.Join(contentsDir, "app")
	err = runner.copyApp(runner.config.BuildDir(), appDir)
	if err != nil {
		return newDescriptiveError(err, "Failed to copy compiled droplet")
	}

	err = os.MkdirAll(path.Join(contentsDir, "tmp"), 0755)
	if err != nil {
		return newDescriptiveError(err, "Failed to set up droplet filesystem")
	}

	err = os.MkdirAll(path.Join(contentsDir, "logs"), 0755)
	if err != nil {
		return newDescriptiveError(err, "Failed to set up droplet filesystem")
	}

	err = exec.Command(tarPath, "-czf", runner.config.OutputDroplet(), "-C", contentsDir, ".").Run()
	if err != nil {
		return newDescriptiveError(err, "Failed to compress droplet")
	}

	//prepare the build artifacts cache output directory
	err = os.MkdirAll(filepath.Dir(runner.config.OutputBuildArtifactsCache()), 0755)
	if err != nil {
		return newDescriptiveError(err, "Failed to create output build artifacts cache dir")
	}

	err = exec.Command(tarPath, "-czf", runner.config.OutputBuildArtifactsCache(), "-C", runner.config.BuildArtifactsCacheDir(), ".").Run()
	if err != nil {
		return newDescriptiveError(err, "Failed to compress build artifacts")
	}

	return nil
}

func (runner *runner) makeDirectories() error {
	if err := os.MkdirAll(filepath.Dir(runner.config.OutputDroplet()), 0755); err != nil {
		return err
	}

	if err := os.MkdirAll(filepath.Dir(runner.config.OutputMetadata()), 0755); err != nil {
		return err
	}

	if err := os.MkdirAll(runner.config.BuildArtifactsCacheDir(), 0755); err != nil {
		return err
	}

	return nil
}

func (runner *runner) downloadBuildpacks() error {
	// Do we have a custom buildpack?
	for _, buildpackName := range runner.config.BuildpackOrder() {
		buildpackUrl, err := url.Parse(buildpackName)
		if err != nil {
			return fmt.Errorf("Invalid buildpack url (%s): %s", buildpackName, err.Error())
		}
		if !buildpackUrl.IsAbs() {
			continue
		}

		destination := runner.config.BuildpackPath(buildpackName)

		if IsZipFile(buildpackUrl.Path) {
			var size uint64
			zipDownloader := NewZipDownloader(runner.config.SkipCertVerify())
			size, err = zipDownloader.DownloadAndExtract(buildpackUrl, destination)
			if err == nil {
				fmt.Printf("Downloaded buildpack `%s` (%s)\n", buildpackUrl.String(), bytefmt.ByteSize(size))
			}
		} else {
			err = GitClone(*buildpackUrl, destination)
		}
		if err != nil {
			return err
		}
	}

	return nil
}

func (runner *runner) buildpackPath(buildpack string) (string, error) {
	buildpackPath := runner.config.BuildpackPath(buildpack)

	if runner.pathHasBinDirectory(buildpackPath) {
		return buildpackPath, nil
	}

	files, err := ioutil.ReadDir(buildpackPath)
	if err != nil {
		return "", newDescriptiveError(nil, "Failed to read buildpack directory '%s' for buildpack '%s'", buildpackPath, buildpack)
	}

	if len(files) == 1 {
		nestedPath := path.Join(buildpackPath, files[0].Name())

		if runner.pathHasBinDirectory(nestedPath) {
			return nestedPath, nil
		}
	}

	return "", newDescriptiveError(nil, "malformed buildpack does not contain a /bin dir: %s", buildpack)
}

func (runner *runner) pathHasBinDirectory(pathToTest string) bool {
	_, err := os.Stat(path.Join(pathToTest, "bin"))
	return err == nil
}

// returns buildpack name,  buildpack path, buildpack detect output, ok
func (runner *runner) detect() (string, string, string, bool) {
	for _, buildpack := range runner.config.BuildpackOrder() {

		buildpackPath, err := runner.buildpackPath(buildpack)
		if err != nil {
			printError(err.Error())
			continue
		}

		if runner.config.SkipDetect() {
			return buildpack, buildpackPath, "", true
		}

		output := new(bytes.Buffer)
		err = runner.run(exec.Command(path.Join(buildpackPath, "bin", "detect"), runner.config.BuildDir()), output)

		if err == nil {
			return buildpack, buildpackPath, strings.TrimRight(output.String(), "\n"), true
		}
	}

	return "", "", "", false
}

func (runner *runner) readProcfile() (map[string]string, error) {
	processes := map[string]string{}

	procFile, err := os.Open(filepath.Join(runner.config.BuildDir(), "Procfile"))
	if err != nil {
		if os.IsNotExist(err) {
			// Procfiles are optional
			return processes, nil
		}

		return processes, err
	}

	defer procFile.Close()

	err = candiedyaml.NewDecoder(procFile).Decode(&processes)
	if err != nil {
		// clobber candiedyaml's super low-level error
		return processes, errors.New("invalid YAML")
	}

	return processes, nil
}

func (runner *runner) compile(buildpackDir string) error {
	return runner.run(exec.Command(path.Join(buildpackDir, "bin", "compile"), runner.config.BuildDir(), runner.config.BuildArtifactsCacheDir()), os.Stdout)
}

func (runner *runner) release(buildpackDir string, startCommands map[string]string) (Release, error) {
	output := new(bytes.Buffer)

	err := runner.run(exec.Command(path.Join(buildpackDir, "bin", "release"), runner.config.BuildDir()), output)
	if err != nil {
		return Release{}, err
	}

	decoder := candiedyaml.NewDecoder(output)

	parsedRelease := Release{}

	err = decoder.Decode(&parsedRelease)
	if err != nil {
		return Release{}, newDescriptiveError(err, "buildpack's release output invalid")
	}

	if len(startCommands) > 0 {
		parsedRelease.DefaultProcessTypes = startCommands
	}

	return parsedRelease, nil
}

func (runner *runner) saveInfo(infoFilePath, buildpack, detectOutput string, releaseInfo Release) error {
	deaInfoFile, err := os.Create(infoFilePath)
	if err != nil {
		return err
	}
	defer deaInfoFile.Close()

	// JSON ⊂ YAML
	err = json.NewEncoder(deaInfoFile).Encode(DeaStagingInfo{
		DetectedBuildpack: detectOutput,
		StartCommand:      releaseInfo.DefaultProcessTypes["web"],
	})
	if err != nil {
		return err
	}

	resultFile, err := os.Create(runner.config.OutputMetadata())
	if err != nil {
		return err
	}
	defer resultFile.Close()

	err = json.NewEncoder(resultFile).Encode(buildpackapplifecycle.NewStagingResult(
		releaseInfo.DefaultProcessTypes,
		buildpackapplifecycle.LifecycleMetadata{
			BuildpackKey:      buildpack,
			DetectedBuildpack: detectOutput,
		},
	))
	if err != nil {
		return err
	}

	return nil
}

func (runner *runner) copyApp(buildDir, stageDir string) error {
	return runner.run(exec.Command("cp", "-a", buildDir, stageDir), os.Stdout)
}

func (runner *runner) run(cmd *exec.Cmd, output io.Writer) error {
	cmd.Stdout = output
	cmd.Stderr = os.Stderr

	return cmd.Run()
}

func printError(message string) {
	println(message)
}
<|endoftext|>


<commit_before>package main

import (
	twodee "../libs/twodee"
	"fmt"
	"math"
	"math/rand"
	"time"
)

type PlanetaryState int32

const (
	_                  = iota
	Sun PlanetaryState = 1 << iota
	Fertile
	TooClose
	TooFar
)

var PlanetaryAnimations = map[PlanetaryState][]int{
	Sun:      []int{0},
	Fertile:  []int{4},
	TooClose: []int{8},
	TooFar:   []int{12},
}

type PlanetaryBody struct {
	*twodee.AnimatingEntity
	Velocity             twodee.Point
	Mass                 float32
	Population           float32
	MaxPopulation        float32
	PopulationGrowthRate float32
	State                PlanetaryState
}

func NewSun() *PlanetaryBody {
	body := &PlanetaryBody{
		AnimatingEntity: twodee.NewAnimatingEntity(
			0, 0,
			32.0/PxPerUnit, 32.0/PxPerUnit,
			0,
			twodee.Step10Hz,
			[]int{0},
		),
		Mass:                 1000.0,
		Population:           0.0,
		MaxPopulation:        0.0,
		PopulationGrowthRate: 0.0,
	}
	body.SetState(Sun)
	return body
}

func NewPlanet(x, y float32) *PlanetaryBody {
	var (
		Mass float32
	)

	body := &PlanetaryBody{
		AnimatingEntity: twodee.NewAnimatingEntity(
			x, y,
			32.0/PxPerUnit, 32.0/PxPerUnit,
			0,
			twodee.Step10Hz,
			[]int{0},
		),
		Velocity:             twodee.Pt(rand.Float32(), rand.Float32()),
		Mass:                 2000.0,
		Population:           100.0,
		MaxPopulation:        Mass * 1000.0,
		PopulationGrowthRate: 1.0,
	}
	body.SetState(Fertile)
	fmt.Printf("Growth Rate: %f\n", body.PopulationGrowthRate)
	fmt.Printf("Max Population: %f\n", body.MaxPopulation)
	fmt.Printf("Population: %f\n", body.Population)
	return body
}

func (p *PlanetaryBody) MoveToward(sc twodee.Point) {
	var (
		pc = p.Pos()
		dx = float64(sc.X - pc.X)
		dy = float64(sc.Y - pc.Y)
		h  = math.Hypot(dx, dy)
		vx = float32(math.Max(1, 5-h) * 0.3 * dx / h)
		vy = float32(math.Max(1, 5-h) * 0.3 * dy / h)
	)
	p.Velocity.X += (vx - p.Velocity.X)
	p.Velocity.Y += (vy - p.Velocity.Y)
}

func (p *PlanetaryBody) GravitateToward(sc twodee.Point) {
	var (
		pc  = p.Pos()
		avx = float64(sc.X - pc.X)
		avy = float64(sc.Y - pc.Y)
		d   = math.Hypot(avx, avy)
	)
	// Normalize vector and include sensible constraints.
	avx = avx / d
	avy = avy / d
	av := twodee.Pt(float32(math.Max(1, 5-d)*0.3*avx), float32(math.Max(1, 5-d)*0.3*avy))

	// There are two possible orthogonal 'circulation' vectors.
	cv1 := twodee.Pt(-av.Y, av.X)
	cv2 := twodee.Pt(av.Y, -av.X)
	cv := cv1

	// Compute whichever circulation vector is closer to our present vector.
	// cos(theta) = A -dot- B / ||A||*||B||
	dp1 := p.Velocity.X*cv1.X + p.Velocity.Y*cv1.Y
	denom := math.Sqrt(float64(p.Velocity.X*p.Velocity.X + p.Velocity.Y*p.Velocity.Y))
	theta1 := dp1 / float32(denom)
	dp2 := p.Velocity.X*cv2.X + p.Velocity.Y*cv2.Y
	theta2 := dp2 / float32(denom)
	if theta1 >= theta2 {
		cv = cv1
	} else {
		cv = cv2
	}

	// Now do some vector addition.
	fv := twodee.Pt(av.X+cv.X, av.Y+cv.Y)
	p.Velocity.X += (fv.X - p.Velocity.X) / 30
	p.Velocity.Y += (fv.Y - p.Velocity.Y) / 30
}

func (p *PlanetaryBody) UpdatePopulation(elapsed time.Duration) {
	p.Population = p.MaxPopulation / (1 + ((p.MaxPopulation/p.Population)-1)*float32(math.Exp(-1*float64(p.PopulationGrowthRate)*float64(elapsed))))
}

func (p *PlanetaryBody) Update(elapsed time.Duration) {
	p.AnimatingEntity.Update(elapsed)
	p.UpdatePopulation(elapsed)
	pos := p.Pos()
	p.MoveTo(twodee.Pt(pos.X+p.Velocity.X, pos.Y+p.Velocity.Y))
}

func (p *PlanetaryBody) RemState(state PlanetaryState) {
	p.SetState(p.State & ^state)
}

func (p *PlanetaryBody) AddState(state PlanetaryState) {
	p.SetState(p.State | state)
}

func (p *PlanetaryBody) SwapState(rem, add PlanetaryState) {
	p.SetState(p.State & ^rem | add)
}

func (p *PlanetaryBody) SetState(state PlanetaryState) {
	if state != p.State {
		p.State = state
		if frames, ok := PlanetaryAnimations[p.State]; ok {
			p.SetFrames(frames)
		}
	}
}
<commit_msg>In progress population growth tracking<commit_after>package main

import (
	twodee "../libs/twodee"
	//"fmt"
	"math"
	"math/rand"
	"time"
)

type PlanetaryState int32

const (
	_                  = iota
	Sun PlanetaryState = 1 << iota
	Fertile
	TooClose
	TooFar
)

var PlanetaryAnimations = map[PlanetaryState][]int{
	Sun:      []int{0},
	Fertile:  []int{4},
	TooClose: []int{8},
	TooFar:   []int{12},
}

type PlanetaryBody struct {
	*twodee.AnimatingEntity
	Velocity             twodee.Point
	Mass                 float32
	Population           float32
	MaxPopulation        float32
	PopulationGrowthRate float32
	State                PlanetaryState
}

func NewSun() *PlanetaryBody {
	body := &PlanetaryBody{
		AnimatingEntity: twodee.NewAnimatingEntity(
			0, 0,
			32.0/PxPerUnit, 32.0/PxPerUnit,
			0,
			twodee.Step10Hz,
			[]int{0},
		),
		Mass:                 1000.0,
		Population:           0.0,
		MaxPopulation:        0.0,
		PopulationGrowthRate: 0.0,
	}
	body.SetState(Sun)
	return body
}

func NewPlanet(x, y float32) *PlanetaryBody {
	body := &PlanetaryBody{
		AnimatingEntity: twodee.NewAnimatingEntity(
			x, y,
			32.0/PxPerUnit, 32.0/PxPerUnit,
			0,
			twodee.Step10Hz,
			[]int{0},
		),
		Velocity:             twodee.Pt(rand.Float32(), rand.Float32()),
		Mass:                 2000.0,
		Population:           100.0,
		MaxPopulation:        0.0,
		PopulationGrowthRate: 1.0,
	}
	body.SetState(Fertile)
	body.MaxPopulation = body.Mass * 1000
	return body
}

func (p *PlanetaryBody) MoveToward(sc twodee.Point) {
	var (
		pc = p.Pos()
		dx = float64(sc.X - pc.X)
		dy = float64(sc.Y - pc.Y)
		h  = math.Hypot(dx, dy)
		vx = float32(math.Max(1, 5-h) * 0.3 * dx / h)
		vy = float32(math.Max(1, 5-h) * 0.3 * dy / h)
	)
	p.Velocity.X += (vx - p.Velocity.X)
	p.Velocity.Y += (vy - p.Velocity.Y)
}

func (p *PlanetaryBody) GravitateToward(sc twodee.Point) {
	var (
		pc  = p.Pos()
		avx = float64(sc.X - pc.X)
		avy = float64(sc.Y - pc.Y)
		d   = math.Hypot(avx, avy)
	)
	// Normalize vector and include sensible constraints.
	avx = avx / d
	avy = avy / d
	av := twodee.Pt(float32(math.Max(1, 5-d)*0.3*avx), float32(math.Max(1, 5-d)*0.3*avy))

	// There are two possible orthogonal 'circulation' vectors.
	cv1 := twodee.Pt(-av.Y, av.X)
	cv2 := twodee.Pt(av.Y, -av.X)
	cv := cv1

	// Compute whichever circulation vector is closer to our present vector.
	// cos(theta) = A -dot- B / ||A||*||B||
	dp1 := p.Velocity.X*cv1.X + p.Velocity.Y*cv1.Y
	denom := math.Sqrt(float64(p.Velocity.X*p.Velocity.X + p.Velocity.Y*p.Velocity.Y))
	theta1 := dp1 / float32(denom)
	dp2 := p.Velocity.X*cv2.X + p.Velocity.Y*cv2.Y
	theta2 := dp2 / float32(denom)
	if theta1 >= theta2 {
		cv = cv1
	} else {
		cv = cv2
	}

	// Now do some vector addition.
	fv := twodee.Pt(av.X+cv.X, av.Y+cv.Y)
	p.Velocity.X += (fv.X - p.Velocity.X) / 30
	p.Velocity.Y += (fv.Y - p.Velocity.Y) / 30
}

func (p *PlanetaryBody) UpdatePopulation(elapsed time.Duration) {
	/*var (
		growing = 1.0
	)

	if (p.State == 1) || (p.State == 2) {
		growing = 1.0
	} else {
		growing = -1.0
	}*/

	p.Population = p.MaxPopulation / (1 + ((p.MaxPopulation/p.Population)-1)*float32(math.Exp(-1*float64(p.PopulationGrowthRate)*float64(elapsed))))
	//fmt.Printf("Growth Rate: %f\n", p.PopulationGrowthRate)
	//fmt.Printf("Max Population: %f\n", p.MaxPopulation)
	//fmt.Printf("Population: %f\n", p.Population)
	//fmt.Printf("growing: %f\n", growing)
}

func (p *PlanetaryBody) Update(elapsed time.Duration) {
	p.AnimatingEntity.Update(elapsed)
	p.UpdatePopulation(elapsed)
	pos := p.Pos()
	p.MoveTo(twodee.Pt(pos.X+p.Velocity.X, pos.Y+p.Velocity.Y))
}

func (p *PlanetaryBody) RemState(state PlanetaryState) {
	p.SetState(p.State & ^state)
}

func (p *PlanetaryBody) AddState(state PlanetaryState) {
	p.SetState(p.State | state)
}

func (p *PlanetaryBody) SwapState(rem, add PlanetaryState) {
	p.SetState(p.State & ^rem | add)
}

func (p *PlanetaryBody) SetState(state PlanetaryState) {
	if state != p.State {
		p.State = state
		if frames, ok := PlanetaryAnimations[p.State]; ok {
			p.SetFrames(frames)
		}
	}
}
<|endoftext|>