<commit_before>package http2

import (
	"crypto/tls"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/dropbox/godropbox/errors"
)

const (
	// This is a hax to propagate DialError with the Do call
	dialErrorMsgPrefix = "dial "
)

// Pool of persistent HTTP connections. The only limit is on the max # of idle connections we
// cache. Like Python's dropbox.curllib.CurlConnectionPool.
type SimplePool struct {
	// no locking needed, because http.Client has its own locking
	client    *http.Client
	transport *http.Transport

	addr   string
	params ConnectionParams
}

// get from http://golang.org/src/pkg/net/http/transport.go
func getenvEitherCase(k string) string {
	if v := os.Getenv(strings.ToUpper(k)); v != "" {
		return v
	}
	return os.Getenv(strings.ToLower(k))
}

// Creates a new HTTP connection pool using the given address and pool parameters.
//
// 'addr' is a net.Dial()-style 'host:port' destination for making the TCP connection for
// HTTP/HTTPS traffic.  It will be used as the hostname by default for virtual hosting
// and SSL certificate validation; if you'd like to use a different hostname,
// set params.HostHeader.
func NewSimplePool(addr string, params ConnectionParams) *SimplePool {
	pool := &SimplePool{
		addr:   addr,
		params: params,
		client: new(http.Client),
	}

	// setup HTTP transport
	transport := new(http.Transport)
	transport.ResponseHeaderTimeout = params.ResponseTimeout
	transport.MaxIdleConnsPerHost = params.MaxIdle

	if params.Proxy != nil {
		transport.Proxy = params.Proxy
	} else {
		transport.Proxy = http.ProxyFromEnvironment
	}

	if params.Dial == nil {
		// dialTimeout could only be used in none proxy requests since it talks directly
		// to pool.addr
		if getenvEitherCase("HTTP_PROXY") == "" && params.Proxy == nil {
			transport.Dial = pool.dialTimeout
		}
	} else {
		transport.Dial = params.Dial
	}
	pool.transport = transport
	pool.client.Transport = transport

	if params.UseSSL && params.SkipVerifySSL {
		transport.TLSClientConfig = &tls.Config{
			InsecureSkipVerify: true,
		}
	}

	return pool
}

// Adds connection timeout for HTTP client
//
// Note - we do not use the addr passed in, which golang's http library
// has parsed from the URL, so that we can connect to whatever specific host
// was passed in originally as addr to NewSimplePool().
func (pool *SimplePool) dialTimeout(network, addr string) (net.Conn, error) {
	c, err := net.DialTimeout(network, pool.addr, pool.params.ConnectTimeout)
	if err == nil {
		tcp := c.(*net.TCPConn)
		tcp.SetKeepAlive(true)
		tcp.SetKeepAlivePeriod(10 * time.Second)
	}
	return c, err
}

// Performs the HTTP request using our HTTP client
func (pool *SimplePool) Do(req *http.Request) (resp *http.Response, err error) {
	conn, err := pool.Get()
	if err != nil {
		return nil, errors.Wrap(err, err.Error())
	}

	if pool.params.UseSSL {
		req.URL.Scheme = "https"
	} else {
		req.URL.Scheme = "http"
	}
	if pool.params.HostHeader != nil {
		req.URL.Host = *pool.params.HostHeader
	} else {
		req.URL.Host = pool.addr
	}

	// transport.ResponseHeaderTimeout doesn't encompass the time it takes to write the full
	// request, thus timeout for reading the body should be handled by the caller who is
	// consuming the response.
	resp, err = conn.Do(req)
	if err != nil {
		var isDialError bool
		if urlErr, ok := err.(*url.Error); ok {
			if strings.HasPrefix(urlErr.Err.Error(), dialErrorMsgPrefix) {
				isDialError = true
			}
		}
		if isDialError {
			err = DialError{errors.Wrap(err, "SimplePool: Dial Error")}
		} else {
			err = errors.Wrap(err, err.Error())
		}
	}
	return
}

// Returns the HTTP client, which is thread-safe.
//
// Note that we use http.Client, rather than httputil.ClientConn, despite http.Client being higher-
// level. This is normally a liability for backend code, but it has more robust error handling and
// provides functionality that's more comparable to pycurl/curllib.
func (pool *SimplePool) Get() (*http.Client, error) {
	return pool.client, nil
}

// Closes all idle connections in this pool
func (pool *SimplePool) Close() {
	pool.transport.CloseIdleConnections()
}
<commit_msg>set http.Client.Timeout in net2/http2.SimplePool if supported (>= go-1.3)<commit_after>package http2

import (
	"crypto/tls"
	"net"
	"net/http"
	"net/url"
	"os"
	"reflect"
	"strings"
	"time"

	"github.com/dropbox/godropbox/errors"
)

const (
	// This is a hax to propagate DialError with the Do call
	dialErrorMsgPrefix = "dial "
)

// Pool of persistent HTTP connections. The only limit is on the max # of idle connections we
// cache. Like Python's dropbox.curllib.CurlConnectionPool.
type SimplePool struct {
	// no locking needed, because http.Client has its own locking
	client    *http.Client
	transport *http.Transport

	addr   string
	params ConnectionParams
}

// get from http://golang.org/src/pkg/net/http/transport.go
func getenvEitherCase(k string) string {
	if v := os.Getenv(strings.ToUpper(k)); v != "" {
		return v
	}
	return os.Getenv(strings.ToLower(k))
}

// Go 1.3+ supports specifying a global timeout on the client. See
// http://golang.org/pkg/net/http/#Client.
// It's desirable to enforce the timeout at the client-level since it includes
// the connection time, redirects and the time to finish reading the full response.
// Unlike ResponseHeaderTimeout supported by `http.Transport` which merely accounts
// for the timeout to receive the first response header byte. It ignores the time to
// send the request or the time to read the full response.
func isClientTimeoutSupported(client *http.Client) bool {
	clientType := reflect.TypeOf(client)
	if clientType.Kind() == reflect.Ptr {
		clientType = clientType.Elem()

		if clientType.Kind() == reflect.Struct {
			_, found := clientType.FieldByName("Timeout")
			return found
		}
	}

	return false
}

// Creates a new HTTP connection pool using the given address and pool parameters.
//
// 'addr' is a net.Dial()-style 'host:port' destination for making the TCP connection for
// HTTP/HTTPS traffic.  It will be used as the hostname by default for virtual hosting
// and SSL certificate validation; if you'd like to use a different hostname,
// set params.HostHeader.
func NewSimplePool(addr string, params ConnectionParams) *SimplePool {
	pool := &SimplePool{
		addr:   addr,
		params: params,
		client: new(http.Client),
	}

	if isClientTimeoutSupported(pool.client) {
		pool.client.Timeout = params.ResponseTimeout
	}

	// setup HTTP transport
	transport := new(http.Transport)
	transport.ResponseHeaderTimeout = params.ResponseTimeout
	transport.MaxIdleConnsPerHost = params.MaxIdle

	if params.Proxy != nil {
		transport.Proxy = params.Proxy
	} else {
		transport.Proxy = http.ProxyFromEnvironment
	}

	if params.Dial == nil {
		// dialTimeout could only be used in none proxy requests since it talks directly
		// to pool.addr
		if getenvEitherCase("HTTP_PROXY") == "" && params.Proxy == nil {
			transport.Dial = pool.dialTimeout
		}
	} else {
		transport.Dial = params.Dial
	}
	pool.transport = transport
	pool.client.Transport = transport

	if params.UseSSL && params.SkipVerifySSL {
		transport.TLSClientConfig = &tls.Config{
			InsecureSkipVerify: true,
		}
	}

	return pool
}

// Adds connection timeout for HTTP client
//
// Note - we do not use the addr passed in, which golang's http library
// has parsed from the URL, so that we can connect to whatever specific host
// was passed in originally as addr to NewSimplePool().
func (pool *SimplePool) dialTimeout(network, addr string) (net.Conn, error) {
	c, err := net.DialTimeout(network, pool.addr, pool.params.ConnectTimeout)
	if err == nil {
		tcp := c.(*net.TCPConn)
		tcp.SetKeepAlive(true)
		tcp.SetKeepAlivePeriod(10 * time.Second)
	}
	return c, err
}

// Performs the HTTP request using our HTTP client
func (pool *SimplePool) Do(req *http.Request) (resp *http.Response, err error) {
	conn, err := pool.Get()
	if err != nil {
		return nil, errors.Wrap(err, err.Error())
	}

	if pool.params.UseSSL {
		req.URL.Scheme = "https"
	} else {
		req.URL.Scheme = "http"
	}
	if pool.params.HostHeader != nil {
		req.URL.Host = *pool.params.HostHeader
	} else {
		req.URL.Host = pool.addr
	}

	// transport.ResponseHeaderTimeout doesn't encompass the time it takes to write the full
	// request, thus timeout for reading the body should be handled by the caller who is
	// consuming the response.
	resp, err = conn.Do(req)
	if err != nil {
		var isDialError bool
		if urlErr, ok := err.(*url.Error); ok {
			if strings.HasPrefix(urlErr.Err.Error(), dialErrorMsgPrefix) {
				isDialError = true
			}
		}
		if isDialError {
			err = DialError{errors.Wrap(err, "SimplePool: Dial Error")}
		} else {
			err = errors.Wrap(err, err.Error())
		}
	}
	return
}

// Returns the HTTP client, which is thread-safe.
//
// Note that we use http.Client, rather than httputil.ClientConn, despite http.Client being higher-
// level. This is normally a liability for backend code, but it has more robust error handling and
// provides functionality that's more comparable to pycurl/curllib.
func (pool *SimplePool) Get() (*http.Client, error) {
	return pool.client, nil
}

// Closes all idle connections in this pool
func (pool *SimplePool) Close() {
	pool.transport.CloseIdleConnections()
}
<|endoftext|>


<commit_before>package roaring

// to run just these tests: go test -run TestFastAggregations*

import (
	"container/heap"
	. "github.com/smartystreets/goconvey/convey"
	"testing"
)

func TestFastAggregations(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb1.Add(1)
		rb2.Add(2)

		So(FastAnd(rb1, rb2).GetCardinality(), ShouldEqual, 0)
		So(FastOr(rb1, rb2).GetCardinality(), ShouldEqual, 2)
		So(HeapXor(rb1, rb2).GetCardinality(), ShouldEqual, 2)
	})
}

func TestFastAggregationsNothing(t *testing.T) {
	Convey("Fast", t, func() {
		So(FastAnd().GetCardinality(), ShouldEqual, 0)
		So(FastOr().GetCardinality(), ShouldEqual, 0)
		So(HeapXor().GetCardinality(), ShouldEqual, 0)
	})
}

func TestFastAggregationsOneEmpty(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb1.Add(1)

		So(FastAnd(rb1, rb2).GetCardinality(), ShouldEqual, 0)
		So(FastOr(rb1, rb2).GetCardinality(), ShouldEqual, 1)
		So(HeapXor(rb1, rb2).GetCardinality(), ShouldEqual, 1)
	})
}

func TestFastAggregations3(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()
		rb1.Add(1)
		rb1.Add(100000)
		rb2.Add(200000)
		rb3.Add(1)
		rb3.Add(300000)

		So(FastAnd(rb1, rb2, rb3).GetCardinality(), ShouldEqual, 0)
		So(FastOr(rb1, rb2, rb3).GetCardinality(), ShouldEqual, 4)
		So(HeapXor(rb1, rb2, rb3).GetCardinality(), ShouldEqual, 3)
	})
}

func TestFastAggregationsSize(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()
		for i := uint32(0); i < 1000000; i += 3 {
			rb1.Add(i)
		}
		for i := uint32(0); i < 1000000; i += 7 {
			rb2.Add(i)
		}
		for i := uint32(0); i < 1000000; i += 1001 {
			rb3.Add(i)
		}
		pq := make(priorityQueue, 3)
		pq[0] = &item{rb1, 0}
		pq[1] = &item{rb2, 1}
		pq[2] = &item{rb3, 2}
		heap.Init(&pq)
		So(heap.Pop(&pq).(*item).value.GetSizeInBytes(), ShouldEqual, rb3.GetSizeInBytes())
		So(heap.Pop(&pq).(*item).value.GetSizeInBytes(), ShouldEqual, rb2.GetSizeInBytes())
		So(heap.Pop(&pq).(*item).value.GetSizeInBytes(), ShouldEqual, rb1.GetSizeInBytes())
	})
}

func TestFastAggregationsCont(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()
		for i := uint32(0); i < 10; i += 3 {
			rb1.Add(i)
		}
		for i := uint32(0); i < 10; i += 7 {
			rb2.Add(i)
		}
		for i := uint32(0); i < 10; i += 1001 {
			rb3.Add(i)
		}
		for i := uint32(1000000); i < 1000000+10; i += 1001 {
			rb1.Add(i)
		}
		for i := uint32(1000000); i < 1000000+10; i += 7 {
			rb2.Add(i)
		}
		for i := uint32(1000000); i < 1000000+10; i += 3 {
			rb3.Add(i)
		}
		rb1.Add(500000)
		pq := make(containerPriorityQueue, 3)
		pq[0] = &containeritem{rb1, 0, 0}
		pq[1] = &containeritem{rb2, 0, 1}
		pq[2] = &containeritem{rb3, 0, 2}
		heap.Init(&pq)
		expected := []int{6, 4, 5, 6, 5, 4, 6}
		counter := 0
		for pq.Len() > 0 {
			x1 := heap.Pop(&pq).(*containeritem)
			So(x1.value.GetCardinality(), ShouldEqual, expected[counter])
			counter++
			x1.keyindex++
			if x1.keyindex < x1.value.highlowcontainer.size() {
				heap.Push(&pq, x1)
			}
		}
	})
}
func TestFastAggregationsAdvanced(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()
		for i := uint32(0); i < 1000000; i += 3 {
			rb1.Add(i)
		}
		for i := uint32(0); i < 1000000; i += 7 {
			rb2.Add(i)
		}
		for i := uint32(0); i < 1000000; i += 1001 {
			rb3.Add(i)
		}
		for i := uint32(1000000); i < 2000000; i += 1001 {
			rb1.Add(i)
		}
		for i := uint32(1000000); i < 2000000; i += 3 {
			rb2.Add(i)
		}
		for i := uint32(1000000); i < 2000000; i += 7 {
			rb3.Add(i)
		}
		rb1.Or(rb2)
		rb1.Or(rb3)
		bigand := And(And(rb1, rb2), rb3)
		bigxor := Xor(Xor(rb1, rb2), rb3)
		So(FastOr(rb1, rb2, rb3).Equals(rb1), ShouldEqual, true)
		So(HeapOr(rb1, rb2, rb3).Equals(rb1), ShouldEqual, true)
		So(HeapOr(rb1, rb2, rb3).GetCardinality(), ShouldEqual, rb1.GetCardinality())
		So(HeapXor(rb1, rb2, rb3).Equals(bigxor), ShouldEqual, true)
		So(FastAnd(rb1, rb2, rb3).Equals(bigand), ShouldEqual, true)
	})
}

func TestFastAggregationsAdvanced_run(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()
		for i := uint32(500); i < 75000; i++ {
			rb1.Add(i)
		}
		for i := uint32(0); i < 1000000; i += 7 {
			rb2.Add(i)
		}
		for i := uint32(0); i < 1000000; i += 1001 {
			rb3.Add(i)
		}
		for i := uint32(1000000); i < 2000000; i += 1001 {
			rb1.Add(i)
		}
		for i := uint32(1000000); i < 2000000; i += 3 {
			rb2.Add(i)
		}
		for i := uint32(1000000); i < 2000000; i += 7 {
			rb3.Add(i)
		}
		rb1.RunOptimize()
		rb1.Or(rb2)
		rb1.Or(rb3)
		bigand := And(And(rb1, rb2), rb3)
		bigxor := Xor(Xor(rb1, rb2), rb3)
		So(FastOr(rb1, rb2, rb3).Equals(rb1), ShouldEqual, true)
		So(HeapOr(rb1, rb2, rb3).Equals(rb1), ShouldEqual, true)
		So(HeapOr(rb1, rb2, rb3).GetCardinality(), ShouldEqual, rb1.GetCardinality())
		So(HeapXor(rb1, rb2, rb3).Equals(bigxor), ShouldEqual, true)
		So(FastAnd(rb1, rb2, rb3).Equals(bigand), ShouldEqual, true)
	})
}

func TestFastAggregationsXOR(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()

		for i := uint32(0); i < 40000; i++ {
			rb1.Add(i)
		}
		for i := uint32(0); i < 40000; i += 4000 {
			rb2.Add(i)
		}
		for i := uint32(0); i < 40000; i += 5000 {
			rb3.Add(i)
		}
		So(rb1.GetCardinality() == 40000, ShouldEqual, true)

		xor1 := Xor(rb1, rb2)
		xor1alt := Xor(rb2, rb1)
		So(xor1alt.Equals(xor1), ShouldEqual, true)
		So(HeapXor(rb1, rb2).Equals(xor1), ShouldEqual, true)

		xor2 := Xor(rb2, rb3)
		xor2alt := Xor(rb3, rb2)
		So(xor2alt.Equals(xor2), ShouldEqual, true)
		So(HeapXor(rb2, rb3).Equals(xor2), ShouldEqual, true)

		bigxor := Xor(Xor(rb1, rb2), rb3)
		bigxoralt1 := Xor(rb1, Xor(rb2, rb3))
		bigxoralt2 := Xor(rb1, Xor(rb3, rb2))
		bigxoralt3 := Xor(rb3, Xor(rb1, rb2))
		bigxoralt4 := Xor(Xor(rb1, rb2), rb3)

		So(bigxoralt2.Equals(bigxor), ShouldEqual, true)
		So(bigxoralt1.Equals(bigxor), ShouldEqual, true)
		So(bigxoralt3.Equals(bigxor), ShouldEqual, true)
		So(bigxoralt4.Equals(bigxor), ShouldEqual, true)

		So(HeapXor(rb1, rb2, rb3).Equals(bigxor), ShouldEqual, true)
	})
}

func TestFastAggregationsXOR_run(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()

		for i := uint32(0); i < 40000; i++ {
			rb1.Add(i)
		}
		rb1.RunOptimize()
		for i := uint32(0); i < 40000; i += 4000 {
			rb2.Add(i)
		}
		for i := uint32(0); i < 40000; i += 5000 {
			rb3.Add(i)
		}
		So(rb1.GetCardinality() == 40000, ShouldEqual, true)

		xor1 := Xor(rb1, rb2)
		xor1alt := Xor(rb2, rb1)
		So(xor1alt.Equals(xor1), ShouldEqual, true)
		So(HeapXor(rb1, rb2).Equals(xor1), ShouldEqual, true)

		xor2 := Xor(rb2, rb3)
		xor2alt := Xor(rb3, rb2)
		So(xor2alt.Equals(xor2), ShouldEqual, true)
		So(HeapXor(rb2, rb3).Equals(xor2), ShouldEqual, true)

		bigxor := Xor(Xor(rb1, rb2), rb3)
		bigxoralt1 := Xor(rb1, Xor(rb2, rb3))
		bigxoralt2 := Xor(rb1, Xor(rb3, rb2))
		bigxoralt3 := Xor(rb3, Xor(rb1, rb2))
		bigxoralt4 := Xor(Xor(rb1, rb2), rb3)

		So(bigxoralt2.Equals(bigxor), ShouldEqual, true)
		So(bigxoralt1.Equals(bigxor), ShouldEqual, true)
		So(bigxoralt3.Equals(bigxor), ShouldEqual, true)
		So(bigxoralt4.Equals(bigxor), ShouldEqual, true)

		So(HeapXor(rb1, rb2, rb3).Equals(bigxor), ShouldEqual, true)
	})
}
<commit_msg>Adding a few more tests.<commit_after>package roaring

// to run just these tests: go test -run TestFastAggregations*

import (
	"container/heap"
	. "github.com/smartystreets/goconvey/convey"
	"testing"
)

func TestFastAggregations(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb1.Add(1)
		rb2.Add(2)

		So(FastAnd(rb1, rb2).GetCardinality(), ShouldEqual, 0)
		So(FastOr(rb1, rb2).GetCardinality(), ShouldEqual, 2)
		So(HeapXor(rb1, rb2).GetCardinality(), ShouldEqual, 2)
	})
}

func TestFastAggregationsNothing(t *testing.T) {
	Convey("Fast", t, func() {
		So(FastAnd().GetCardinality(), ShouldEqual, 0)
		So(FastOr().GetCardinality(), ShouldEqual, 0)
		So(HeapXor().GetCardinality(), ShouldEqual, 0)
	})
}

func TestFastAggregationsOneEmpty(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb1.Add(1)

		So(FastAnd(rb1, rb2).GetCardinality(), ShouldEqual, 0)
		So(FastOr(rb1, rb2).GetCardinality(), ShouldEqual, 1)
		So(HeapXor(rb1, rb2).GetCardinality(), ShouldEqual, 1)
	})
}

func TestFastAggregationsReversed3COW(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()
		rb1.SetCopyOnWrite(true)
		rb2.SetCopyOnWrite(true)
		rb3.SetCopyOnWrite(true)
		rb1.Add(1)
		rb1.Add(100000)
		rb2.Add(200000)
		rb3.Add(1)
		rb3.Add(300000)

		So(FastAnd(rb2, rb1, rb3).GetCardinality(), ShouldEqual, 0)
		So(FastOr(rb2, rb1, rb3).GetCardinality(), ShouldEqual, 4)
		So(HeapXor(rb2, rb1, rb3).GetCardinality(), ShouldEqual, 3)
	})
}

func TestFastAggregationsReversed3(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()
		rb1.Add(1)
		rb1.Add(100000)
		rb2.Add(200000)
		rb3.Add(1)
		rb3.Add(300000)

		So(FastAnd(rb2, rb1, rb3).GetCardinality(), ShouldEqual, 0)
		So(FastOr(rb2, rb1, rb3).GetCardinality(), ShouldEqual, 4)
		So(HeapXor(rb2, rb1, rb3).GetCardinality(), ShouldEqual, 3)
	})
}

func TestFastAggregations3(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()
		rb1.Add(1)
		rb1.Add(100000)
		rb2.Add(200000)
		rb3.Add(1)
		rb3.Add(300000)

		So(FastAnd(rb1, rb2, rb3).GetCardinality(), ShouldEqual, 0)
		So(FastOr(rb1, rb2, rb3).GetCardinality(), ShouldEqual, 4)
		So(HeapXor(rb1, rb2, rb3).GetCardinality(), ShouldEqual, 3)
	})
}

func TestFastAggregations3COW(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()
		rb1.SetCopyOnWrite(true)
		rb2.SetCopyOnWrite(true)
		rb3.SetCopyOnWrite(true)
		rb1.Add(1)
		rb1.Add(100000)
		rb2.Add(200000)
		rb3.Add(1)
		rb3.Add(300000)

		So(FastAnd(rb1, rb2, rb3).GetCardinality(), ShouldEqual, 0)
		So(FastOr(rb1, rb2, rb3).GetCardinality(), ShouldEqual, 4)
		So(HeapXor(rb1, rb2, rb3).GetCardinality(), ShouldEqual, 3)
	})
}

func TestFastAggregationsSize(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()
		for i := uint32(0); i < 1000000; i += 3 {
			rb1.Add(i)
		}
		for i := uint32(0); i < 1000000; i += 7 {
			rb2.Add(i)
		}
		for i := uint32(0); i < 1000000; i += 1001 {
			rb3.Add(i)
		}
		pq := make(priorityQueue, 3)
		pq[0] = &item{rb1, 0}
		pq[1] = &item{rb2, 1}
		pq[2] = &item{rb3, 2}
		heap.Init(&pq)
		So(heap.Pop(&pq).(*item).value.GetSizeInBytes(), ShouldEqual, rb3.GetSizeInBytes())
		So(heap.Pop(&pq).(*item).value.GetSizeInBytes(), ShouldEqual, rb2.GetSizeInBytes())
		So(heap.Pop(&pq).(*item).value.GetSizeInBytes(), ShouldEqual, rb1.GetSizeInBytes())
	})
}

func TestFastAggregationsCont(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()
		for i := uint32(0); i < 10; i += 3 {
			rb1.Add(i)
		}
		for i := uint32(0); i < 10; i += 7 {
			rb2.Add(i)
		}
		for i := uint32(0); i < 10; i += 1001 {
			rb3.Add(i)
		}
		for i := uint32(1000000); i < 1000000+10; i += 1001 {
			rb1.Add(i)
		}
		for i := uint32(1000000); i < 1000000+10; i += 7 {
			rb2.Add(i)
		}
		for i := uint32(1000000); i < 1000000+10; i += 3 {
			rb3.Add(i)
		}
		rb1.Add(500000)
		pq := make(containerPriorityQueue, 3)
		pq[0] = &containeritem{rb1, 0, 0}
		pq[1] = &containeritem{rb2, 0, 1}
		pq[2] = &containeritem{rb3, 0, 2}
		heap.Init(&pq)
		expected := []int{6, 4, 5, 6, 5, 4, 6}
		counter := 0
		for pq.Len() > 0 {
			x1 := heap.Pop(&pq).(*containeritem)
			So(x1.value.GetCardinality(), ShouldEqual, expected[counter])
			counter++
			x1.keyindex++
			if x1.keyindex < x1.value.highlowcontainer.size() {
				heap.Push(&pq, x1)
			}
		}
	})
}
func TestFastAggregationsAdvanced(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()
		for i := uint32(0); i < 1000000; i += 3 {
			rb1.Add(i)
		}
		for i := uint32(0); i < 1000000; i += 7 {
			rb2.Add(i)
		}
		for i := uint32(0); i < 1000000; i += 1001 {
			rb3.Add(i)
		}
		for i := uint32(1000000); i < 2000000; i += 1001 {
			rb1.Add(i)
		}
		for i := uint32(1000000); i < 2000000; i += 3 {
			rb2.Add(i)
		}
		for i := uint32(1000000); i < 2000000; i += 7 {
			rb3.Add(i)
		}
		rb1.Or(rb2)
		rb1.Or(rb3)
		bigand := And(And(rb1, rb2), rb3)
		bigxor := Xor(Xor(rb1, rb2), rb3)
		So(FastOr(rb1, rb2, rb3).Equals(rb1), ShouldEqual, true)
		So(HeapOr(rb1, rb2, rb3).Equals(rb1), ShouldEqual, true)
		So(HeapOr(rb1, rb2, rb3).GetCardinality(), ShouldEqual, rb1.GetCardinality())
		So(HeapXor(rb1, rb2, rb3).Equals(bigxor), ShouldEqual, true)
		So(FastAnd(rb1, rb2, rb3).Equals(bigand), ShouldEqual, true)
	})
}

func TestFastAggregationsAdvanced_run(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()
		for i := uint32(500); i < 75000; i++ {
			rb1.Add(i)
		}
		for i := uint32(0); i < 1000000; i += 7 {
			rb2.Add(i)
		}
		for i := uint32(0); i < 1000000; i += 1001 {
			rb3.Add(i)
		}
		for i := uint32(1000000); i < 2000000; i += 1001 {
			rb1.Add(i)
		}
		for i := uint32(1000000); i < 2000000; i += 3 {
			rb2.Add(i)
		}
		for i := uint32(1000000); i < 2000000; i += 7 {
			rb3.Add(i)
		}
		rb1.RunOptimize()
		rb1.Or(rb2)
		rb1.Or(rb3)
		bigand := And(And(rb1, rb2), rb3)
		bigxor := Xor(Xor(rb1, rb2), rb3)
		So(FastOr(rb1, rb2, rb3).Equals(rb1), ShouldEqual, true)
		So(HeapOr(rb1, rb2, rb3).Equals(rb1), ShouldEqual, true)
		So(HeapOr(rb1, rb2, rb3).GetCardinality(), ShouldEqual, rb1.GetCardinality())
		So(HeapXor(rb1, rb2, rb3).Equals(bigxor), ShouldEqual, true)
		So(FastAnd(rb1, rb2, rb3).Equals(bigand), ShouldEqual, true)
	})
}

func TestFastAggregationsXOR(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()

		for i := uint32(0); i < 40000; i++ {
			rb1.Add(i)
		}
		for i := uint32(0); i < 40000; i += 4000 {
			rb2.Add(i)
		}
		for i := uint32(0); i < 40000; i += 5000 {
			rb3.Add(i)
		}
		So(rb1.GetCardinality() == 40000, ShouldEqual, true)

		xor1 := Xor(rb1, rb2)
		xor1alt := Xor(rb2, rb1)
		So(xor1alt.Equals(xor1), ShouldEqual, true)
		So(HeapXor(rb1, rb2).Equals(xor1), ShouldEqual, true)

		xor2 := Xor(rb2, rb3)
		xor2alt := Xor(rb3, rb2)
		So(xor2alt.Equals(xor2), ShouldEqual, true)
		So(HeapXor(rb2, rb3).Equals(xor2), ShouldEqual, true)

		bigxor := Xor(Xor(rb1, rb2), rb3)
		bigxoralt1 := Xor(rb1, Xor(rb2, rb3))
		bigxoralt2 := Xor(rb1, Xor(rb3, rb2))
		bigxoralt3 := Xor(rb3, Xor(rb1, rb2))
		bigxoralt4 := Xor(Xor(rb1, rb2), rb3)

		So(bigxoralt2.Equals(bigxor), ShouldEqual, true)
		So(bigxoralt1.Equals(bigxor), ShouldEqual, true)
		So(bigxoralt3.Equals(bigxor), ShouldEqual, true)
		So(bigxoralt4.Equals(bigxor), ShouldEqual, true)

		So(HeapXor(rb1, rb2, rb3).Equals(bigxor), ShouldEqual, true)
	})
}

func TestFastAggregationsXOR_run(t *testing.T) {
	Convey("Fast", t, func() {
		rb1 := NewBitmap()
		rb2 := NewBitmap()
		rb3 := NewBitmap()

		for i := uint32(0); i < 40000; i++ {
			rb1.Add(i)
		}
		rb1.RunOptimize()
		for i := uint32(0); i < 40000; i += 4000 {
			rb2.Add(i)
		}
		for i := uint32(0); i < 40000; i += 5000 {
			rb3.Add(i)
		}
		So(rb1.GetCardinality() == 40000, ShouldEqual, true)

		xor1 := Xor(rb1, rb2)
		xor1alt := Xor(rb2, rb1)
		So(xor1alt.Equals(xor1), ShouldEqual, true)
		So(HeapXor(rb1, rb2).Equals(xor1), ShouldEqual, true)

		xor2 := Xor(rb2, rb3)
		xor2alt := Xor(rb3, rb2)
		So(xor2alt.Equals(xor2), ShouldEqual, true)
		So(HeapXor(rb2, rb3).Equals(xor2), ShouldEqual, true)

		bigxor := Xor(Xor(rb1, rb2), rb3)
		bigxoralt1 := Xor(rb1, Xor(rb2, rb3))
		bigxoralt2 := Xor(rb1, Xor(rb3, rb2))
		bigxoralt3 := Xor(rb3, Xor(rb1, rb2))
		bigxoralt4 := Xor(Xor(rb1, rb2), rb3)

		So(bigxoralt2.Equals(bigxor), ShouldEqual, true)
		So(bigxoralt1.Equals(bigxor), ShouldEqual, true)
		So(bigxoralt3.Equals(bigxor), ShouldEqual, true)
		So(bigxoralt4.Equals(bigxor), ShouldEqual, true)

		So(HeapXor(rb1, rb2, rb3).Equals(bigxor), ShouldEqual, true)
	})
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"log/syslog"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"syscall"

	"github.com/mathpl/logmetrics"
)

var configFile = flag.String("c", "/etc/logmetrics_collector.conf", "Full path to config file.")
var threads = flag.Int("j", 1, "Thread count.")
var logToConsole = flag.Bool("d", false, "Print to console.")
var doNotSend = flag.Bool("D", false, "Print data instead of sending over network.")
var profile = flag.String("P", "", "Create a pprof file with this filename.")

func main() {
	//Process execution flags
	flag.Parse()

	var pf *os.File
	if *profile != "" {
		var err error
		pf, err = os.Create(*profile)
		if err != nil {
			log.Fatal(err)
		}

		log.Print("Starting profiler")
		pprof.StartCPUProfile(pf)
	}

	//Channel to stop the program
	stop := make(chan bool)

	//Signal handling
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)
	go func() {
		s := <-sigc
		log.Printf("Received signal: %s", s)
		stop <- true
	}()

	//Set the number of real threads to start
	runtime.GOMAXPROCS(*threads)

	//Config
	config := logmetrics.LoadConfig(*configFile)

	//Logger
	logger, err := syslog.New(config.GetSyslogFacility(), "logmetrics_collector")
	if err != nil {
		log.Fatal(err)
	}
	defer logger.Close()

	if !*logToConsole {
		log.SetOutput(logger)
	} else {
		log.SetFlags(0)
	}

	//Start the out channels
	tsd_pushers := make([]chan []string, config.GetPusherNumber())
	for i := 0; i < config.GetPusherNumber(); i++ {
		tsd_pushers[i] = make(chan []string, 1000)
	}

	//Start log tails
	fps := logmetrics.StartTails(&config, tsd_pushers)

	//Start datapools
	dps := logmetrics.StartDataPools(&config, tsd_pushers)

	//Start TSD pusher
	ps := logmetrics.StartTsdPushers(&config, tsd_pushers, *doNotSend)

	//Block until we're told to stop
	<-stop

	log.Print("Stopping all goroutines...")

	//Stop file checkers
	for _, fp := range fps {
		fp.Bye <- true
	}

	//Stop data pools
	for _, dp := range dps {
		dp.Bye <- true
	}

	//Stop tsd pushers
	for _, ps := range ps {
		ps.Bye <- true
	}

	if *profile != "" {
		pprof.StopCPUProfile()
		pf.Close()
		log.Print("Stopped profiler")
	}

	log.Print("All stopped")
}
<commit_msg>Add a little sleep to give the process a change of stopping.<commit_after>package main

import (
	"flag"
	"log"
	"log/syslog"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"syscall"
	"time"

	"github.com/mathpl/logmetrics"
)

var configFile = flag.String("c", "/etc/logmetrics_collector.conf", "Full path to config file.")
var threads = flag.Int("j", 1, "Thread count.")
var logToConsole = flag.Bool("d", false, "Print to console.")
var doNotSend = flag.Bool("D", false, "Print data instead of sending over network.")
var profile = flag.String("P", "", "Create a pprof file with this filename.")

func main() {
	//Process execution flags
	flag.Parse()

	var pf *os.File
	if *profile != "" {
		var err error
		pf, err = os.Create(*profile)
		if err != nil {
			log.Fatal(err)
		}

		log.Print("Starting profiler")
		pprof.StartCPUProfile(pf)
	}

	//Channel to stop the program
	stop := make(chan bool)

	//Signal handling
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)
	go func() {
		s := <-sigc
		log.Printf("Received signal: %s", s)
		stop <- true
	}()

	//Set the number of real threads to start
	runtime.GOMAXPROCS(*threads)

	//Config
	config := logmetrics.LoadConfig(*configFile)

	//Logger
	logger, err := syslog.New(config.GetSyslogFacility(), "logmetrics_collector")
	if err != nil {
		log.Fatal(err)
	}
	defer logger.Close()

	if !*logToConsole {
		log.SetOutput(logger)
	} else {
		log.SetFlags(0)
	}

	//Start the out channels
	tsd_pushers := make([]chan []string, config.GetPusherNumber())
	for i := 0; i < config.GetPusherNumber(); i++ {
		tsd_pushers[i] = make(chan []string, 1000)
	}

	//Start log tails
	fps := logmetrics.StartTails(&config, tsd_pushers)

	//Start datapools
	dps := logmetrics.StartDataPools(&config, tsd_pushers)

	//Start TSD pusher
	ps := logmetrics.StartTsdPushers(&config, tsd_pushers, *doNotSend)

	//Block until we're told to stop
	<-stop

	log.Print("Stopping all goroutines...")

	//Stop file checkers
	for _, fp := range fps {
		fp.Bye <- true
	}

	//Stop data pools
	for _, dp := range dps {
		dp.Bye <- true
	}

	//Stop tsd pushers
	for _, ps := range ps {
		ps.Bye <- true
	}

	if *profile != "" {
		pprof.StopCPUProfile()
		pf.Close()
		log.Print("Stopped profiler")
		time.Sleep(time.Duration(10 * time.Second))
	}

	log.Print("All stopped")
}
<|endoftext|>


<commit_before>package captcha

import (
	"captcha/cv"
	"log"
	"sort"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

type Mask struct {
	Img   *cv.BinaryImage
	Label string
}

type MaskIndex struct {
	masks []*Mask
	index map[int][]int
}

func NewMaskIndex() *MaskIndex {
	ret := MaskIndex{}
	ret.masks = []*Mask{}
	ret.index = make(map[int][]int)
	return &ret
}

func (self *MaskIndex) Clear() {
	self.masks = []*Mask{}
	self.index = make(map[int][]int)
}

func (self *Mask) MatchChType(chType int) bool {
	if chType == MIX {
		return true
	} else if chType == NUMBER {
		return self.Label >= "0" && self.Label <= "9"
	} else if chType == ALPHA {
		return self.Label >= "a" && self.Label <= "z"
	}
	return false
}

func (self *MaskIndex) Masks() []*Mask {
	return self.masks
}

func (self *MaskIndex) AddMask(mk *Mask) {
	self.masks = append(self.masks, mk)
	h := mk.Img.FeatureEncode()
	_, ok := self.index[h]
	if !ok {
		self.index[h] = []int{}
	}
	self.index[h] = append(self.index[h], len(self.masks)-1)
}

func (self *MaskIndex) QueryMask(img *cv.BinaryImage) ([]int, bool) {
	h := img.FeatureEncode()
	ret, ok := self.index[h]
	return ret, ok
}

func (self *MaskIndex) FindBestMatchedMasks(img *cv.BinaryImage, chType int, widthWeight int) []*Result {
	h := img.FeatureEncode()
	ids, ok := self.index[h]
	if !ok {
		return nil
	}

	results := ResultSorter{}
	for _, i := range ids {
		mk := self.masks[i]
		if !mk.MatchChType(chType) {
			continue
		}
		sim := mk.Match(img, 0, 0)
		sim *= float64(img.Width)
		sim /= float64(img.Width + widthWeight)
		results = append(results, NewResult(mk.Label, sim))
	}
	sort.Sort(results)
	return results
}

func (self *Mask) Match(src *cv.BinaryImage, srcLeft int, srcTop int) float64 {
	ret := 0.0
	for x := srcLeft; x < srcLeft+self.Img.Width && x < src.Width; x++ {
		for y := srcTop; y < srcTop+self.Img.Height && y < src.Height; y++ {
			if src.IsOpen(x, y) && self.Img.IsOpen(x-srcLeft, y-srcTop) {
				ret += 1.0
			}
		}
	}
	ret *= 2.0
	ret /= float64(self.Img.FrontSize() + src.FrontSize() + 1)
	return ret
}

func (self *Mask) ToString() string {
	ret := self.Label
	ret += "\t"
	ret += strconv.Itoa(self.Img.Width)
	ret += "\t"
	ret += strconv.Itoa(self.Img.Height)
	for x := 0; x < self.Img.Width; x++{
		for y := 0; y < self.Img.Height; y++{
			if !self.Img.IsOpen(x, y) {
				continue
			}
			ret += "\t"
			ret += strconv.Itoa(x)
			ret += ":"
			ret += strconv.Itoa(y)
		}
	}
	return ret
}

func (self *Mask) FromString(line string) {
	tks := strings.Split(line, "\t")
	self.Label = tks[0]
	width, _ := strconv.Atoi(tks[1])
	height, _ := strconv.Atoi(tks[2])
	self.Img = cv.NewBinaryImage(width, height)
	for i := 3; i < len(tks); i++ {
		kv := strings.Split(tks[i], ":")
		x, _ := strconv.Atoi(kv[0])
		y, _ := strconv.Atoi(kv[1])
		self.Img.Open(x, y)
	}
}

func NewMask(line string) *Mask {
	ret := Mask{}
	ret.FromString(line)
	return &ret
}

func LoadMasks(folder string) *MaskIndex {
	scaler := cv.ScaleBinaryImage{Height : SCALE_HEIGHT}
	bounder := cv.BoundBinaryImage{}
	ret := NewMaskIndex()
	n := 0
	err := filepath.Walk(folder, func(root string, f os.FileInfo, err error) error {
		if f == nil {
			return err
		}
		if f.IsDir() {
			label := f.Name()
			if len(label) > 2{
				return nil
			}
			log.Println(label, root)
			filepath.Walk(root, func(root2 string, f2 os.FileInfo, err2 error) error {
				if f2 == nil {
					return err
				}
				if f2.IsDir() {
					return nil
				}
				img := LoadImage(root2)
				if img != nil {
					mk := Mask{}
					mk.Img = scaler.Process(bounder.Process(cv.Convert2BinaryImage(img)))
					mk.Label = label
					ret.AddMask(&mk)
					n += 1
				}
				return nil
			})
			return nil
		}
		return nil
	})
	if err != nil {
		panic(err)
	}
	log.Println("mask count", n)
	return ret
}
<commit_msg>do not use # folder<commit_after>package captcha

import (
	"captcha/cv"
	"log"
	"sort"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

type Mask struct {
	Img   *cv.BinaryImage
	Label string
}

type MaskIndex struct {
	masks []*Mask
	index map[int][]int
}

func NewMaskIndex() *MaskIndex {
	ret := MaskIndex{}
	ret.masks = []*Mask{}
	ret.index = make(map[int][]int)
	return &ret
}

func (self *MaskIndex) Clear() {
	self.masks = []*Mask{}
	self.index = make(map[int][]int)
}

func (self *Mask) MatchChType(chType int) bool {
	if chType == MIX {
		return true
	} else if chType == NUMBER {
		return self.Label >= "0" && self.Label <= "9"
	} else if chType == ALPHA {
		return self.Label >= "a" && self.Label <= "z"
	}
	return false
}

func (self *MaskIndex) Masks() []*Mask {
	return self.masks
}

func (self *MaskIndex) AddMask(mk *Mask) {
	self.masks = append(self.masks, mk)
	h := mk.Img.FeatureEncode()
	_, ok := self.index[h]
	if !ok {
		self.index[h] = []int{}
	}
	self.index[h] = append(self.index[h], len(self.masks)-1)
}

func (self *MaskIndex) QueryMask(img *cv.BinaryImage) ([]int, bool) {
	h := img.FeatureEncode()
	ret, ok := self.index[h]
	return ret, ok
}

func (self *MaskIndex) FindBestMatchedMasks(img *cv.BinaryImage, chType int, widthWeight int) []*Result {
	h := img.FeatureEncode()
	ids, ok := self.index[h]
	if !ok {
		return nil
	}

	results := ResultSorter{}
	for _, i := range ids {
		mk := self.masks[i]
		if !mk.MatchChType(chType) {
			continue
		}
		sim := mk.Match(img, 0, 0)
		sim *= float64(img.Width)
		sim /= float64(img.Width + widthWeight)
		results = append(results, NewResult(mk.Label, sim))
	}
	sort.Sort(results)
	return results
}

func (self *Mask) Match(src *cv.BinaryImage, srcLeft int, srcTop int) float64 {
	ret := 0.0
	for x := srcLeft; x < srcLeft+self.Img.Width && x < src.Width; x++ {
		for y := srcTop; y < srcTop+self.Img.Height && y < src.Height; y++ {
			if src.IsOpen(x, y) && self.Img.IsOpen(x-srcLeft, y-srcTop) {
				ret += 1.0
			}
		}
	}
	ret *= 2.0
	ret /= float64(self.Img.FrontSize() + src.FrontSize() + 1)
	return ret
}

func (self *Mask) ToString() string {
	ret := self.Label
	ret += "\t"
	ret += strconv.Itoa(self.Img.Width)
	ret += "\t"
	ret += strconv.Itoa(self.Img.Height)
	for x := 0; x < self.Img.Width; x++{
		for y := 0; y < self.Img.Height; y++{
			if !self.Img.IsOpen(x, y) {
				continue
			}
			ret += "\t"
			ret += strconv.Itoa(x)
			ret += ":"
			ret += strconv.Itoa(y)
		}
	}
	return ret
}

func (self *Mask) FromString(line string) {
	tks := strings.Split(line, "\t")
	self.Label = tks[0]
	width, _ := strconv.Atoi(tks[1])
	height, _ := strconv.Atoi(tks[2])
	self.Img = cv.NewBinaryImage(width, height)
	for i := 3; i < len(tks); i++ {
		kv := strings.Split(tks[i], ":")
		x, _ := strconv.Atoi(kv[0])
		y, _ := strconv.Atoi(kv[1])
		self.Img.Open(x, y)
	}
}

func NewMask(line string) *Mask {
	ret := Mask{}
	ret.FromString(line)
	return &ret
}

func LoadMasks(folder string) *MaskIndex {
	scaler := cv.ScaleBinaryImage{Height : SCALE_HEIGHT}
	bounder := cv.BoundBinaryImage{}
	ret := NewMaskIndex()
	n := 0
	err := filepath.Walk(folder, func(root string, f os.FileInfo, err error) error {
		if f == nil {
			return err
		}
		if f.IsDir() {
			label := f.Name()
			if len(label) > 2 || label == "#"{
				return nil
			}
			log.Println(label, root)
			filepath.Walk(root, func(root2 string, f2 os.FileInfo, err2 error) error {
				if f2 == nil {
					return err
				}
				if f2.IsDir() {
					return nil
				}
				img := LoadImage(root2)
				if img != nil {
					mk := Mask{}
					mk.Img = scaler.Process(bounder.Process(cv.Convert2BinaryImage(img)))
					mk.Label = label
					ret.AddMask(&mk)
					n += 1
				}
				return nil
			})
			return nil
		}
		return nil
	})
	if err != nil {
		panic(err)
	}
	log.Println("mask count", n)
	return ret
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"regexp"

	"github.com/cep21/xdgbasedir"
	"gopkg.in/yaml.v1"
)

func main() {
	lf := NewLocalFiles("")
	Main(os.Args, os.Stdout, os.Stderr, &lf)
}

// The programs usage help.
var Usage = `dwi.

Usage:
	dwi <file>
`

// FileIndex provides methods for manipulating a list of files.
type FileIndex interface {
	Exists(filename string) bool
	Open(filename string) (*os.File, error)
}

// LocalFiles is a FileIndex for files in the local filesystem.
type LocalFiles struct {
	Root string
}

// Exists returns true if a file given by path exists.
func (lf *LocalFiles) Exists(filename string) bool {
	if _, err := os.Stat(lf.getFullPath(filename)); err != nil {
		return false
	}
	return true
}

// Open opens a file or returns an error.
func (lf *LocalFiles) Open(filename string) (*os.File, error) {
	return os.Open(filename)
}

// GetFullPath returns the full path.
func (lf *LocalFiles) getFullPath(filename string) string {
	return path.Join(lf.Root, filename)
}

// NewLocalFiles creates and initializes a new LocalFiles with a given path root.
func NewLocalFiles(root string) LocalFiles {
	return LocalFiles{root}
}

// Main function.
func Main(args []string, out io.Writer, eout io.Writer, fi FileIndex) {
	if len(args) < 2 {
		fmt.Fprintln(out, Usage)
		return
	}

	files := ParseArgs(args)
	if AnyFilesDontExist(eout, fi, files) {
		return
	}

	configDir, err := xdgbasedir.ConfigHomeDirectory()
	if err != nil {
		return
	}

	rules := LoadRules(path.Join(configDir, "/dealwithit/rules.yaml"), fi, eout)

	ForEachMatchingFile(files, rules, eout, fi)
}

// AnyFilesDontExist will look for each of the given files in the given FileIndex,
// output an error to eout for any file that doesn't exist and return true if where any missing.
func AnyFilesDontExist(eout io.Writer, fi FileIndex, files []string) bool {
	err := false
	for _, f := range files {
		if !fi.Exists(f) {
			fmt.Fprintln(eout, "Error: File not found:", f)
			err = true
		}
	}
	return err
}

// ParseArgs takes a list of args and returns the list of files.
// The first arg must be the executable name.
func ParseArgs(args []string) []string {
	return args[1:]
}

type Rule map[string]string
type Rules []Rule

// LoadRules loads the rules file given by filename from the given FileIndex.
func LoadRules(filename string, fi FileIndex, eout io.Writer) Rules {
	var rules Rules

	file, err := fi.Open(filename)
	if err != nil {
		fmt.Fprintln(eout, "Error: Could not open rules file", filename, err)
		return rules
	}

	data, err := ioutil.ReadAll(file)
	if err != nil {
		fmt.Fprintln(eout, "Error: Could not read rules file", filename, err)
		return rules
	}

	err = yaml.Unmarshal(data, &rules)
	if err != nil {
		fmt.Fprintln(eout, "Error: Could not parse rules file", filename, err)
		return rules
	}

	return rules
}

// ForEachMatchingFile executes a function on each file if the function match returns true.
func ForEachMatchingFile(files []string, rules Rules, eout io.Writer, fi FileIndex) {
	for _, f := range files {
		matches, rule := FileMatchesRules(f, rules, eout, fi)

		if !matches {
			continue
		}

		ExecuteRule(f, rule, fi)

	}
}

// FileMatchesRules if filename matches a rule given in rules returns true and the rule.
func FileMatchesRules(filename string, rules Rules, eout io.Writer, fi FileIndex) (bool, Rule) {
	for _, rule := range rules {
		if FileMatchesRule(filename, rule, eout) {
			return true, rule
		}
	}
	return false, nil
}

// FileMatchesRule return true if filename matches the rule given.
func FileMatchesRule(filename string, rule Rule, eout io.Writer) bool {
	re, err := regexp.Compile(rule["file"])
	if err != nil {
		fmt.Fprintln(eout, "Error: Could not compile regexp", err)
		return false
	}
	return re.MatchString(filename)
}

// ExecuteRule executes the given rules on file.
func ExecuteRule(filename string, rule Rule, fi FileIndex) {
	fmt.Println("DEBUG: ", filename, "matches", rule)
}
<commit_msg>Split loading of rules file and parsing of it.<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"regexp"

	"github.com/cep21/xdgbasedir"
	"gopkg.in/yaml.v1"
)

func main() {
	lf := NewLocalFiles("")
	Main(os.Args, os.Stdout, os.Stderr, &lf)
}

// The programs usage help.
var Usage = `dwi.

Usage:
	dwi <file>
`

// FileIndex provides methods for manipulating a list of files.
type FileIndex interface {
	Exists(filename string) bool
	Open(filename string) (*os.File, error)
}

// LocalFiles is a FileIndex for files in the local filesystem.
type LocalFiles struct {
	Root string
}

// Exists returns true if a file given by path exists.
func (lf *LocalFiles) Exists(filename string) bool {
	if _, err := os.Stat(lf.getFullPath(filename)); err != nil {
		return false
	}
	return true
}

// Open opens a file or returns an error.
func (lf *LocalFiles) Open(filename string) (*os.File, error) {
	return os.Open(filename)
}

// GetFullPath returns the full path.
func (lf *LocalFiles) getFullPath(filename string) string {
	return path.Join(lf.Root, filename)
}

// NewLocalFiles creates and initializes a new LocalFiles with a given path root.
func NewLocalFiles(root string) LocalFiles {
	return LocalFiles{root}
}

// Main function.
func Main(args []string, out io.Writer, eout io.Writer, fi FileIndex) {
	if len(args) < 2 {
		fmt.Fprintln(out, Usage)
		return
	}

	files := ParseArgs(args)
	if AnyFilesDontExist(eout, fi, files) {
		return
	}

	configDir, err := xdgbasedir.ConfigHomeDirectory()
	if err != nil {
		return
	}

	rules := LoadRules(path.Join(configDir, "/dealwithit/rules.yaml"), fi, eout)

	ForEachMatchingFile(files, rules, eout, fi)
}

// AnyFilesDontExist will look for each of the given files in the given FileIndex,
// output an error to eout for any file that doesn't exist and return true if where any missing.
func AnyFilesDontExist(eout io.Writer, fi FileIndex, files []string) bool {
	err := false
	for _, f := range files {
		if !fi.Exists(f) {
			fmt.Fprintln(eout, "Error: File not found:", f)
			err = true
		}
	}
	return err
}

// ParseArgs takes a list of args and returns the list of files.
// The first arg must be the executable name.
func ParseArgs(args []string) []string {
	return args[1:]
}

type Rule map[string]string
type Rules []Rule

// LoadRules loads the rules file given by filename from the given FileIndex.
func LoadRules(filename string, fi FileIndex, eout io.Writer) Rules {
	var rules Rules

	file, err := fi.Open(filename)
	if err != nil {
		fmt.Fprintln(eout, "Error: Could not open rules file", filename, err)
		return rules
	}

	data, err := ioutil.ReadAll(file)
	if err != nil {
		fmt.Fprintln(eout, "Error: Could not read rules file", filename, err)
		return rules
	}

	rules = ParseRules(data, eout)
	return rules
}

// ParseRules parses the YAML.
func ParseRules(data []byte, eout io.Writer) Rules {
	var rules Rules
	err := yaml.Unmarshal(data, &rules)
	if err != nil {
		fmt.Fprintln(eout, "Error: Could not parse rules file", err)
		return rules
	}

	return rules
}

// ForEachMatchingFile executes a function on each file if the function match returns true.
func ForEachMatchingFile(files []string, rules Rules, eout io.Writer, fi FileIndex) {
	for _, f := range files {
		matches, rule := FileMatchesRules(f, rules, eout, fi)

		if !matches {
			continue
		}

		ExecuteRule(f, rule, fi)

	}
}

// FileMatchesRules if filename matches a rule given in rules returns true and the rule.
func FileMatchesRules(filename string, rules Rules, eout io.Writer, fi FileIndex) (bool, Rule) {
	for _, rule := range rules {
		if FileMatchesRule(filename, rule, eout) {
			return true, rule
		}
	}
	return false, nil
}

// FileMatchesRule return true if filename matches the rule given.
func FileMatchesRule(filename string, rule Rule, eout io.Writer) bool {
	re, err := regexp.Compile(rule["file"])
	if err != nil {
		fmt.Fprintln(eout, "Error: Could not compile regexp", err)
		return false
	}
	return re.MatchString(filename)
}

// ExecuteRule executes the given rules on file.
func ExecuteRule(filename string, rule Rule, fi FileIndex) {
	fmt.Println("DEBUG: ", filename, "matches", rule)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"regexp"
	"sort"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
)

// Ec2Instances is list of EC2 instance.
type Ec2Instances []*ec2.Instance

func (instance Ec2Instances) Len() int {
	return len(instance)
}

func (instance Ec2Instances) Swap(i, j int) {
	instance[i], instance[j] = instance[j], instance[i]
}

func (instance Ec2Instances) Less(i, j int) bool {
	return GetTagValue(instance[i], "Name") < GetTagValue(instance[j], "Name")
}

// ParseFilter parse filter option.
func ParseFilter(filters string) []*ec2.Filter {

	// filters e.g. "Name=tag:Foo,Values=Bar Name=instance-type,Values=m1.small"
	var ec2Filter []*ec2.Filter

	reSpace := regexp.MustCompile(`\s+`)
	reName := regexp.MustCompile(`Name=`)
	reValue := regexp.MustCompile(`,Values=`)
	for _, i := range reSpace.Split(filters, -1) {
		for _, j := range reName.Split(i, -1) {
			if len(j) != 0 {
				v := reValue.Split(j, -1)
				name := v[0]
				ec2Filter = append(ec2Filter, &ec2.Filter{
					Name: aws.String(name),
					Values: []*string{
						aws.String(v[1]),
					},
				})
			}
		}
	}
	return ec2Filter
}

func generateSession(region, profile string) (*session.Session, error) {

	sessOpt := session.Options{
		SharedConfigState: session.SharedConfigEnable,
	}
	if len(region) != 0 {
		sessOpt.Config = aws.Config{Region: aws.String(region)}
	}
	if len(profile) != 0 {
		sessOpt.Profile = profile
	}
	return session.NewSessionWithOptions(sessOpt)
}

// DescribeInstances describe one or more of your instances.
func DescribeInstances(region, profile, filters string) (Ec2Instances, error) {

	var instances Ec2Instances
	sess, err := generateSession(region, profile)
	if err != nil {
		return nil, err
	}
	svc := ec2.New(sess)

	params := &ec2.DescribeInstancesInput{}
	if len(filters) != 0 {
		params = &ec2.DescribeInstancesInput{
			Filters: ParseFilter(filters),
		}
	}
	resp, err := svc.DescribeInstances(params)
	if err != nil {
		return nil, err
	}
	if len(resp.Reservations) == 0 {
		return Ec2Instances{}, nil
	}
	for _, res := range resp.Reservations {
		for _, instance := range res.Instances {
			instances = append(instances, instance)
		}
	}
	sort.Sort(instances)
	return instances, nil
}

// PrintInstances is output stdout.
func PrintInstances(instances Ec2Instances) {
	for _, instance := range instances {
		fmt.Printf("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
			GetTagValue(instance, "Name"),
			GetPrivateIPAddress(instance),
			GetPublicIPAddress(instance),
			*instance.InstanceId,
			*instance.InstanceType,
			*instance.Placement.AvailabilityZone,
			*instance.State.Name,
			GetPlatform(instance),
		)
	}
}

// GetTagValue returns values of EC2 tag.
func GetTagValue(instance *ec2.Instance, tagName string) string {
	for _, t := range instance.Tags {
		if *t.Key == tagName {
			return *t.Value
		}
	}
	return ""
}

// GetPrivateIPAddress returns value of EC2 private ip address, if there is a value.
func GetPrivateIPAddress(instance *ec2.Instance) string {
	if instance.PrivateIpAddress != nil {
		return *instance.PrivateIpAddress
	}
	return ""
}

// GetPublicIPAddress returns value of EC2 public ip address, if there is a value.
func GetPublicIPAddress(instance *ec2.Instance) string {
	if instance.PublicIpAddress != nil {
		return *instance.PublicIpAddress
	}
	return ""
}

// GetPlatform returns platform name(linux or windows).
func GetPlatform(instance *ec2.Instance) string {
	if instance.Platform != nil {
		return *instance.Platform
	}
	return "linux"
}
<commit_msg>Use the strings package instead of regexp for performance improvement.<commit_after>package main

import (
	"fmt"
	"regexp"
	"sort"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
)

// Ec2Instances is list of EC2 instance.
type Ec2Instances []*ec2.Instance

func (instance Ec2Instances) Len() int {
	return len(instance)
}

func (instance Ec2Instances) Swap(i, j int) {
	instance[i], instance[j] = instance[j], instance[i]
}

func (instance Ec2Instances) Less(i, j int) bool {
	return GetTagValue(instance[i], "Name") < GetTagValue(instance[j], "Name")
}

// ParseFilter parse filter option.
func ParseFilter(filters string) []*ec2.Filter {

	// filters e.g. "Name=tag:Foo,Values=Bar Name=instance-type,Values=m1.small"
	var ec2Filter []*ec2.Filter

	re := regexp.MustCompile(`Name=(.+),Values=(.+)`)
	for _, i := range strings.Fields(filters) {
		matches := re.FindAllStringSubmatch(i, -1)
		ec2Filter = append(ec2Filter, &ec2.Filter{
			Name: aws.String(matches[0][1]),
			Values: []*string{
				aws.String(matches[0][2]),
			},
		})
	}
	return ec2Filter
}

func generateSession(region, profile string) (*session.Session, error) {

	sessOpt := session.Options{
		SharedConfigState: session.SharedConfigEnable,
	}
	if len(region) != 0 {
		sessOpt.Config = aws.Config{Region: aws.String(region)}
	}
	if len(profile) != 0 {
		sessOpt.Profile = profile
	}
	return session.NewSessionWithOptions(sessOpt)
}

// DescribeInstances describe one or more of your instances.
func DescribeInstances(region, profile, filters string) (Ec2Instances, error) {

	var instances Ec2Instances
	sess, err := generateSession(region, profile)
	if err != nil {
		return nil, err
	}
	svc := ec2.New(sess)

	params := &ec2.DescribeInstancesInput{}
	if len(filters) != 0 {
		params = &ec2.DescribeInstancesInput{
			Filters: ParseFilter(filters),
		}
	}
	resp, err := svc.DescribeInstances(params)
	if err != nil {
		return nil, err
	}
	if len(resp.Reservations) == 0 {
		return Ec2Instances{}, nil
	}
	for _, res := range resp.Reservations {
		for _, instance := range res.Instances {
			instances = append(instances, instance)
		}
	}
	sort.Sort(instances)
	return instances, nil
}

// PrintInstances is output stdout.
func PrintInstances(instances Ec2Instances) {
	for _, instance := range instances {
		fmt.Printf("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
			GetTagValue(instance, "Name"),
			GetPrivateIPAddress(instance),
			GetPublicIPAddress(instance),
			*instance.InstanceId,
			*instance.InstanceType,
			*instance.Placement.AvailabilityZone,
			*instance.State.Name,
			GetPlatform(instance),
		)
	}
}

// GetTagValue returns values of EC2 tag.
func GetTagValue(instance *ec2.Instance, tagName string) string {
	for _, t := range instance.Tags {
		if *t.Key == tagName {
			return *t.Value
		}
	}
	return ""
}

// GetPrivateIPAddress returns value of EC2 private ip address, if there is a value.
func GetPrivateIPAddress(instance *ec2.Instance) string {
	if instance.PrivateIpAddress != nil {
		return *instance.PrivateIpAddress
	}
	return ""
}

// GetPublicIPAddress returns value of EC2 public ip address, if there is a value.
func GetPublicIPAddress(instance *ec2.Instance) string {
	if instance.PublicIpAddress != nil {
		return *instance.PublicIpAddress
	}
	return ""
}

// GetPlatform returns platform name(linux or windows).
func GetPlatform(instance *ec2.Instance) string {
	if instance.Platform != nil {
		return *instance.Platform
	}
	return "linux"
}
<|endoftext|>


<commit_before>package drivers

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/units"
)

func wipeDirectory(path string) error {
	// List all entries
	entries, err := ioutil.ReadDir(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
	}

	// Individually wipe all entries
	for _, entry := range entries {
		entryPath := filepath.Join(path, entry.Name())
		err := os.RemoveAll(entryPath)
		if err != nil {
			return err
		}
	}

	return nil
}

func forceUnmount(path string) (bool, error) {
	unmounted := false

	for {
		// Check if already unmounted
		if !shared.IsMountPoint(path) {
			return unmounted, nil
		}

		// Try a clean unmount first
		err := TryUnmount(path, 0)
		if err != nil {
			// Fallback to lazy unmounting
			err = unix.Unmount(path, unix.MNT_DETACH)
			if err != nil {
				return false, err
			}
		}

		unmounted = true
	}
}

func mountReadOnly(srcPath string, dstPath string) (bool, error) {
	// Check if already mounted.
	if shared.IsMountPoint(dstPath) {
		return false, nil
	}

	// Create a mount entry.
	err := TryMount(srcPath, dstPath, "none", unix.MS_BIND, "")
	if err != nil {
		return false, err
	}

	// Make it read-only.
	err = TryMount("", dstPath, "none", unix.MS_BIND|unix.MS_RDONLY|unix.MS_REMOUNT, "")
	if err != nil {
		forceUnmount(dstPath)
		return false, err
	}

	return true, nil
}

func sameMount(srcPath string, dstPath string) bool {
	// Get the source vfs path information
	var srcFsStat unix.Statfs_t
	err := unix.Statfs(srcPath, &srcFsStat)
	if err != nil {
		return false
	}

	// Get the destination vfs path information
	var dstFsStat unix.Statfs_t
	err = unix.Statfs(dstPath, &dstFsStat)
	if err != nil {
		return false
	}

	// Compare statfs
	if srcFsStat.Type != dstFsStat.Type || srcFsStat.Fsid != dstFsStat.Fsid {
		return false
	}

	// Get the source path information
	var srcStat unix.Stat_t
	err = unix.Stat(srcPath, &srcStat)
	if err != nil {
		return false
	}

	// Get the destination path information
	var dstStat unix.Stat_t
	err = unix.Stat(dstPath, &dstStat)
	if err != nil {
		return false
	}

	// Compare inode
	if srcStat.Ino != dstStat.Ino {
		return false
	}

	return true
}

// TryMount tries mounting a filesystem multiple times. This is useful for unreliable backends.
func TryMount(src string, dst string, fs string, flags uintptr, options string) error {
	var err error

	// Attempt 20 mounts over 10s
	for i := 0; i < 20; i++ {
		err = unix.Mount(src, dst, fs, flags, options)
		if err == nil {
			break
		}

		time.Sleep(500 * time.Millisecond)
	}

	if err != nil {
		return err
	}

	return nil
}

// TryUnmount tries unmounting a filesystem multiple times. This is useful for unreliable backends.
func TryUnmount(path string, flags int) error {
	var err error

	for i := 0; i < 20; i++ {
		err = unix.Unmount(path, flags)
		if err == nil {
			break
		}

		time.Sleep(500 * time.Millisecond)
	}

	if err != nil {
		return err
	}

	return nil
}

func tryExists(path string) bool {
	// Attempt 20 checks over 10s
	for i := 0; i < 20; i++ {
		if shared.PathExists(path) {
			return true
		}

		time.Sleep(500 * time.Millisecond)
	}

	return false
}

func fsUUID(path string) (string, error) {
	return shared.RunCommand("blkid", "-s", "UUID", "-o", "value", path)
}

func hasFilesystem(path string, fsType int64) bool {
	fs := unix.Statfs_t{}

	err := unix.Statfs(path, &fs)
	if err != nil {
		return false
	}

	if int64(fs.Type) != fsType {
		return false
	}

	return true
}

// GetPoolMountPath returns the mountpoint of the given pool.
// {LXD_DIR}/storage-pools/<pool>
func GetPoolMountPath(poolName string) string {
	return shared.VarPath("storage-pools", poolName)
}

// GetVolumeMountPath returns the mount path for a specific volume based on its pool and type and
// whether it is a snapshot or not. For VolumeTypeImage the volName is the image fingerprint.
func GetVolumeMountPath(poolName string, volType VolumeType, volName string) string {
	if shared.IsSnapshot(volName) {
		return shared.VarPath("storage-pools", poolName, fmt.Sprintf("%s-snapshots", string(volType)), volName)
	}

	return shared.VarPath("storage-pools", poolName, string(volType), volName)
}

// GetVolumeSnapshotDir gets the snapshot mount directory for the parent volume.
func GetVolumeSnapshotDir(poolName string, volType VolumeType, volName string) string {
	parent, _, _ := shared.InstanceGetParentAndSnapshotName(volName)
	return shared.VarPath("storage-pools", poolName, fmt.Sprintf("%s-snapshots", string(volType)), parent)
}

// GetSnapshotVolumeName returns the full volume name for a parent volume and snapshot name.
func GetSnapshotVolumeName(parentName, snapshotName string) string {
	return fmt.Sprintf("%s%s%s", parentName, shared.SnapshotDelimiter, snapshotName)
}

// createParentSnapshotDirIfMissing creates the parent directory for volume snapshots
func createParentSnapshotDirIfMissing(poolName string, volType VolumeType, volName string) error {
	snapshotsPath := GetVolumeSnapshotDir(poolName, volType, volName)

	// If it's missing, create it.
	if !shared.PathExists(snapshotsPath) {
		return os.Mkdir(snapshotsPath, 0700)
	}

	return nil
}

// deleteParentSnapshotDirIfEmpty removes the parent snapshot directory if it is empty.
// It accepts the pool name, volume type and parent volume name.
func deleteParentSnapshotDirIfEmpty(poolName string, volType VolumeType, volName string) error {
	snapshotsPath := GetVolumeSnapshotDir(poolName, volType, volName)

	// If it exists, try to delete it.
	if shared.PathExists(snapshotsPath) {
		isEmpty, err := shared.PathIsEmpty(snapshotsPath)
		if err != nil {
			return err
		}

		if isEmpty {
			err := os.Remove(snapshotsPath)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// createSparseFile creates a sparse empty file at specified location with specified size.
func createSparseFile(filePath string, sizeBytes int64) error {
	f, err := os.Create(filePath)
	if err != nil {
		return fmt.Errorf("Failed to open %s: %s", filePath, err)
	}
	defer f.Close()

	err = f.Chmod(0600)
	if err != nil {
		return fmt.Errorf("Failed to chmod %s: %s", filePath, err)
	}

	err = f.Truncate(sizeBytes)
	if err != nil {
		return fmt.Errorf("Failed to create sparse file %s: %s", filePath, err)
	}

	return nil
}

// ensureVolumeBlockFile creates or resizes the raw block file for a volume.
func ensureVolumeBlockFile(vol Volume, path string) error {
	blockSize := vol.ExpandedConfig("size")
	if blockSize == "" {
		blockSize = defaultBlockSize
	}

	blockSizeBytes, err := units.ParseByteSizeString(blockSize)
	if err != nil {
		return err
	}

	if shared.PathExists(path) {
		_, err = shared.RunCommand("qemu-img", "resize", "-f", "raw", path, fmt.Sprintf("%d", blockSizeBytes))
		if err != nil {
			return fmt.Errorf("Failed resizing disk image %s to size %s: %v", path, blockSize, err)
		}
	} else {
		// If path doesn't exist, then there has been no filler function
		// supplied to create it from another source. So instead create an empty
		// volume (use for PXE booting a VM).
		_, err = shared.RunCommand("qemu-img", "create", "-f", "raw", path, fmt.Sprintf("%d", blockSizeBytes))
		if err != nil {
			return fmt.Errorf("Failed creating disk image %s as size %s: %v", path, blockSize, err)
		}
	}

	return nil
}

// mkfsOptions represents options for filesystem creation.
type mkfsOptions struct {
	Label string
}

// makeFSType creates the provided filesystem.
func makeFSType(path string, fsType string, options *mkfsOptions) (string, error) {
	var err error
	var msg string

	fsOptions := options
	if fsOptions == nil {
		fsOptions = &mkfsOptions{}
	}

	cmd := []string{fmt.Sprintf("mkfs.%s", fsType), path}
	if fsOptions.Label != "" {
		cmd = append(cmd, "-L", fsOptions.Label)
	}

	if fsType == "ext4" {
		cmd = append(cmd, "-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0")
	}

	msg, err = shared.TryRunCommand(cmd[0], cmd[1:]...)
	if err != nil {
		return msg, err
	}

	return "", nil
}

// mountOption represents an individual mount option.
type mountOption struct {
	capture bool
	flag    uintptr
}

// mountOptions represents a list of possible mount options.
var mountOptions = map[string]mountOption{
	"async":         {false, unix.MS_SYNCHRONOUS},
	"atime":         {false, unix.MS_NOATIME},
	"bind":          {true, unix.MS_BIND},
	"defaults":      {true, 0},
	"dev":           {false, unix.MS_NODEV},
	"diratime":      {false, unix.MS_NODIRATIME},
	"dirsync":       {true, unix.MS_DIRSYNC},
	"exec":          {false, unix.MS_NOEXEC},
	"lazytime":      {true, unix.MS_LAZYTIME},
	"mand":          {true, unix.MS_MANDLOCK},
	"noatime":       {true, unix.MS_NOATIME},
	"nodev":         {true, unix.MS_NODEV},
	"nodiratime":    {true, unix.MS_NODIRATIME},
	"noexec":        {true, unix.MS_NOEXEC},
	"nomand":        {false, unix.MS_MANDLOCK},
	"norelatime":    {false, unix.MS_RELATIME},
	"nostrictatime": {false, unix.MS_STRICTATIME},
	"nosuid":        {true, unix.MS_NOSUID},
	"rbind":         {true, unix.MS_BIND | unix.MS_REC},
	"relatime":      {true, unix.MS_RELATIME},
	"remount":       {true, unix.MS_REMOUNT},
	"ro":            {true, unix.MS_RDONLY},
	"rw":            {false, unix.MS_RDONLY},
	"strictatime":   {true, unix.MS_STRICTATIME},
	"suid":          {false, unix.MS_NOSUID},
	"sync":          {true, unix.MS_SYNCHRONOUS},
}

// resolveMountOptions resolves the provided mount options.
func resolveMountOptions(options string) (uintptr, string) {
	mountFlags := uintptr(0)
	tmp := strings.SplitN(options, ",", -1)
	for i := 0; i < len(tmp); i++ {
		opt := tmp[i]
		do, ok := mountOptions[opt]
		if !ok {
			continue
		}

		if do.capture {
			mountFlags |= do.flag
		} else {
			mountFlags &= ^do.flag
		}

		copy(tmp[i:], tmp[i+1:])
		tmp[len(tmp)-1] = ""
		tmp = tmp[:len(tmp)-1]
		i--
	}

	return mountFlags, strings.Join(tmp, ",")
}
<commit_msg>lxd/storage/drivers/utils: Adds volume filesystem shrink and grow functions<commit_after>package drivers

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/units"
)

func wipeDirectory(path string) error {
	// List all entries
	entries, err := ioutil.ReadDir(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
	}

	// Individually wipe all entries
	for _, entry := range entries {
		entryPath := filepath.Join(path, entry.Name())
		err := os.RemoveAll(entryPath)
		if err != nil {
			return err
		}
	}

	return nil
}

func forceUnmount(path string) (bool, error) {
	unmounted := false

	for {
		// Check if already unmounted
		if !shared.IsMountPoint(path) {
			return unmounted, nil
		}

		// Try a clean unmount first
		err := TryUnmount(path, 0)
		if err != nil {
			// Fallback to lazy unmounting
			err = unix.Unmount(path, unix.MNT_DETACH)
			if err != nil {
				return false, err
			}
		}

		unmounted = true
	}
}

func mountReadOnly(srcPath string, dstPath string) (bool, error) {
	// Check if already mounted.
	if shared.IsMountPoint(dstPath) {
		return false, nil
	}

	// Create a mount entry.
	err := TryMount(srcPath, dstPath, "none", unix.MS_BIND, "")
	if err != nil {
		return false, err
	}

	// Make it read-only.
	err = TryMount("", dstPath, "none", unix.MS_BIND|unix.MS_RDONLY|unix.MS_REMOUNT, "")
	if err != nil {
		forceUnmount(dstPath)
		return false, err
	}

	return true, nil
}

func sameMount(srcPath string, dstPath string) bool {
	// Get the source vfs path information
	var srcFsStat unix.Statfs_t
	err := unix.Statfs(srcPath, &srcFsStat)
	if err != nil {
		return false
	}

	// Get the destination vfs path information
	var dstFsStat unix.Statfs_t
	err = unix.Statfs(dstPath, &dstFsStat)
	if err != nil {
		return false
	}

	// Compare statfs
	if srcFsStat.Type != dstFsStat.Type || srcFsStat.Fsid != dstFsStat.Fsid {
		return false
	}

	// Get the source path information
	var srcStat unix.Stat_t
	err = unix.Stat(srcPath, &srcStat)
	if err != nil {
		return false
	}

	// Get the destination path information
	var dstStat unix.Stat_t
	err = unix.Stat(dstPath, &dstStat)
	if err != nil {
		return false
	}

	// Compare inode
	if srcStat.Ino != dstStat.Ino {
		return false
	}

	return true
}

// TryMount tries mounting a filesystem multiple times. This is useful for unreliable backends.
func TryMount(src string, dst string, fs string, flags uintptr, options string) error {
	var err error

	// Attempt 20 mounts over 10s
	for i := 0; i < 20; i++ {
		err = unix.Mount(src, dst, fs, flags, options)
		if err == nil {
			break
		}

		time.Sleep(500 * time.Millisecond)
	}

	if err != nil {
		return err
	}

	return nil
}

// TryUnmount tries unmounting a filesystem multiple times. This is useful for unreliable backends.
func TryUnmount(path string, flags int) error {
	var err error

	for i := 0; i < 20; i++ {
		err = unix.Unmount(path, flags)
		if err == nil {
			break
		}

		time.Sleep(500 * time.Millisecond)
	}

	if err != nil {
		return err
	}

	return nil
}

func tryExists(path string) bool {
	// Attempt 20 checks over 10s
	for i := 0; i < 20; i++ {
		if shared.PathExists(path) {
			return true
		}

		time.Sleep(500 * time.Millisecond)
	}

	return false
}

func fsUUID(path string) (string, error) {
	return shared.RunCommand("blkid", "-s", "UUID", "-o", "value", path)
}

func hasFilesystem(path string, fsType int64) bool {
	fs := unix.Statfs_t{}

	err := unix.Statfs(path, &fs)
	if err != nil {
		return false
	}

	if int64(fs.Type) != fsType {
		return false
	}

	return true
}

// GetPoolMountPath returns the mountpoint of the given pool.
// {LXD_DIR}/storage-pools/<pool>
func GetPoolMountPath(poolName string) string {
	return shared.VarPath("storage-pools", poolName)
}

// GetVolumeMountPath returns the mount path for a specific volume based on its pool and type and
// whether it is a snapshot or not. For VolumeTypeImage the volName is the image fingerprint.
func GetVolumeMountPath(poolName string, volType VolumeType, volName string) string {
	if shared.IsSnapshot(volName) {
		return shared.VarPath("storage-pools", poolName, fmt.Sprintf("%s-snapshots", string(volType)), volName)
	}

	return shared.VarPath("storage-pools", poolName, string(volType), volName)
}

// GetVolumeSnapshotDir gets the snapshot mount directory for the parent volume.
func GetVolumeSnapshotDir(poolName string, volType VolumeType, volName string) string {
	parent, _, _ := shared.InstanceGetParentAndSnapshotName(volName)
	return shared.VarPath("storage-pools", poolName, fmt.Sprintf("%s-snapshots", string(volType)), parent)
}

// GetSnapshotVolumeName returns the full volume name for a parent volume and snapshot name.
func GetSnapshotVolumeName(parentName, snapshotName string) string {
	return fmt.Sprintf("%s%s%s", parentName, shared.SnapshotDelimiter, snapshotName)
}

// createParentSnapshotDirIfMissing creates the parent directory for volume snapshots
func createParentSnapshotDirIfMissing(poolName string, volType VolumeType, volName string) error {
	snapshotsPath := GetVolumeSnapshotDir(poolName, volType, volName)

	// If it's missing, create it.
	if !shared.PathExists(snapshotsPath) {
		return os.Mkdir(snapshotsPath, 0700)
	}

	return nil
}

// deleteParentSnapshotDirIfEmpty removes the parent snapshot directory if it is empty.
// It accepts the pool name, volume type and parent volume name.
func deleteParentSnapshotDirIfEmpty(poolName string, volType VolumeType, volName string) error {
	snapshotsPath := GetVolumeSnapshotDir(poolName, volType, volName)

	// If it exists, try to delete it.
	if shared.PathExists(snapshotsPath) {
		isEmpty, err := shared.PathIsEmpty(snapshotsPath)
		if err != nil {
			return err
		}

		if isEmpty {
			err := os.Remove(snapshotsPath)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// createSparseFile creates a sparse empty file at specified location with specified size.
func createSparseFile(filePath string, sizeBytes int64) error {
	f, err := os.Create(filePath)
	if err != nil {
		return fmt.Errorf("Failed to open %s: %s", filePath, err)
	}
	defer f.Close()

	err = f.Chmod(0600)
	if err != nil {
		return fmt.Errorf("Failed to chmod %s: %s", filePath, err)
	}

	err = f.Truncate(sizeBytes)
	if err != nil {
		return fmt.Errorf("Failed to create sparse file %s: %s", filePath, err)
	}

	return nil
}

// ensureVolumeBlockFile creates or resizes the raw block file for a volume.
func ensureVolumeBlockFile(vol Volume, path string) error {
	blockSize := vol.ExpandedConfig("size")
	if blockSize == "" {
		blockSize = defaultBlockSize
	}

	blockSizeBytes, err := units.ParseByteSizeString(blockSize)
	if err != nil {
		return err
	}

	if shared.PathExists(path) {
		_, err = shared.RunCommand("qemu-img", "resize", "-f", "raw", path, fmt.Sprintf("%d", blockSizeBytes))
		if err != nil {
			return fmt.Errorf("Failed resizing disk image %s to size %s: %v", path, blockSize, err)
		}
	} else {
		// If path doesn't exist, then there has been no filler function
		// supplied to create it from another source. So instead create an empty
		// volume (use for PXE booting a VM).
		_, err = shared.RunCommand("qemu-img", "create", "-f", "raw", path, fmt.Sprintf("%d", blockSizeBytes))
		if err != nil {
			return fmt.Errorf("Failed creating disk image %s as size %s: %v", path, blockSize, err)
		}
	}

	return nil
}

// mkfsOptions represents options for filesystem creation.
type mkfsOptions struct {
	Label string
}

// makeFSType creates the provided filesystem.
func makeFSType(path string, fsType string, options *mkfsOptions) (string, error) {
	var err error
	var msg string

	fsOptions := options
	if fsOptions == nil {
		fsOptions = &mkfsOptions{}
	}

	cmd := []string{fmt.Sprintf("mkfs.%s", fsType), path}
	if fsOptions.Label != "" {
		cmd = append(cmd, "-L", fsOptions.Label)
	}

	if fsType == "ext4" {
		cmd = append(cmd, "-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0")
	}

	msg, err = shared.TryRunCommand(cmd[0], cmd[1:]...)
	if err != nil {
		return msg, err
	}

	return "", nil
}

// mountOption represents an individual mount option.
type mountOption struct {
	capture bool
	flag    uintptr
}

// mountOptions represents a list of possible mount options.
var mountOptions = map[string]mountOption{
	"async":         {false, unix.MS_SYNCHRONOUS},
	"atime":         {false, unix.MS_NOATIME},
	"bind":          {true, unix.MS_BIND},
	"defaults":      {true, 0},
	"dev":           {false, unix.MS_NODEV},
	"diratime":      {false, unix.MS_NODIRATIME},
	"dirsync":       {true, unix.MS_DIRSYNC},
	"exec":          {false, unix.MS_NOEXEC},
	"lazytime":      {true, unix.MS_LAZYTIME},
	"mand":          {true, unix.MS_MANDLOCK},
	"noatime":       {true, unix.MS_NOATIME},
	"nodev":         {true, unix.MS_NODEV},
	"nodiratime":    {true, unix.MS_NODIRATIME},
	"noexec":        {true, unix.MS_NOEXEC},
	"nomand":        {false, unix.MS_MANDLOCK},
	"norelatime":    {false, unix.MS_RELATIME},
	"nostrictatime": {false, unix.MS_STRICTATIME},
	"nosuid":        {true, unix.MS_NOSUID},
	"rbind":         {true, unix.MS_BIND | unix.MS_REC},
	"relatime":      {true, unix.MS_RELATIME},
	"remount":       {true, unix.MS_REMOUNT},
	"ro":            {true, unix.MS_RDONLY},
	"rw":            {false, unix.MS_RDONLY},
	"strictatime":   {true, unix.MS_STRICTATIME},
	"suid":          {false, unix.MS_NOSUID},
	"sync":          {true, unix.MS_SYNCHRONOUS},
}

// resolveMountOptions resolves the provided mount options.
func resolveMountOptions(options string) (uintptr, string) {
	mountFlags := uintptr(0)
	tmp := strings.SplitN(options, ",", -1)
	for i := 0; i < len(tmp); i++ {
		opt := tmp[i]
		do, ok := mountOptions[opt]
		if !ok {
			continue
		}

		if do.capture {
			mountFlags |= do.flag
		} else {
			mountFlags &= ^do.flag
		}

		copy(tmp[i:], tmp[i+1:])
		tmp[len(tmp)-1] = ""
		tmp = tmp[:len(tmp)-1]
		i--
	}

	return mountFlags, strings.Join(tmp, ",")
}

// shrinkFileSystem shrinks a filesystem if it is supported. Ext4 volumes will be unmounted temporarily if needed.
func shrinkFileSystem(fsType string, devPath string, vol Volume, byteSize int64) error {
	strSize := fmt.Sprintf("%dK", byteSize/1024)

	switch fsType {
	case "": // if not specified, default to ext4.
		fallthrough
	case "xfs":
		return fmt.Errorf(`Shrinking not supported for filesystem type "%s". A dump, mkfs, and restore are required`, fsType)
	case "ext4":
		return vol.UnmountTask(func(op *operations.Operation) error {
			_, err := shared.TryRunCommand("e2fsck", "-f", "-y", devPath)
			if err != nil {
				return err
			}

			_, err = shared.TryRunCommand("resize2fs", devPath, strSize)
			if err != nil {
				return err
			}

			return nil
		}, nil)
	case "btrfs":
		_, err := shared.TryRunCommand("btrfs", "filesystem", "resize", strSize, vol.MountPath())
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf(`Shrinking not supported for filesystem type "%s"`, fsType)
	}

	return nil
}

// growFileSystem grows a filesystem if it is supported. The volume will be mounted temporarily if needed.
func growFileSystem(fsType string, devPath string, vol Volume) error {
	return vol.MountTask(func(mountPath string, op *operations.Operation) error {
		var msg string
		var err error
		switch fsType {
		case "": // if not specified, default to ext4
			fallthrough
		case "ext4":
			msg, err = shared.TryRunCommand("resize2fs", devPath)
		case "xfs":
			msg, err = shared.TryRunCommand("xfs_growfs", devPath)
		case "btrfs":
			msg, err = shared.TryRunCommand("btrfs", "filesystem", "resize", "max", mountPath)
		default:
			return fmt.Errorf(`Growing not supported for filesystem type "%s"`, fsType)
		}

		if err != nil {
			errorMsg := fmt.Sprintf(`Could not extend underlying %s filesystem for "%s": %s`, fsType, devPath, msg)
			return fmt.Errorf(errorMsg)
		}

		return nil
	}, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"

	"github.com/lxc/lxd/lxd/backup"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/db/cluster"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/state"
	storagePools "github.com/lxc/lxd/lxd/storage"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/version"
)

var supportedVolumeTypes = []int{db.StoragePoolVolumeTypeContainer, db.StoragePoolVolumeTypeVM, db.StoragePoolVolumeTypeCustom, db.StoragePoolVolumeTypeImage}

func storagePoolVolumeUpdateUsers(d *Daemon, projectName string, oldPoolName string, oldVol *api.StorageVolume, newPoolName string, newVol *api.StorageVolume) error {
	s := d.State()

	// Update all instances that are using the volume with a local (non-expanded) device.
	err := storagePools.VolumeUsedByInstanceDevices(s, oldPoolName, projectName, oldVol, false, func(dbInst db.InstanceArgs, project api.Project, usedByDevices []string) error {
		inst, err := instance.Load(s, dbInst, nil)
		if err != nil {
			return err
		}

		localDevices := inst.LocalDevices()
		for _, devName := range usedByDevices {
			_, exists := localDevices[devName]
			if exists {
				localDevices[devName]["pool"] = newPoolName
				localDevices[devName]["source"] = newVol.Name
			}
		}

		args := db.InstanceArgs{
			Architecture: inst.Architecture(),
			Description:  inst.Description(),
			Config:       inst.LocalConfig(),
			Devices:      localDevices,
			Ephemeral:    inst.IsEphemeral(),
			Profiles:     inst.Profiles(),
			Project:      inst.Project(),
			Type:         inst.Type(),
			Snapshot:     inst.IsSnapshot(),
		}

		err = inst.Update(args, false)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return err
	}

	// Update all profiles that are using the volume with a device.
	err = storagePools.VolumeUsedByProfileDevices(s, oldPoolName, projectName, oldVol, func(profileID int64, profile api.Profile, p cluster.Project, usedByDevices []string) error {
		for name, dev := range profile.Devices {
			if shared.StringInSlice(name, usedByDevices) {
				dev["pool"] = newPoolName
				dev["source"] = newVol.Name
			}
		}

		pUpdate := api.ProfilePut{}
		pUpdate.Config = profile.Config
		pUpdate.Description = profile.Description
		pUpdate.Devices = profile.Devices
		err = doProfileUpdate(d, p.Name, profile.Name, profileID, &profile, pUpdate)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return err
	}

	return nil
}

// volumeUsedBy = append(volumeUsedBy, fmt.Sprintf("/%s/containers/%s", version.APIVersion, ct)).
func storagePoolVolumeUsedByGet(s *state.State, projectName string, poolName string, vol *api.StorageVolume) ([]string, error) {
	// Handle instance volumes.
	if vol.Type == db.StoragePoolVolumeTypeNameContainer || vol.Type == db.StoragePoolVolumeTypeNameVM {
		cName, sName, snap := api.GetParentAndSnapshotName(vol.Name)
		if snap {
			if projectName == project.Default {
				return []string{fmt.Sprintf("/%s/instances/%s/snapshots/%s", version.APIVersion, cName, sName)}, nil
			}

			return []string{fmt.Sprintf("/%s/instances/%s/snapshots/%s?project=%s", version.APIVersion, cName, sName, projectName)}, nil
		}

		if projectName == project.Default {
			return []string{fmt.Sprintf("/%s/instances/%s", version.APIVersion, cName)}, nil
		}

		return []string{fmt.Sprintf("/%s/instances/%s?project=%s", version.APIVersion, cName, projectName)}, nil
	}

	// Handle image volumes.
	if vol.Type == db.StoragePoolVolumeTypeNameImage {
		if projectName == project.Default {
			return []string{fmt.Sprintf("/%s/images/%s", version.APIVersion, vol.Name)}, nil
		}

		return []string{fmt.Sprintf("/%s/images/%s?project=%s", version.APIVersion, vol.Name, projectName)}, nil
	}

	// Check if the daemon itself is using it.
	used, err := storagePools.VolumeUsedByDaemon(s, poolName, vol.Name)
	if err != nil {
		return []string{}, err
	}

	if used {
		return []string{fmt.Sprintf("/%s", version.APIVersion)}, nil
	}

	// Look for instances using this volume.
	volumeUsedBy := []string{}

	// Pass false to expandDevices, as we only want to see instances directly using a volume, rather than their
	// profiles using a volume.
	err = storagePools.VolumeUsedByInstanceDevices(s, poolName, projectName, vol, false, func(inst db.InstanceArgs, p api.Project, usedByDevices []string) error {
		if inst.Project == project.Default {
			volumeUsedBy = append(volumeUsedBy, fmt.Sprintf("/%s/instances/%s", version.APIVersion, inst.Name))
		} else {
			volumeUsedBy = append(volumeUsedBy, fmt.Sprintf("/%s/instances/%s?project=%s", version.APIVersion, inst.Name, inst.Project))
		}

		return nil
	})
	if err != nil {
		return []string{}, err
	}

	err = storagePools.VolumeUsedByProfileDevices(s, poolName, projectName, vol, func(profileID int64, profile api.Profile, p cluster.Project, usedByDevices []string) error {
		if p.Name == project.Default {
			volumeUsedBy = append(volumeUsedBy, fmt.Sprintf("/%s/profiles/%s", version.APIVersion, profile.Name))
		} else {
			volumeUsedBy = append(volumeUsedBy, fmt.Sprintf("/%s/profiles/%s?project=%s", version.APIVersion, profile.Name, p.Name))
		}

		return nil
	})
	if err != nil {
		return []string{}, err
	}

	return volumeUsedBy, nil
}

func storagePoolVolumeBackupLoadByName(s *state.State, projectName, poolName, backupName string) (*backup.VolumeBackup, error) {
	b, err := s.DB.Cluster.GetStoragePoolVolumeBackup(projectName, poolName, backupName)
	if err != nil {
		return nil, err
	}

	volumeName := strings.Split(backupName, "/")[0]
	backup := backup.NewVolumeBackup(s, projectName, poolName, volumeName, b.ID, b.Name, b.CreationDate, b.ExpiryDate, b.VolumeOnly, b.OptimizedStorage)

	return backup, nil
}
<commit_msg>lxd/storage/volumes/utils: Updates storagePoolVolumeUsedByGet to add target param<commit_after>package main

import (
	"strings"

	"github.com/lxc/lxd/lxd/backup"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/db/cluster"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/state"
	storagePools "github.com/lxc/lxd/lxd/storage"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/version"
)

var supportedVolumeTypes = []int{db.StoragePoolVolumeTypeContainer, db.StoragePoolVolumeTypeVM, db.StoragePoolVolumeTypeCustom, db.StoragePoolVolumeTypeImage}

func storagePoolVolumeUpdateUsers(d *Daemon, projectName string, oldPoolName string, oldVol *api.StorageVolume, newPoolName string, newVol *api.StorageVolume) error {
	s := d.State()

	// Update all instances that are using the volume with a local (non-expanded) device.
	err := storagePools.VolumeUsedByInstanceDevices(s, oldPoolName, projectName, oldVol, false, func(dbInst db.InstanceArgs, project api.Project, usedByDevices []string) error {
		inst, err := instance.Load(s, dbInst, nil)
		if err != nil {
			return err
		}

		localDevices := inst.LocalDevices()
		for _, devName := range usedByDevices {
			_, exists := localDevices[devName]
			if exists {
				localDevices[devName]["pool"] = newPoolName
				localDevices[devName]["source"] = newVol.Name
			}
		}

		args := db.InstanceArgs{
			Architecture: inst.Architecture(),
			Description:  inst.Description(),
			Config:       inst.LocalConfig(),
			Devices:      localDevices,
			Ephemeral:    inst.IsEphemeral(),
			Profiles:     inst.Profiles(),
			Project:      inst.Project(),
			Type:         inst.Type(),
			Snapshot:     inst.IsSnapshot(),
		}

		err = inst.Update(args, false)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return err
	}

	// Update all profiles that are using the volume with a device.
	err = storagePools.VolumeUsedByProfileDevices(s, oldPoolName, projectName, oldVol, func(profileID int64, profile api.Profile, p cluster.Project, usedByDevices []string) error {
		for name, dev := range profile.Devices {
			if shared.StringInSlice(name, usedByDevices) {
				dev["pool"] = newPoolName
				dev["source"] = newVol.Name
			}
		}

		pUpdate := api.ProfilePut{}
		pUpdate.Config = profile.Config
		pUpdate.Description = profile.Description
		pUpdate.Devices = profile.Devices
		err = doProfileUpdate(d, p.Name, profile.Name, profileID, &profile, pUpdate)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return err
	}

	return nil
}

// storagePoolVolumeUsedByGet returns a list of URL resources that use the volume.
func storagePoolVolumeUsedByGet(s *state.State, projectName string, poolName string, vol *api.StorageVolume) ([]string, error) {
	// Handle instance volumes.
	if vol.Type == db.StoragePoolVolumeTypeNameContainer || vol.Type == db.StoragePoolVolumeTypeNameVM {
		volName, snapName, isSnap := api.GetParentAndSnapshotName(vol.Name)
		if isSnap {
			return []string{api.NewURL().Path(version.APIVersion, "instances", volName, "snapshots", snapName).Project(projectName).String()}, nil
		}

		return []string{api.NewURL().Path(version.APIVersion, "instances", volName).Project(projectName).String()}, nil
	}

	// Handle image volumes.
	if vol.Type == db.StoragePoolVolumeTypeNameImage {
		return []string{api.NewURL().Path(version.APIVersion, "images", vol.Name).Project(projectName).Target(vol.Location).String()}, nil
	}

	// Check if the daemon itself is using it.
	used, err := storagePools.VolumeUsedByDaemon(s, poolName, vol.Name)
	if err != nil {
		return []string{}, err
	}

	if used {
		return []string{api.NewURL().Path(version.APIVersion).String()}, nil
	}

	// Look for instances using this volume.
	volumeUsedBy := []string{}

	// Pass false to expandDevices, as we only want to see instances directly using a volume, rather than their
	// profiles using a volume.
	err = storagePools.VolumeUsedByInstanceDevices(s, poolName, projectName, vol, false, func(inst db.InstanceArgs, p api.Project, usedByDevices []string) error {
		volumeUsedBy = append(volumeUsedBy, api.NewURL().Path(version.APIVersion, "instances", inst.Name).Project(inst.Project).String())
		return nil
	})
	if err != nil {
		return []string{}, err
	}

	err = storagePools.VolumeUsedByProfileDevices(s, poolName, projectName, vol, func(profileID int64, profile api.Profile, p cluster.Project, usedByDevices []string) error {
		volumeUsedBy = append(volumeUsedBy, api.NewURL().Path(version.APIVersion, "profiles", profile.Name).Project(p.Name).String())
		return nil
	})
	if err != nil {
		return []string{}, err
	}

	return volumeUsedBy, nil
}

func storagePoolVolumeBackupLoadByName(s *state.State, projectName, poolName, backupName string) (*backup.VolumeBackup, error) {
	b, err := s.DB.Cluster.GetStoragePoolVolumeBackup(projectName, poolName, backupName)
	if err != nil {
		return nil, err
	}

	volumeName := strings.Split(backupName, "/")[0]
	backup := backup.NewVolumeBackup(s, projectName, poolName, volumeName, b.ID, b.Name, b.CreationDate, b.ExpiryDate, b.VolumeOnly, b.OptimizedStorage)

	return backup, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"text/template"
)

var (
	aliases []string = []string{
		"go",
		"godep",
	}
	git_http_re *regexp.Regexp = regexp.MustCompile("^https?://(.+).git$")
	git_ssh_re  *regexp.Regexp = regexp.MustCompile("^.+@([^:]+):(.+).git$")
	hg_http_re  *regexp.Regexp = regexp.MustCompile("^https?://(.+)$")
	hg_ssh_re   *regexp.Regexp = regexp.MustCompile("^ssh://[^@]+@(.+)$")
)

type Environment struct {
	Package string
	Root    string
	GOPATH  string
}

type envWrap struct {
	Env     *Environment
	Aliases []string
}

func LoadEnvfile() (*Environment, error) {
	root, e := getRoot()
	if e != nil {
		return nil, e
	}

	env_path := filepath.Join(root, ".goenv", "bin", "activate")
	fi, e := os.OpenFile(env_path, os.O_RDONLY, os.ModePerm)
	if e != nil {
		return nil, e
	}

	env := new(Environment)

	r := bufio.NewReader(fi)
	for true {
		line, e := r.ReadString('\n')
		if e != nil {
			break
		}
		switch {
		case strings.HasPrefix(line, "ENV_DIR"):
			env.Root = strings.SplitN(line, "=", 1)[1]
			break
		case strings.HasPrefix(line, "GOPACKAGE"):
			env.Package = strings.SplitN(line, "=", 1)[1]
			break
		}
	}

	if env.Package == "" {
		return env, errors.New("Package name is empty. it looks like broken .env file")
	}

	return env, nil
}

func getPackageNameGit() (string, error) {
	cmd := exec.Command("git", "config", "--get", "remote.origin.url")
	out, e := cmd.StdoutPipe()
	if e != nil {
		return "", e
	}
	cmd.Start()
	url := make([]byte, 512)
	length, e := out.Read(url)
	if e != nil {
		return "", e
	}
	e = cmd.Wait()
	if e != nil {
		return "", e
	}
	buf := bytes.NewBuffer(url)
	buf.Truncate(length)
	package_url := strings.TrimSpace(buf.String())
	if strs := git_http_re.FindStringSubmatch(package_url); len(strs) != 0 {
		return strs[1], nil
	} else if strs := git_ssh_re.FindStringSubmatch(package_url); len(strs) != 0 {
		return fmt.Sprintf("%s/%s", strs[1], strs[2]), nil
	} else {
		return "", errors.New("not matched")
	}
}

func getPackageNameHg() (string, error) {
	cmd := exec.Command("hg", "paths", "default")
	out, e := cmd.StdoutPipe()
	if e != nil {
		return "", e
	}
	cmd.Start()
	url := make([]byte, 512)
	length, e := out.Read(url)
	if e != nil {
		return "", e
	}
	e = cmd.Wait()
	if e != nil {
		return "", e
	}
	buf := bytes.NewBuffer(url)
	buf.Truncate(length)
	package_url := strings.TrimSpace(buf.String())
	if strs := hg_http_re.FindStringSubmatch(package_url); len(strs) != 0 {
		return strs[1], nil
	} else if strs := hg_ssh_re.FindStringSubmatch(package_url); len(strs) != 0 {
		return strs[1], nil
	} else {
		return "", errors.New("not matched")
	}
}

func getPackage() (string, error) {
	var (
		name string
		e    error
	)
	name, e = getPackageNameGit()
	if len(name) == 0 {
		name, e = getPackageNameHg()
	}
	if len(name) == 0 {
		return name, e
	}
	return name, e
}

func getRootGit() (string, error) {
	cmd := exec.Command("git", "rev-parse", "--show-toplevel")
	out, e := cmd.StdoutPipe()
	e = cmd.Start()
	if e != nil {
		return "", e
	}
	path := make([]byte, 512)
	length, e := out.Read(path)
	if e != nil {
		return "", e
	}
	e = cmd.Wait()
	if e != nil {
		return "", e
	}
	buf := bytes.NewBuffer(path)
	buf.Truncate(length)
	return strings.TrimSpace(buf.String()), nil
}

func getRootHg() (string, error) {
	cmd := exec.Command("hg", "root")
	out, e := cmd.StdoutPipe()
	e = cmd.Start()
	if e != nil {
		return "", e
	}
	path := make([]byte, 512)
	length, e := out.Read(path)
	if e != nil {
		return "", e
	}
	e = cmd.Wait()
	if e != nil {
		return "", e
	}
	buf := bytes.NewBuffer(path)
	buf.Truncate(length)
	return strings.TrimSpace(buf.String()), nil
}

func getRoot() (string, error) {
	var root string
	root, _ = getRootGit()
	if len(root) == 0 {
		root, _ = getRootHg()
	}
	if len(root) == 0 {
		return "", errors.New("Can't find root of local repository for working directory")
	}
	switch runtime.GOOS {
	case "windows":
		root = strings.Replace(root, "/", "\\", -1)
	}
	return root, nil
}

func writeEnvFile(wrap envWrap, writer io.Writer, templateStr string) error {
	t, e := template.New("env_script").Parse(templateStr)
	if e != nil {
		return e
	}
	return t.Execute(writer, wrap)
}

func writeWrap(env *Environment, filename string, function func(*Environment, io.Writer) error) {
	file, e := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
	defer file.Close()
	if e != nil {
		log.Println("Open failed : %q", e)
	} else {
		e = function(env, file)
		if e != nil {
			log.Println("Write failed : %q", e)
		}
	}
}
<commit_msg>fix LoadEnv function<commit_after>package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"text/template"
)

var (
	aliases []string = []string{
		"go",
		"godep",
	}
	git_http_re *regexp.Regexp = regexp.MustCompile("^https?://(.+).git$")
	git_ssh_re  *regexp.Regexp = regexp.MustCompile("^.+@([^:]+):(.+).git$")
	hg_http_re  *regexp.Regexp = regexp.MustCompile("^https?://(.+)$")
	hg_ssh_re   *regexp.Regexp = regexp.MustCompile("^ssh://[^@]+@(.+)$")
)

type Environment struct {
	Package string
	Root    string
	GOPATH  string
}

type envWrap struct {
	Env     *Environment
	Aliases []string
}

func LoadEnvfile() (*Environment, error) {
	root, e := getRoot()
	if e != nil {
		return nil, e
	}

	env_path := filepath.Join(root, ".goenv", "bin", "activate")
	fi, e := os.OpenFile(env_path, os.O_RDONLY, os.ModePerm)
	if e != nil {
		return nil, e
	}

	env := new(Environment)

	r := bufio.NewReader(fi)
	for count := 3; count > 0; {
		line, e := r.ReadString('\n')
		if e != nil {
			break
		}
		switch {
		case strings.HasPrefix(line, "ENV_DIR="):
			env.Root = strings.TrimSpace(strings.SplitN(line, "=", 2)[1])
			count--
		case strings.HasPrefix(line, "GOPACKAGE="):
			env.Package = strings.TrimSpace(strings.SplitN(line, "=", 2)[1])
			count--
		case strings.HasPrefix(line, "GOPATH="):
			env.GOPATH = strings.TrimSpace(strings.SplitN(line, "=", 2)[1])
			count--
		}
	}

	if env.Package == "" {
		return env, errors.New("Package name is empty. it looks like broken .env file")
	}

	return env, nil
}

func getPackageNameGit() (string, error) {
	cmd := exec.Command("git", "config", "--get", "remote.origin.url")
	out, e := cmd.StdoutPipe()
	if e != nil {
		return "", e
	}
	cmd.Start()
	url := make([]byte, 512)
	length, e := out.Read(url)
	if e != nil {
		return "", e
	}
	e = cmd.Wait()
	if e != nil {
		return "", e
	}
	buf := bytes.NewBuffer(url)
	buf.Truncate(length)
	package_url := strings.TrimSpace(buf.String())
	if strs := git_http_re.FindStringSubmatch(package_url); len(strs) != 0 {
		return strs[1], nil
	} else if strs := git_ssh_re.FindStringSubmatch(package_url); len(strs) != 0 {
		return fmt.Sprintf("%s/%s", strs[1], strs[2]), nil
	} else {
		return "", errors.New("not matched")
	}
}

func getPackageNameHg() (string, error) {
	cmd := exec.Command("hg", "paths", "default")
	out, e := cmd.StdoutPipe()
	if e != nil {
		return "", e
	}
	cmd.Start()
	url := make([]byte, 512)
	length, e := out.Read(url)
	if e != nil {
		return "", e
	}
	e = cmd.Wait()
	if e != nil {
		return "", e
	}
	buf := bytes.NewBuffer(url)
	buf.Truncate(length)
	package_url := strings.TrimSpace(buf.String())
	if strs := hg_http_re.FindStringSubmatch(package_url); len(strs) != 0 {
		return strs[1], nil
	} else if strs := hg_ssh_re.FindStringSubmatch(package_url); len(strs) != 0 {
		return strs[1], nil
	} else {
		return "", errors.New("not matched")
	}
}

func getPackage() (string, error) {
	var (
		name string
		e    error
	)
	name, e = getPackageNameGit()
	if len(name) == 0 {
		name, e = getPackageNameHg()
	}
	if len(name) == 0 {
		return name, e
	}
	return name, e
}

func getRootGit() (string, error) {
	cmd := exec.Command("git", "rev-parse", "--show-toplevel")
	out, e := cmd.StdoutPipe()
	e = cmd.Start()
	if e != nil {
		return "", e
	}
	path := make([]byte, 512)
	length, e := out.Read(path)
	if e != nil {
		return "", e
	}
	e = cmd.Wait()
	if e != nil {
		return "", e
	}
	buf := bytes.NewBuffer(path)
	buf.Truncate(length)
	return strings.TrimSpace(buf.String()), nil
}

func getRootHg() (string, error) {
	cmd := exec.Command("hg", "root")
	out, e := cmd.StdoutPipe()
	e = cmd.Start()
	if e != nil {
		return "", e
	}
	path := make([]byte, 512)
	length, e := out.Read(path)
	if e != nil {
		return "", e
	}
	e = cmd.Wait()
	if e != nil {
		return "", e
	}
	buf := bytes.NewBuffer(path)
	buf.Truncate(length)
	return strings.TrimSpace(buf.String()), nil
}

func getRoot() (string, error) {
	var root string
	root, _ = getRootGit()
	if len(root) == 0 {
		root, _ = getRootHg()
	}
	if len(root) == 0 {
		return "", errors.New("Can't find root of local repository for working directory")
	}
	switch runtime.GOOS {
	case "windows":
		root = strings.Replace(root, "/", "\\", -1)
	}
	return root, nil
}

func writeEnvFile(wrap envWrap, writer io.Writer, templateStr string) error {
	t, e := template.New("env_script").Parse(templateStr)
	if e != nil {
		return e
	}
	return t.Execute(writer, wrap)
}

func writeWrap(env *Environment, filename string, function func(*Environment, io.Writer) error) {
	file, e := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
	defer file.Close()
	if e != nil {
		log.Println("Open failed : %q", e)
	} else {
		e = function(env, file)
		if e != nil {
			log.Println("Write failed : %q", e)
		}
	}
}
<|endoftext|>