<commit_before>package dcmnet

import (
	"bytes"
	"io"
	"io/ioutil"
	"strings"
	"testing"

	"github.com/jeremyhuiskamp/dcm/dcm"
	. "github.com/onsi/gomega"
)

// TODO: tests for unexpected errors

func TestReadSingleMessageElementSinglePDV(t *testing.T) {
	data := bufcat(bufpdv(1, Data, true, "data"))

	med := NewMessageElementDecoder(NewPDVDecoder(&data))

	expectMessageElement(t, med, 1, Data, "data")
	expectNoMoreMessageElements(t, med)
}

func TestReadSingleMessageElementTwoPDVs(t *testing.T) {
	data := bufcat(
		bufpdv(1, Data, false, "data1"),
		bufpdv(1, Data, true, "data2"))

	med := NewMessageElementDecoder(NewPDVDecoder(&data))

	expectMessageElement(t, med, 1, Data, "data1data2")
	expectNoMoreMessageElements(t, med)
}

func TestReadTwoMessageElements(t *testing.T) {
	data := bufcat(
		bufpdv(1, Data, true, "data"),
		bufpdv(2, Command, true, "command"))

	med := NewMessageElementDecoder(NewPDVDecoder(&data))

	expectMessageElement(t, med, 1, Data, "data")
	expectMessageElement(t, med, 2, Command, "command")
	expectNoMoreMessageElements(t, med)
}

func TestReadDrainMessageElement(t *testing.T) {
	data := bufcat(
		bufpdv(1, Data, true, "data"),
		bufpdv(2, Command, true, "command"))

	med := NewMessageElementDecoder(NewPDVDecoder(&data))

	// don't read contents...
	_ = expectNextElement(t, med)

	expectMessageElement(t, med, 2, Command, "command")
	expectNoMoreMessageElements(t, med)
}

func TestReadMessageElementUnexpectedPDVType(t *testing.T) {
	data := bufcat(
		bufpdv(1, Data, false, "data"),
		bufpdv(1, Command, true, "command"))

	med := NewMessageElementDecoder(NewPDVDecoder(&data))

	expectMessageElementError(t, med, "unexpected type")
}

func TestReadMessageElementUnexpectedPresentationContext(t *testing.T) {
	data := bufcat(
		bufpdv(1, Data, false, "data1"),
		bufpdv(2, Data, true, "data2"))

	med := NewMessageElementDecoder(NewPDVDecoder(&data))

	expectMessageElementError(t, med, "unexpected presentation context")
}

func TestReadMessageElementUnexpectedMissingPDV(t *testing.T) {
	data := bufcat(
		bufpdv(1, Command, false, "data1"))

	med := NewMessageElementDecoder(NewPDVDecoder(&data))

	msg := expectNextElement(t, med)
	_, err := ioutil.ReadAll(msg.Data)
	if io.ErrUnexpectedEOF != err {
		t.Errorf("expected unexpected eof but got %q", err)
	}
}

func TestReadMessageElementWithNoPDVs(t *testing.T) {
	data := bufcat()
	med := NewMessageElementDecoder(NewPDVDecoder(&data))
	expectNoMoreMessageElements(t, med)
}

func TestWriteOneMessageElement(t *testing.T) {
	input := "abcdefghijklmnopqrstuvwxyz0123456789"

	// hits edge cases in pdu lengths
	// nb: a pdu that can't contain any data after the pdv header is illegal:
	for pdulen := uint32(pdvHeaderLen + 1); pdulen < 20; pdulen++ {
		for inputlen := 0; inputlen <= len(input); inputlen++ {
			curinput := input[:inputlen]

			data := new(bytes.Buffer)
			pdus := NewPDUEncoder(data)
			msgs := NewMessageElementEncoder(pdus, pdulen)

			msgs.NextMessageElement(MessageElement{
				Context: 1,
				Type:    Command,
				Data:    toBufferP(curinput),
			})

			output := new(bytes.Buffer)
			last := false
			for data.Len() > 0 && !last {
				pduData := expectPDU(t, data, PDUPresentationData)
				pdvLast, pdvData := expectPDV(t, &pduData, PCID(1), Command)
				output.Write(pdvData.Bytes())
				last = pdvLast
			}

			if data.Len() != 0 {
				t.Fatal("expected no data left in pdu")
			}

			if (len(curinput) == 0) == last {
				// TODO: explain what this means in error message
				t.Fatalf("last: %t, len(curinput): %d",
					last, len(curinput))
			}

			if curinput != output.String() {
				t.Fatalf("expected output %q but got %q, "+
					"where pdulen=%d and inputlen=%d",
					curinput, output, pdulen, inputlen)
			}
		}
	}
}

func TestWriteMultipleMessageElements(t *testing.T) {
	inputs := []string{
		"short",
		strings.Repeat("long", 5),
		strings.Repeat("reallylong", 30),
	}

	data := new(bytes.Buffer)
	pdus := NewPDUEncoder(data)
	msgs := NewMessageElementEncoder(pdus, 30)

	for context, value := range inputs {
		msgs.NextMessageElement(MessageElement{
			Context: PCID(context),
			Type:    Command,
			Data:    toBufferP("command" + value),
		})
		msgs.NextMessageElement(MessageElement{
			Context: PCID(context),
			Type:    Data,
			Data:    toBufferP("data" + value),
		})
	}

	expectMessageElement := func(context PCID, typ PDVType) string {
		last := false
		output := new(bytes.Buffer)

		for !last {
			pduData := expectPDU(t, data, PDUPresentationData)
			pdvLast, pdvData := expectPDV(t, &pduData, context, typ)
			output.Write(pdvData.Bytes())
			last = pdvLast
		}

		return output.String()
	}

	for context, value := range inputs {
		expCmdValue := "command" + value
		gotCmdValue := expectMessageElement(PCID(context), Command)
		if expCmdValue != gotCmdValue {
			t.Fatalf("expected %q, got %q", expCmdValue, gotCmdValue)
		}

		expDataValue := "data" + value
		gotDataValue := expectMessageElement(PCID(context), Data)
		if expDataValue != gotDataValue {
			t.Fatalf("expected %q, got %q", expDataValue, gotDataValue)
		}
	}

	if data.Len() != 0 {
		t.Fatalf("expected no more data but got %q", data)
	}
}

func TestReadNoMessages(t *testing.T) {
	RegisterTestingT(t)

	var data bytes.Buffer
	var pcs PresentationContexts
	md := NewMessageDecoder(pcs, NewMessageElementDecoder(NewPDVDecoder(&data)))

	msg, err := md.NextMessage()
	Expect(msg).To(BeNil())
	Expect(err).To(BeNil())
}

func TestReadUnexpectedData(t *testing.T) {
	RegisterTestingT(t)

	data := bufpdv(1, Data, true, "xxxx")
	var pcs PresentationContexts
	md := NewMessageDecoder(pcs, NewMessageElementDecoder(NewPDVDecoder(&data)))

	_, err := md.NextMessage()
	Expect(err).To(MatchError(ContainSubstring("Expected a command message")))
}

func TestReadUnexpectedPCID(t *testing.T) {
	RegisterTestingT(t)

	data := bufpdv(1, Command, true, "xxx")
	var pcs PresentationContexts
	md := NewMessageDecoder(pcs, NewMessageElementDecoder(NewPDVDecoder(&data)))

	_, err := md.NextMessage()
	Expect(err).To(MatchError(ContainSubstring("Unrecognized presentation context")))
}

func TestExpectDatasetButFindNone(t *testing.T) {
	RegisterTestingT(t)

	pcs := PresentationContexts{
		Requested: []PresentationContext{
			{
				ID:             1,
				AbstractSyntax: "??",
			},
		},
		Accepted: []PresentationContext{
			{
				ID:     1,
				Result: PCAcceptance,
				TransferSyntaxes: []dcm.TransferSyntax{
					dcm.ImplicitVRLittleEndian,
				},
			},
		},
	}

	data := bufpdv(1, Command, true,
		// data set type element indicating that data should be expected:
		[]byte{0x00, 0x00, 0x00, 0x08, 0x02, 0x00, 0x00, 0x00, 0x02, 0x01})
	md := NewMessageDecoder(pcs, NewMessageElementDecoder(NewPDVDecoder(&data)))

	_, err := md.NextMessage()
	Expect(err).To(Equal(io.ErrUnexpectedEOF))
}

func expectNextElement(
	t *testing.T,
	msgs *MessageElementDecoder,
) *MessageElement {
	msg, err := msgs.NextMessageElement()
	if err != nil {
		t.Fatal(err)
	}
	if msg == nil {
		t.Fatal("expected message not found")
	}
	return msg
}

func expectMessageElement(
	t *testing.T,
	msgs *MessageElementDecoder,
	context PCID,
	tipe PDVType,
	value string,
) {
	msg := expectNextElement(t, msgs)
	if context != msg.Context {
		t.Errorf("expected presentation context %d but got %d",
			context, msg.Context)
	}
	if tipe != msg.Type {
		t.Errorf("expected pdv type %s but got %s", tipe, msg.Type)
	}
	gotValue := toString(msg.Data)
	if value != gotValue {
		t.Errorf("expected message data %q but got %q", value, gotValue)
	}
}

func expectMessageElementError(
	t *testing.T,
	msgs *MessageElementDecoder,
	errSubstring string,
) {
	msg, err := msgs.NextMessageElement()
	if err != nil {
		t.Fatal(err)
	}
	if msg == nil {
		t.Fatal("expected message not found")
	}

	_, err = ioutil.ReadAll(msg.Data)
	if err == nil {
		t.Fatal("expected error while reading message data")
	}
	if !strings.Contains(err.Error(), errSubstring) {
		t.Error("expected %q to contain %q", err.Error(), errSubstring)
	}
}

func expectNoMoreMessageElements(t *testing.T, msgs *MessageElementDecoder) {
	msg, err := msgs.NextMessageElement()
	if err != nil {
		t.Fatal(err)
	}
	if msg != nil {
		t.Fatalf("expected no more elements, but got %+V", msg)
	}
}

func expectPDV(
	t *testing.T,
	in *bytes.Buffer,
	expCtx PCID,
	expType PDVType,
) (bool, bytes.Buffer) {
	gotCtx, gotType, last, data, err := getpdv(in)
	if err != nil {
		t.Fatal(err)
	}
	if expCtx != gotCtx {
		t.Fatalf("expected presentation context %d, got %d", expCtx, gotCtx)
	}
	if expType != gotType {
		t.Fatalf("expected %s, got %s", expType, gotType)
	}
	return last, data
}
<commit_msg>less gomega<commit_after>package dcmnet

import (
	"bytes"
	"io"
	"io/ioutil"
	"strings"
	"testing"

	"github.com/jeremyhuiskamp/dcm/dcm"
)

// TODO: tests for unexpected errors

func TestReadSingleMessageElementSinglePDV(t *testing.T) {
	data := bufcat(bufpdv(1, Data, true, "data"))

	med := NewMessageElementDecoder(NewPDVDecoder(&data))

	expectMessageElement(t, med, 1, Data, "data")
	expectNoMoreMessageElements(t, med)
}

func TestReadSingleMessageElementTwoPDVs(t *testing.T) {
	data := bufcat(
		bufpdv(1, Data, false, "data1"),
		bufpdv(1, Data, true, "data2"))

	med := NewMessageElementDecoder(NewPDVDecoder(&data))

	expectMessageElement(t, med, 1, Data, "data1data2")
	expectNoMoreMessageElements(t, med)
}

func TestReadTwoMessageElements(t *testing.T) {
	data := bufcat(
		bufpdv(1, Data, true, "data"),
		bufpdv(2, Command, true, "command"))

	med := NewMessageElementDecoder(NewPDVDecoder(&data))

	expectMessageElement(t, med, 1, Data, "data")
	expectMessageElement(t, med, 2, Command, "command")
	expectNoMoreMessageElements(t, med)
}

func TestReadDrainMessageElement(t *testing.T) {
	data := bufcat(
		bufpdv(1, Data, true, "data"),
		bufpdv(2, Command, true, "command"))

	med := NewMessageElementDecoder(NewPDVDecoder(&data))

	// don't read contents...
	_ = expectNextElement(t, med)

	expectMessageElement(t, med, 2, Command, "command")
	expectNoMoreMessageElements(t, med)
}

func TestReadMessageElementUnexpectedPDVType(t *testing.T) {
	data := bufcat(
		bufpdv(1, Data, false, "data"),
		bufpdv(1, Command, true, "command"))

	med := NewMessageElementDecoder(NewPDVDecoder(&data))

	expectMessageElementError(t, med, "unexpected type")
}

func TestReadMessageElementUnexpectedPresentationContext(t *testing.T) {
	data := bufcat(
		bufpdv(1, Data, false, "data1"),
		bufpdv(2, Data, true, "data2"))

	med := NewMessageElementDecoder(NewPDVDecoder(&data))

	expectMessageElementError(t, med, "unexpected presentation context")
}

func TestReadMessageElementUnexpectedMissingPDV(t *testing.T) {
	data := bufcat(
		bufpdv(1, Command, false, "data1"))

	med := NewMessageElementDecoder(NewPDVDecoder(&data))

	msg := expectNextElement(t, med)
	_, err := ioutil.ReadAll(msg.Data)
	if io.ErrUnexpectedEOF != err {
		t.Errorf("expected unexpected eof but got %q", err)
	}
}

func TestReadMessageElementWithNoPDVs(t *testing.T) {
	data := bufcat()
	med := NewMessageElementDecoder(NewPDVDecoder(&data))
	expectNoMoreMessageElements(t, med)
}

func TestWriteOneMessageElement(t *testing.T) {
	input := "abcdefghijklmnopqrstuvwxyz0123456789"

	// hits edge cases in pdu lengths
	// nb: a pdu that can't contain any data after the pdv header is illegal:
	for pdulen := uint32(pdvHeaderLen + 1); pdulen < 20; pdulen++ {
		for inputlen := 0; inputlen <= len(input); inputlen++ {
			curinput := input[:inputlen]

			data := new(bytes.Buffer)
			pdus := NewPDUEncoder(data)
			msgs := NewMessageElementEncoder(pdus, pdulen)

			msgs.NextMessageElement(MessageElement{
				Context: 1,
				Type:    Command,
				Data:    toBufferP(curinput),
			})

			output := new(bytes.Buffer)
			last := false
			for data.Len() > 0 && !last {
				pduData := expectPDU(t, data, PDUPresentationData)
				pdvLast, pdvData := expectPDV(t, &pduData, PCID(1), Command)
				output.Write(pdvData.Bytes())
				last = pdvLast
			}

			if data.Len() != 0 {
				t.Fatal("expected no data left in pdu")
			}

			if (len(curinput) == 0) == last {
				// TODO: explain what this means in error message
				t.Fatalf("last: %t, len(curinput): %d",
					last, len(curinput))
			}

			if curinput != output.String() {
				t.Fatalf("expected output %q but got %q, "+
					"where pdulen=%d and inputlen=%d",
					curinput, output, pdulen, inputlen)
			}
		}
	}
}

func TestWriteMultipleMessageElements(t *testing.T) {
	inputs := []string{
		"short",
		strings.Repeat("long", 5),
		strings.Repeat("reallylong", 30),
	}

	data := new(bytes.Buffer)
	pdus := NewPDUEncoder(data)
	msgs := NewMessageElementEncoder(pdus, 30)

	for context, value := range inputs {
		msgs.NextMessageElement(MessageElement{
			Context: PCID(context),
			Type:    Command,
			Data:    toBufferP("command" + value),
		})
		msgs.NextMessageElement(MessageElement{
			Context: PCID(context),
			Type:    Data,
			Data:    toBufferP("data" + value),
		})
	}

	expectMessageElement := func(context PCID, typ PDVType) string {
		last := false
		output := new(bytes.Buffer)

		for !last {
			pduData := expectPDU(t, data, PDUPresentationData)
			pdvLast, pdvData := expectPDV(t, &pduData, context, typ)
			output.Write(pdvData.Bytes())
			last = pdvLast
		}

		return output.String()
	}

	for context, value := range inputs {
		expCmdValue := "command" + value
		gotCmdValue := expectMessageElement(PCID(context), Command)
		if expCmdValue != gotCmdValue {
			t.Fatalf("expected %q, got %q", expCmdValue, gotCmdValue)
		}

		expDataValue := "data" + value
		gotDataValue := expectMessageElement(PCID(context), Data)
		if expDataValue != gotDataValue {
			t.Fatalf("expected %q, got %q", expDataValue, gotDataValue)
		}
	}

	if data.Len() != 0 {
		t.Fatalf("expected no more data but got %q", data)
	}
}

func TestReadNoMessages(t *testing.T) {
	var data bytes.Buffer
	var pcs PresentationContexts
	md := NewMessageDecoder(pcs, NewMessageElementDecoder(NewPDVDecoder(&data)))

	msg, err := md.NextMessage()
	if msg != nil || err != nil {
		t.Fatal("expected neither msg (%V) nor error (%q)", msg, err)
	}
}

func TestReadUnexpectedData(t *testing.T) {
	data := bufpdv(1, Data, true, "xxxx")
	var pcs PresentationContexts
	md := NewMessageDecoder(pcs, NewMessageElementDecoder(NewPDVDecoder(&data)))

	_, err := md.NextMessage()
	if err == nil || !strings.Contains(err.Error(), "Expected a command message") {
		t.Fatalf("unexpected error: %q", err)
	}
}

func TestReadUnexpectedPCID(t *testing.T) {
	data := bufpdv(1, Command, true, "xxx")
	var pcs PresentationContexts
	md := NewMessageDecoder(pcs, NewMessageElementDecoder(NewPDVDecoder(&data)))

	_, err := md.NextMessage()
	if err == nil || !strings.Contains(err.Error(), "Unrecognized presentation context") {
		t.Fatalf("unexpected error: %q", err)
	}
}

func TestExpectDatasetButFindNone(t *testing.T) {
	pcs := PresentationContexts{
		Requested: []PresentationContext{{
			ID:             1,
			AbstractSyntax: "??",
		}},
		Accepted: []PresentationContext{{
			ID:     1,
			Result: PCAcceptance,
			TransferSyntaxes: []dcm.TransferSyntax{
				dcm.ImplicitVRLittleEndian,
			},
		}},
	}

	data := bufpdv(1, Command, true,
		// data set type element indicating that data should be expected:
		[]byte{0x00, 0x00, 0x00, 0x08, 0x02, 0x00, 0x00, 0x00, 0x02, 0x01})
	md := NewMessageDecoder(pcs, NewMessageElementDecoder(NewPDVDecoder(&data)))

	if _, err := md.NextMessage(); err != io.ErrUnexpectedEOF {
		t.Fatalf("expected eof but got: %s", err)
	}
}

func expectNextElement(
	t *testing.T,
	msgs *MessageElementDecoder,
) *MessageElement {
	msg, err := msgs.NextMessageElement()
	if err != nil {
		t.Fatal(err)
	}
	if msg == nil {
		t.Fatal("expected message not found")
	}
	return msg
}

func expectMessageElement(
	t *testing.T,
	msgs *MessageElementDecoder,
	context PCID,
	tipe PDVType,
	value string,
) {
	msg := expectNextElement(t, msgs)
	if context != msg.Context {
		t.Errorf("expected presentation context %d but got %d",
			context, msg.Context)
	}
	if tipe != msg.Type {
		t.Errorf("expected pdv type %s but got %s", tipe, msg.Type)
	}
	gotValue := toString(msg.Data)
	if value != gotValue {
		t.Errorf("expected message data %q but got %q", value, gotValue)
	}
}

func expectMessageElementError(
	t *testing.T,
	msgs *MessageElementDecoder,
	errSubstring string,
) {
	msg, err := msgs.NextMessageElement()
	if err != nil {
		t.Fatal(err)
	}
	if msg == nil {
		t.Fatal("expected message not found")
	}

	_, err = ioutil.ReadAll(msg.Data)
	if err == nil {
		t.Fatal("expected error while reading message data")
	}
	if !strings.Contains(err.Error(), errSubstring) {
		t.Error("expected %q to contain %q", err.Error(), errSubstring)
	}
}

func expectNoMoreMessageElements(t *testing.T, msgs *MessageElementDecoder) {
	msg, err := msgs.NextMessageElement()
	if err != nil {
		t.Fatal(err)
	}
	if msg != nil {
		t.Fatalf("expected no more elements, but got %+V", msg)
	}
}

func expectPDV(
	t *testing.T,
	in *bytes.Buffer,
	expCtx PCID,
	expType PDVType,
) (bool, bytes.Buffer) {
	gotCtx, gotType, last, data, err := getpdv(in)
	if err != nil {
		t.Fatal(err)
	}
	if expCtx != gotCtx {
		t.Fatalf("expected presentation context %d, got %d", expCtx, gotCtx)
	}
	if expType != gotType {
		t.Fatalf("expected %s, got %s", expType, gotType)
	}
	return last, data
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"github.com/realglobe-Inc/edo/driver"
	"github.com/realglobe-Inc/edo/util"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"time"
)

type token struct {
	// アクセストークン。
	Id string `json:"id"`
	// 発行したアカウント。
	AccId string `json:"account_id"`
	// 有効期限。
	ExpiDate time.Time `json:"expires"`
}

type tokenContainer interface {
	new(accId string, expiDur time.Duration) (*token, error)
	get(tokId string) (*token, error)
}

type tokenContainerImpl struct {
	// 認可コードの文字数。
	idLen int
	// デフォルトの有効期限。
	expiDur time.Duration
	// 最長の有効期限。
	maxExpiDur time.Duration

	base driver.TimeLimitedKeyValueStore
}

func (this *tokenContainerImpl) new(accId string, expiDur time.Duration) (*token, error) {
	var tokId string
	for {
		buff, err := util.SecureRandomBytes(this.idLen * 6 / 8)
		if err != nil {
			return nil, erro.Wrap(err)
		}

		tokId = base64.URLEncoding.EncodeToString(buff)
		if val, _, err := this.base.Get(tokId, nil); err != nil {
			return nil, erro.Wrap(err)
		} else if val == nil {
			// 昔発行した分とは重複しなかった。
			// 同時並列で発行している分と重複していない保証は無いが、まず大丈夫。
			break
		}
	}

	// アクセストークンが決まった。
	log.Debug("Token was generated")

	if expiDur <= 0 {
		expiDur = this.expiDur
	} else if expiDur > this.maxExpiDur {
		expiDur = this.maxExpiDur
	}

	tok := &token{tokId, accId, time.Now().Add(expiDur)}
	if _, err := this.base.Put(tokId, tok, tok.ExpiDate); err != nil {
		return nil, erro.Wrap(err)
	}

	log.Debug("Access token was published.")
	return tok, nil
}

func (this *tokenContainerImpl) get(tokId string) (*token, error) {
	val, _, err := this.base.Get(tokId, nil)
	if err != nil {
		return nil, erro.Wrap(err)
	} else if val == nil {
		return nil, nil
	}
	return val.(*token), nil
}
<commit_msg>アクセストークンにリフレッシュトークンと独自 scope を紐付けるための領域を追加<commit_after>package main

import (
	"encoding/base64"
	"github.com/realglobe-Inc/edo/driver"
	"github.com/realglobe-Inc/edo/util"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"time"
)

type token struct {
	// アクセストークン。
	Id string `json:"id"`
	// 発行したアカウント。
	AccId string `json:"account_id"`
	// 有効期限。
	ExpiDate time.Time `json:"expires"`
	// リフレッシュトークン。
	RefTok string `json:"refresh_token,omitempty"`
	// scope
	Scops map[string]bool `json:"scope,omitempty"`
}

type tokenContainer interface {
	new(accId string, expiDur time.Duration) (*token, error)
	get(tokId string) (*token, error)
}

type tokenContainerImpl struct {
	// 認可コードの文字数。
	idLen int
	// デフォルトの有効期限。
	expiDur time.Duration
	// 最長の有効期限。
	maxExpiDur time.Duration

	base driver.TimeLimitedKeyValueStore
}

func (this *tokenContainerImpl) new(accId string, expiDur time.Duration) (*token, error) {
	var tokId string
	for {
		buff, err := util.SecureRandomBytes(this.idLen * 6 / 8)
		if err != nil {
			return nil, erro.Wrap(err)
		}

		tokId = base64.URLEncoding.EncodeToString(buff)
		if val, _, err := this.base.Get(tokId, nil); err != nil {
			return nil, erro.Wrap(err)
		} else if val == nil {
			// 昔発行した分とは重複しなかった。
			// 同時並列で発行している分と重複していない保証は無いが、まず大丈夫。
			break
		}
	}

	// アクセストークンが決まった。
	log.Debug("Token was generated")

	if expiDur <= 0 {
		expiDur = this.expiDur
	} else if expiDur > this.maxExpiDur {
		expiDur = this.maxExpiDur
	}

	tok := &token{
		Id:       tokId,
		AccId:    accId,
		ExpiDate: time.Now().Add(expiDur),
	}
	if _, err := this.base.Put(tokId, tok, tok.ExpiDate); err != nil {
		return nil, erro.Wrap(err)
	}

	log.Debug("Access token was published.")
	return tok, nil
}

func (this *tokenContainerImpl) get(tokId string) (*token, error) {
	val, _, err := this.base.Get(tokId, nil)
	if err != nil {
		return nil, erro.Wrap(err)
	} else if val == nil {
		return nil, nil
	}
	return val.(*token), nil
}
<|endoftext|>


<commit_before>// +-------------------------------------------------------------------------
// | Copyright (C) 2016 Yunify, Inc.
// +-------------------------------------------------------------------------
// | Licensed under the Apache License, Version 2.0 (the "License");
// | you may not use this work except in compliance with the License.
// | You may obtain a copy of the License in the LICENSE file, or at:
// |
// | http://www.apache.org/licenses/LICENSE-2.0
// |
// | Unless required by applicable law or agreed to in writing, software
// | distributed under the License is distributed on an "AS IS" BASIS,
// | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// | See the License for the specific language governing permissions and
// | limitations under the License.
// +-------------------------------------------------------------------------

package main

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"

	"github.com/DATA-DOG/godog"

	"github.com/yunify/qingstor-sdk-go/request"
	qs "github.com/yunify/qingstor-sdk-go/service"
)

// ObjectFeatureContext provides feature context for object.
func ObjectFeatureContext(s *godog.Suite) {
	s.Step(`^put object with key "(.{1,})"$`, putObjectWithKey)
	s.Step(`^put object status code is (\d+)$`, putObjectStatusCodeIs)

	s.Step(`^copy object with key "(.{1,})"$`, copyObjectWithKey)
	s.Step(`^copy object status code is (\d+)$`, copyObjectStatusCodeIs)

	s.Step(`^move object with key "(.{1,})"$`, moveObjectWithKey)
	s.Step(`^move object status code is (\d+)$`, moveObjectStatusCodeIs)

	s.Step(`^get object with key "(.{1,})"$`, getObjectWithKey)
	s.Step(`^get object status code is (\d+)$`, getObjectStatusCodeIs)
	s.Step(`^get object content length is (\d+)$`, getObjectContentLengthIs)

	s.Step(`^get object "(.{1,})" with content type "(.{1,})"$`, getObjectWithContentType)
	s.Step(`^get object content type is "(.{1,})"$`, getObjectContentTypeIs)

	s.Step(`^get object "(.{1,})" with query signature$`, getObjectWithQuerySignature)
	s.Step(`^get object with query signature content length is (\d+)$`, getObjectWithQuerySignatureContentLengthIs)

	s.Step(`^head object with key "(.{1,})"$`, headObjectWithKey)
	s.Step(`^head object status code is (\d+)$`, headObjectStatusCodeIs)

	s.Step(`^options object "(.{1,})" with method "([^"]*)" and origin "([^"]*)"$`, optionsObjectWithMethodAndOrigin)
	s.Step(`^options object status code is (\d+)$`, optionsObjectStatusCodeIs)

	s.Step(`^delete object with key "(.{1,})"$`, deleteObjectWithKey)
	s.Step(`^delete object status code is (\d+)$`, deleteObjectStatusCodeIs)
	s.Step(`^delete the move object with key "(.{1,})"$`, deleteTheMoveObjectWithKey)
	s.Step(`^delete the move object status code is (\d+)$`, deleteTheMoveObjectStatusCodeIs)
}

// --------------------------------------------------------------------------
var putObjectOutput *qs.PutObjectOutput

func putObjectWithKey(objectKey string) error {
	_, err = exec.Command("dd", "if=/dev/zero", "of=/tmp/sdk_bin", "bs=1024", "count=1").Output()
	if err != nil {
		return err
	}
	defer os.Remove("/tmp/sdk_bin")

	file, err := os.Open("/tmp/sdk_bin")
	if err != nil {
		return err
	}
	defer file.Close()

	hash := md5.New()
	_, err = io.Copy(hash, file)
	hashInBytes := hash.Sum(nil)[:16]
	md5String := hex.EncodeToString(hashInBytes)

	//file.Seek(0, io.SeekStart)
	file.Seek(0, 0)
	putObjectOutput, err = bucket.PutObject(objectKey, &qs.PutObjectInput{
		ContentType: qs.String("text/plain"),
		ContentMD5:  qs.String(md5String),
		Body:        file,
	})
	return err
}

func putObjectStatusCodeIs(statusCode int) error {
	return checkEqual(qs.IntValue(putObjectOutput.StatusCode), statusCode)
}

// --------------------------------------------------------------------------
var copyObjectOutput *qs.PutObjectOutput

func copyObjectWithKey(objectKey string) error {
	copyObjectKey := fmt.Sprintf(`%s_copy`, objectKey)
	copyObjectOutput, err = bucket.PutObject(copyObjectKey, &qs.PutObjectInput{
		XQSCopySource: qs.String(fmt.Sprintf(`/%s/%s`, tc.BucketName, objectKey)),
	})
	return err
}

func copyObjectStatusCodeIs(statusCode int) error {
	return checkEqual(qs.IntValue(copyObjectOutput.StatusCode), statusCode)
}

// --------------------------------------------------------------------------
var moveObjectOutput *qs.PutObjectOutput

func moveObjectWithKey(objectKey string) error {
	copyObjectKey := fmt.Sprintf(`%s_copy`, objectKey)
	moveObjectKey := fmt.Sprintf(`%s_move`, objectKey)
	moveObjectOutput, err = bucket.PutObject(moveObjectKey, &qs.PutObjectInput{
		XQSMoveSource: qs.String(fmt.Sprintf(`/%s/%s`, tc.BucketName, copyObjectKey)),
	})
	return err
}

func moveObjectStatusCodeIs(statusCode int) error {
	return checkEqual(qs.IntValue(moveObjectOutput.StatusCode), statusCode)
}

// --------------------------------------------------------------------------

var getObjectOutput *qs.GetObjectOutput

func getObjectWithKey(objectKey string) error {
	getObjectOutput, err = bucket.GetObject(objectKey, nil)
	return err
}

func getObjectStatusCodeIs(statusCode int) error {
	return checkEqual(qs.IntValue(getObjectOutput.StatusCode), statusCode)
}

func getObjectContentLengthIs(length int) error {
	buffer := &bytes.Buffer{}
	buffer.ReadFrom(getObjectOutput.Body)
	getObjectOutput.Body.Close()
	return checkEqual(len(buffer.Bytes())*1024, length)
}

// --------------------------------------------------------------------------

var getObjectWithContentTypeRequest *request.Request

func getObjectWithContentType(objectKey, contentType string) error {
	getObjectWithContentTypeRequest, _, err = bucket.GetObjectRequest(
		objectKey,
		&qs.GetObjectInput{
			ResponseContentType: qs.String(contentType),
		},
	)
	if err != nil {
		return err
	}
	err = getObjectWithContentTypeRequest.Send()
	if err != nil {
		return err
	}
	return nil
}

func getObjectContentTypeIs(contentType string) error {
	return checkEqual(getObjectWithContentTypeRequest.HTTPResponse.Header.Get("Content-Type"), contentType)
}

// --------------------------------------------------------------------------

var getObjectWithQuerySignatureURL string

func getObjectWithQuerySignature(objectKey string) error {
	r, _, err := bucket.GetObjectRequest(objectKey, nil)
	if err != nil {
		return err
	}

	err = r.SignQuery(10)
	if err != nil {
		return err
	}

	getObjectWithQuerySignatureURL = r.HTTPRequest.URL.String()
	return nil
}

func getObjectWithQuerySignatureContentLengthIs(length int) error {
	out, err := http.Get(getObjectWithQuerySignatureURL)
	if err != nil {
		return err
	}
	buffer := &bytes.Buffer{}
	buffer.ReadFrom(out.Body)
	out.Body.Close()

	return checkEqual(len(buffer.Bytes())*1024, length)
}

// --------------------------------------------------------------------------

var headObjectOutput *qs.HeadObjectOutput

func headObjectWithKey(objectKey string) error {
	headObjectOutput, err = bucket.HeadObject(objectKey, nil)
	return err
}

func headObjectStatusCodeIs(statusCode int) error {
	return checkEqual(qs.IntValue(headObjectOutput.StatusCode), statusCode)
}

// --------------------------------------------------------------------------

var optionsObjectOutput *qs.OptionsObjectOutput

func optionsObjectWithMethodAndOrigin(objectKey, method, origin string) error {
	optionsObjectOutput, err = bucket.OptionsObject(
		objectKey,
		&qs.OptionsObjectInput{
			AccessControlRequestMethod: qs.String(method),
			Origin: qs.String(origin),
		},
	)
	return err
}

func optionsObjectStatusCodeIs(statusCode int) error {
	return checkEqual(qs.IntValue(optionsObjectOutput.StatusCode), statusCode)
}

// --------------------------------------------------------------------------

var deleteObjectOutput *qs.DeleteObjectOutput
var deleteTheMoveObjectOutput *qs.DeleteObjectOutput

func deleteObjectWithKey(objectKey string) error {
	deleteObjectOutput, err = bucket.DeleteObject(objectKey)
	return err
}

func deleteObjectStatusCodeIs(statusCode int) error {
	return checkEqual(qs.IntValue(deleteObjectOutput.StatusCode), statusCode)
}

func deleteTheMoveObjectWithKey(objectKey string) error {
	deleteTheMoveObjectOutput, err = bucket.DeleteObject(fmt.Sprintf(`%s_move`, objectKey))
	return err
}

func deleteTheMoveObjectStatusCodeIs(statusCode int) error {
	return checkEqual(qs.IntValue(deleteTheMoveObjectOutput.StatusCode), statusCode)
}
<commit_msg>Add concurrency test for objects<commit_after>// +-------------------------------------------------------------------------
// | Copyright (C) 2016 Yunify, Inc.
// +-------------------------------------------------------------------------
// | Licensed under the Apache License, Version 2.0 (the "License");
// | you may not use this work except in compliance with the License.
// | You may obtain a copy of the License in the LICENSE file, or at:
// |
// | http://www.apache.org/licenses/LICENSE-2.0
// |
// | Unless required by applicable law or agreed to in writing, software
// | distributed under the License is distributed on an "AS IS" BASIS,
// | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// | See the License for the specific language governing permissions and
// | limitations under the License.
// +-------------------------------------------------------------------------

package main

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"sync"

	"github.com/DATA-DOG/godog"

	"github.com/yunify/qingstor-sdk-go/request"
	qs "github.com/yunify/qingstor-sdk-go/service"
)

// ObjectFeatureContext provides feature context for object.
func ObjectFeatureContext(s *godog.Suite) {
	s.Step(`^put object with key "(.{1,})"$`, putObjectWithKey)
	s.Step(`^put object status code is (\d+)$`, putObjectStatusCodeIs)

	s.Step(`^copy object with key "(.{1,})"$`, copyObjectWithKey)
	s.Step(`^copy object status code is (\d+)$`, copyObjectStatusCodeIs)

	s.Step(`^move object with key "(.{1,})"$`, moveObjectWithKey)
	s.Step(`^move object status code is (\d+)$`, moveObjectStatusCodeIs)

	s.Step(`^get object with key "(.{1,})"$`, getObjectWithKey)
	s.Step(`^get object status code is (\d+)$`, getObjectStatusCodeIs)
	s.Step(`^get object content length is (\d+)$`, getObjectContentLengthIs)

	s.Step(`^get object "(.{1,})" with content type "(.{1,})"$`, getObjectWithContentType)
	s.Step(`^get object content type is "(.{1,})"$`, getObjectContentTypeIs)

	s.Step(`^get object "(.{1,})" with query signature$`, getObjectWithQuerySignature)
	s.Step(`^get object with query signature content length is (\d+)$`, getObjectWithQuerySignatureContentLengthIs)

	s.Step(`^head object with key "(.{1,})"$`, headObjectWithKey)
	s.Step(`^head object status code is (\d+)$`, headObjectStatusCodeIs)

	s.Step(`^options object "(.{1,})" with method "([^"]*)" and origin "([^"]*)"$`, optionsObjectWithMethodAndOrigin)
	s.Step(`^options object status code is (\d+)$`, optionsObjectStatusCodeIs)

	s.Step(`^delete object with key "(.{1,})"$`, deleteObjectWithKey)
	s.Step(`^delete object status code is (\d+)$`, deleteObjectStatusCodeIs)
	s.Step(`^delete the move object with key "(.{1,})"$`, deleteTheMoveObjectWithKey)
	s.Step(`^delete the move object status code is (\d+)$`, deleteTheMoveObjectStatusCodeIs)
}

// --------------------------------------------------------------------------
const concurrency = 16

var putObjectOutputs [concurrency]*qs.PutObjectOutput

func putObjectWithKey(objectKey string) error {
	_, err = exec.Command("dd", "if=/dev/zero", "of=/tmp/sdk_bin", "bs=1024", "count=1").Output()
	if err != nil {
		return err
	}
	defer os.Remove("/tmp/sdk_bin")

	errChan := make(chan error, concurrency)

	wg := sync.WaitGroup{}
	wg.Add(concurrency)
	for i := 0; i < concurrency; i++ {
		go func(index int, errChan chan<- error) {
			wg.Done()

			file, err := os.Open("/tmp/sdk_bin")
			if err != nil {
				errChan <- err
				return
			}
			defer file.Close()

			hash := md5.New()
			_, err = io.Copy(hash, file)
			if err != nil {
				errChan <- err
				return
			}
			hashInBytes := hash.Sum(nil)[:16]
			md5String := hex.EncodeToString(hashInBytes)

			//file.Seek(0, io.SeekStart)
			file.Seek(0, 0)
			if len(objectKey) > 1000 {
				objectKey = objectKey[:1000]
			}
			putObjectOutput, err := bucket.PutObject(
				fmt.Sprintf("%s-%d", objectKey, index),
				&qs.PutObjectInput{
					ContentType: qs.String("text/plain"),
					ContentMD5:  qs.String(md5String),
					Body:        file,
				},
			)
			if err != nil {
				errChan <- err
				return
			}
			putObjectOutputs[index] = putObjectOutput
			errChan <- nil
			return
		}(i, errChan)
	}
	wg.Wait()

	for i := 0; i < concurrency; i++ {
		err = <-errChan
		if err != nil {
			return err
		}
	}
	return nil
}

func putObjectStatusCodeIs(statusCode int) error {
	for _, output := range putObjectOutputs {
		err = checkEqual(qs.IntValue(output.StatusCode), statusCode)
		if err != nil {
			return err
		}
	}
	return nil
}

// --------------------------------------------------------------------------
var copyObjectOutputs [concurrency]*qs.PutObjectOutput

func copyObjectWithKey(objectKey string) error {
	errChan := make(chan error, concurrency)

	wg := sync.WaitGroup{}
	wg.Add(concurrency)
	for i := 0; i < concurrency; i++ {
		go func(index int, errChan chan<- error) {
			wg.Done()

			if len(objectKey) > 1000 {
				objectKey = objectKey[:1000]
			}
			copyObjectOutput, err := bucket.PutObject(
				fmt.Sprintf("%s-%d-copy", objectKey, index),
				&qs.PutObjectInput{
					XQSCopySource: qs.String(
						fmt.Sprintf("/%s/%s-%d", tc.BucketName, objectKey, index),
					),
				})
			if err != nil {
				errChan <- err
				return
			}
			copyObjectOutputs[index] = copyObjectOutput
			errChan <- nil
			return
		}(i, errChan)
	}
	wg.Wait()

	for i := 0; i < concurrency; i++ {
		err = <-errChan
		if err != nil {
			return err
		}
	}
	return nil
}

func copyObjectStatusCodeIs(statusCode int) error {
	for _, output := range copyObjectOutputs {
		err = checkEqual(qs.IntValue(output.StatusCode), statusCode)
		if err != nil {
			return err
		}
	}
	return nil
}

// --------------------------------------------------------------------------
var moveObjectOutputs [concurrency]*qs.PutObjectOutput

func moveObjectWithKey(objectKey string) error {
	errChan := make(chan error, concurrency)

	wg := sync.WaitGroup{}
	wg.Add(concurrency)
	for i := 0; i < concurrency; i++ {
		go func(index int, errChan chan<- error) {
			wg.Done()

			if len(objectKey) > 1000 {
				objectKey = objectKey[:1000]
			}
			moveObjectOutput, err := bucket.PutObject(
				fmt.Sprintf("%s-%d-move", objectKey, index),
				&qs.PutObjectInput{
					XQSMoveSource: qs.String(
						fmt.Sprintf(`/%s/%s-%d-copy`, tc.BucketName, objectKey, index),
					),
				})
			if err != nil {
				errChan <- err
				return
			}
			moveObjectOutputs[index] = moveObjectOutput
			errChan <- nil
			return
		}(i, errChan)
	}
	wg.Wait()

	for i := 0; i < concurrency; i++ {
		err = <-errChan
		if err != nil {
			return err
		}
	}
	return nil
}

func moveObjectStatusCodeIs(statusCode int) error {
	for _, output := range moveObjectOutputs {
		err = checkEqual(qs.IntValue(output.StatusCode), statusCode)
		if err != nil {
			return err
		}
	}
	return nil
}

// --------------------------------------------------------------------------

var getObjectOutputs [concurrency]*qs.GetObjectOutput

func getObjectWithKey(objectKey string) error {
	errChan := make(chan error, concurrency)

	wg := sync.WaitGroup{}
	wg.Add(concurrency)
	for i := 0; i < concurrency; i++ {
		go func(index int, errChan chan<- error) {
			wg.Done()

			if len(objectKey) > 1000 {
				objectKey = objectKey[:1000]
			}
			getObjectOutput, err := bucket.GetObject(
				fmt.Sprintf("%s-%d", objectKey, index), nil,
			)
			if err != nil {
				errChan <- err
				return
			}
			getObjectOutputs[index] = getObjectOutput
			errChan <- nil
			return
		}(i, errChan)
	}
	wg.Wait()

	for i := 0; i < concurrency; i++ {
		err = <-errChan
		if err != nil {
			return err
		}
	}
	return nil
}

func getObjectStatusCodeIs(statusCode int) error {
	for _, output := range getObjectOutputs {
		err = checkEqual(qs.IntValue(output.StatusCode), statusCode)
		if err != nil {
			return err
		}
	}
	return nil
}

func getObjectContentLengthIs(length int) error {
	buffer := &bytes.Buffer{}
	for _, output := range getObjectOutputs {
		buffer.Truncate(0)
		buffer.ReadFrom(output.Body)
		err = checkEqual(len(buffer.Bytes())*1024, length)
		if err != nil {
			return err
		}
	}
	return nil
}

// --------------------------------------------------------------------------

var getObjectWithContentTypeRequests [concurrency]*request.Request

func getObjectWithContentType(objectKey, contentType string) error {
	errChan := make(chan error, concurrency)

	wg := sync.WaitGroup{}
	wg.Add(concurrency)
	for i := 0; i < concurrency; i++ {
		go func(index int, errChan chan<- error) {
			wg.Done()

			if len(objectKey) > 1000 {
				objectKey = objectKey[:1000]
			}
			getObjectWithContentTypeRequest, _, err := bucket.GetObjectRequest(
				fmt.Sprintf("%s-%d", objectKey, index),
				&qs.GetObjectInput{
					ResponseContentType: qs.String(contentType),
				},
			)
			if err != nil {
				errChan <- err
				return
			}
			err = getObjectWithContentTypeRequest.Send()
			if err != nil {
				errChan <- err
				return
			}
			err = getObjectWithContentTypeRequest.Send()
			if err != nil {
				errChan <- err
				return
			}
			getObjectWithContentTypeRequests[index] = getObjectWithContentTypeRequest
			errChan <- nil
			return
		}(i, errChan)
	}
	wg.Wait()

	for i := 0; i < concurrency; i++ {
		err = <-errChan
		if err != nil {
			return err
		}
	}
	return nil
}

func getObjectContentTypeIs(contentType string) error {
	for _, r := range getObjectWithContentTypeRequests {
		err = checkEqual(r.HTTPResponse.Header.Get("Content-Type"), contentType)
		if err != nil {
			return err
		}
	}
	return nil
}

// --------------------------------------------------------------------------

var getObjectWithQuerySignatureURLs [concurrency]string

func getObjectWithQuerySignature(objectKey string) error {
	errChan := make(chan error, concurrency)

	wg := sync.WaitGroup{}
	wg.Add(concurrency)
	for i := 0; i < concurrency; i++ {
		go func(index int, errChan chan<- error) {
			wg.Done()

			if len(objectKey) > 1000 {
				objectKey = objectKey[:1000]
			}
			r, _, err := bucket.GetObjectRequest(
				fmt.Sprintf("%s-%d", objectKey, index), nil,
			)
			if err != nil {
				errChan <- err
				return
			}
			err = r.SignQuery(10)
			if err != nil {
				errChan <- err
				return
			}

			getObjectWithQuerySignatureURLs[index] = r.HTTPRequest.URL.String()
			errChan <- nil
			return
		}(i, errChan)
	}
	wg.Wait()

	for i := 0; i < concurrency; i++ {
		err = <-errChan
		if err != nil {
			return err
		}
	}
	return nil
}

func getObjectWithQuerySignatureContentLengthIs(length int) error {
	buffer := &bytes.Buffer{}
	for _, url := range getObjectWithQuerySignatureURLs {
		out, err := http.Get(url)
		if err != nil {
			return err
		}
		buffer.Truncate(0)
		buffer.ReadFrom(out.Body)
		out.Body.Close()
		err = checkEqual(len(buffer.Bytes())*1024, length)
		if err != nil {
			return err
		}
	}
	return nil
}

// --------------------------------------------------------------------------

var headObjectOutputs [concurrency]*qs.HeadObjectOutput

func headObjectWithKey(objectKey string) error {
	errChan := make(chan error, concurrency)

	wg := sync.WaitGroup{}
	wg.Add(concurrency)
	for i := 0; i < concurrency; i++ {
		go func(index int, errChan chan<- error) {
			wg.Done()

			if len(objectKey) > 1000 {
				objectKey = objectKey[:1000]
			}
			headObjectOutput, err := bucket.HeadObject(
				fmt.Sprintf("%s-%d", objectKey, index), nil,
			)
			if err != nil {
				errChan <- err
				return
			}
			headObjectOutputs[index] = headObjectOutput
			errChan <- nil
			return
		}(i, errChan)
	}
	wg.Wait()

	for i := 0; i < concurrency; i++ {
		err = <-errChan
		if err != nil {
			return err
		}
	}
	return nil
}

func headObjectStatusCodeIs(statusCode int) error {
	for _, output := range headObjectOutputs {
		err = checkEqual(qs.IntValue(output.StatusCode), statusCode)
		if err != nil {
			return err
		}
	}
	return nil
}

// --------------------------------------------------------------------------

var optionsObjectOutputs [concurrency]*qs.OptionsObjectOutput

func optionsObjectWithMethodAndOrigin(objectKey, method, origin string) error {
	errChan := make(chan error, concurrency)

	wg := sync.WaitGroup{}
	wg.Add(concurrency)
	for i := 0; i < concurrency; i++ {
		go func(index int, errChan chan<- error) {
			wg.Done()

			if len(objectKey) > 1000 {
				objectKey = objectKey[:1000]
			}
			optionsObjectOutput, err := bucket.OptionsObject(
				fmt.Sprintf("%s-%d", objectKey, index),
				&qs.OptionsObjectInput{
					AccessControlRequestMethod: qs.String(method),
					Origin: qs.String(origin),
				},
			)
			if err != nil {
				errChan <- err
				return
			}
			optionsObjectOutputs[index] = optionsObjectOutput
			errChan <- nil
			return
		}(i, errChan)
	}
	wg.Wait()

	for i := 0; i < concurrency; i++ {
		err = <-errChan
		if err != nil {
			return err
		}
	}
	return nil
}

func optionsObjectStatusCodeIs(statusCode int) error {
	for _, output := range optionsObjectOutputs {
		err = checkEqual(qs.IntValue(output.StatusCode), statusCode)
		if err != nil {
			return err
		}
	}
	return nil
}

// --------------------------------------------------------------------------

var deleteObjectOutputs [concurrency]*qs.DeleteObjectOutput
var deleteTheMoveObjectOutputs [concurrency]*qs.DeleteObjectOutput

func deleteObjectWithKey(objectKey string) error {
	errChan := make(chan error, concurrency)

	wg := sync.WaitGroup{}
	wg.Add(concurrency)
	for i := 0; i < concurrency; i++ {
		go func(index int, errChan chan<- error) {
			wg.Done()

			if len(objectKey) > 1000 {
				objectKey = objectKey[:1000]
			}
			deleteObjectOutput, err := bucket.DeleteObject(
				fmt.Sprintf("%s-%d", objectKey, index),
			)
			if err != nil {
				errChan <- err
				return
			}
			deleteObjectOutputs[index] = deleteObjectOutput
			errChan <- nil
			return
		}(i, errChan)
	}
	wg.Wait()

	for i := 0; i < concurrency; i++ {
		err = <-errChan
		if err != nil {
			return err
		}
	}
	return nil
}

func deleteObjectStatusCodeIs(statusCode int) error {
	for _, output := range deleteObjectOutputs {
		err = checkEqual(qs.IntValue(output.StatusCode), statusCode)
		if err != nil {
			return err
		}
	}
	return nil
}

func deleteTheMoveObjectWithKey(objectKey string) error {
	errChan := make(chan error, concurrency)

	wg := sync.WaitGroup{}
	wg.Add(concurrency)
	for i := 0; i < concurrency; i++ {
		go func(index int, errChan chan<- error) {
			wg.Done()

			if len(objectKey) > 1000 {
				objectKey = objectKey[:1000]
			}
			deleteTheMoveObjectOutput, err := bucket.DeleteObject(
				fmt.Sprintf("%s-%d-move", objectKey, index),
			)
			if err != nil {
				errChan <- err
				return
			}
			deleteTheMoveObjectOutputs[index] = deleteTheMoveObjectOutput
			errChan <- nil
			return
		}(i, errChan)
	}
	wg.Wait()

	for i := 0; i < concurrency; i++ {
		err = <-errChan
		if err != nil {
			return err
		}
	}
	return nil
}

func deleteTheMoveObjectStatusCodeIs(statusCode int) error {
	for _, output := range deleteTheMoveObjectOutputs {
		err = checkEqual(qs.IntValue(output.StatusCode), statusCode)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net/http"
	"os"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/trusts"
	tokens3 "github.com/gophercloud/gophercloud/openstack/identity/v3/tokens"
	"gopkg.in/gcfg.v1"

	"github.com/golang/glog"
	netutil "k8s.io/apimachinery/pkg/util/net"
	certutil "k8s.io/client-go/util/cert"
)

type Config struct {
	Global struct {
		CinderEndpoint  string `gcfg:"cinder-endpoint"`
		AuthUrl         string `gcfg:"auth-url"`
		Username        string
		UserId          string `gcfg:"user-id"`
		Password        string
		TenantId        string `gcfg:"tenant-id"`
		TenantName      string `gcfg:"tenant-name"`
		TrustId         string `gcfg:"trust-id"`
		DomainId        string `gcfg:"domain-id"`
		DomainName      string `gcfg:"domain-name"`
		Region          string
		CAFile          string `gcfg:"ca-file"`
       }
}

func (cfg Config) toAuthOptions() gophercloud.AuthOptions {
	return gophercloud.AuthOptions{
		IdentityEndpoint: cfg.Global.AuthUrl,
		Username:         cfg.Global.Username,
		UserID:           cfg.Global.UserId,
		Password:         cfg.Global.Password,
		TenantID:         cfg.Global.TenantId,
		TenantName:       cfg.Global.TenantName,
		DomainID:         cfg.Global.DomainId,
		DomainName:       cfg.Global.DomainName,

		// Persistent service, so we need to be able to renew tokens.
		AllowReauth: true,
	}
}

func (cfg Config) toAuth3Options() tokens3.AuthOptions {
	return tokens3.AuthOptions{
		IdentityEndpoint: cfg.Global.AuthUrl,
		Username:         cfg.Global.Username,
		UserID:           cfg.Global.UserId,
		Password:         cfg.Global.Password,
		DomainID:         cfg.Global.DomainId,
		DomainName:       cfg.Global.DomainName,
		AllowReauth:      true,
	}
}

func getConfig(configFilePath string) (Config, error) {
	if configFilePath != "" {
		var configFile *os.File
		var config Config
		configFile, err := os.Open(configFilePath)
		if err != nil {
			glog.Fatalf("Couldn't open configuration %s: %#v",
				configFilePath, err)
			return Config{}, err
		}

		defer configFile.Close()

		err = gcfg.ReadInto(&config, configFile)
		if err != nil {
			glog.Fatalf("Couldn't read configuration: %#v", err)
			return Config{}, err
		}
		return config, nil

	} else {
		// Pass explicit nil so plugins can actually check for nil. See
		// "Why is my nil error value not equal to nil?" in golang.org/doc/faq.
		glog.Fatal("No config file path specified")
		return Config{}, errors.New("Missing configuration")
	}
}

func getStandaloneVolumeService(cfg Config) (*gophercloud.ServiceClient, error) {
	opts := gophercloud.NoAuthOptions{
		Username:   cfg.Global.Username,
		TenantName: cfg.Global.TenantName,
	}
	provider, err := openstack.UnAuthenticatedClient(opts)
	if err != nil {
		glog.Fatalf("Unable to initialize noauth client: %#v", err)
		return nil, err
	}

	volumeService, err := openstack.NewBlockStorageV2(provider, gophercloud.EndpointOpts{
		Region: cfg.Global.Region,
		CinderEndpoint: cfg.Global.CinderEndpoint,
	})
	if err != nil {
		glog.Fatalf("Unable to get volume service: %#v", err)
		return nil, err
	}
	return volumeService, nil
}

func getKeystoneVolumeService(cfg Config) (*gophercloud.ServiceClient, error) {
	provider, err := openstack.NewClient(cfg.Global.AuthUrl)
	if err != nil {
		return nil, err
	}
	if cfg.Global.CAFile != "" {
		roots, err := certutil.NewPool(cfg.Global.CAFile)
		if err != nil {
			return nil, err
		}
		config := &tls.Config{}
		config.RootCAs = roots
		provider.HTTPClient.Transport = netutil.SetOldTransportDefaults(&http.Transport{TLSClientConfig: config})

	}
	if cfg.Global.TrustId != "" {
		opts := cfg.toAuth3Options()
		authOptsExt := trusts.AuthOptsExt{
			TrustID:            cfg.Global.TrustId,
			AuthOptionsBuilder: &opts,
		}
		err = openstack.AuthenticateV3(provider, authOptsExt, gophercloud.EndpointOpts{})
	} else {
		err = openstack.Authenticate(provider, cfg.toAuthOptions())
	}

	if err != nil {
		return nil, err
	}

	volumeService, err := openstack.NewBlockStorageV2(provider,
		gophercloud.EndpointOpts{
			Region: cfg.Global.Region,
		})
	if err != nil {
		return nil, fmt.Errorf("failed to get volume service: %v", err)
	}
	return volumeService, nil
}

func getVolumeService(configFilePath string) (*gophercloud.ServiceClient, error) {
	config, err := getConfig(configFilePath)
	if err != nil {
		return nil, err
	}

	if config.Global.CinderEndpoint != "" {
		return getStandaloneVolumeService(config)
	} else {
		return getKeystoneVolumeService(config)
	}
}
<commit_msg>Disable standalone cinder<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net/http"
	"os"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/trusts"
	tokens3 "github.com/gophercloud/gophercloud/openstack/identity/v3/tokens"
	"gopkg.in/gcfg.v1"

	"github.com/golang/glog"
	netutil "k8s.io/apimachinery/pkg/util/net"
	certutil "k8s.io/client-go/util/cert"
)

type Config struct {
	Global struct {
		CinderEndpoint  string `gcfg:"cinder-endpoint"`
		AuthUrl         string `gcfg:"auth-url"`
		Username        string
		UserId          string `gcfg:"user-id"`
		Password        string
		TenantId        string `gcfg:"tenant-id"`
		TenantName      string `gcfg:"tenant-name"`
		TrustId         string `gcfg:"trust-id"`
		DomainId        string `gcfg:"domain-id"`
		DomainName      string `gcfg:"domain-name"`
		Region          string
		CAFile          string `gcfg:"ca-file"`
       }
}

func (cfg Config) toAuthOptions() gophercloud.AuthOptions {
	return gophercloud.AuthOptions{
		IdentityEndpoint: cfg.Global.AuthUrl,
		Username:         cfg.Global.Username,
		UserID:           cfg.Global.UserId,
		Password:         cfg.Global.Password,
		TenantID:         cfg.Global.TenantId,
		TenantName:       cfg.Global.TenantName,
		DomainID:         cfg.Global.DomainId,
		DomainName:       cfg.Global.DomainName,

		// Persistent service, so we need to be able to renew tokens.
		AllowReauth: true,
	}
}

func (cfg Config) toAuth3Options() tokens3.AuthOptions {
	return tokens3.AuthOptions{
		IdentityEndpoint: cfg.Global.AuthUrl,
		Username:         cfg.Global.Username,
		UserID:           cfg.Global.UserId,
		Password:         cfg.Global.Password,
		DomainID:         cfg.Global.DomainId,
		DomainName:       cfg.Global.DomainName,
		AllowReauth:      true,
	}
}

func getConfig(configFilePath string) (Config, error) {
	if configFilePath != "" {
		var configFile *os.File
		var config Config
		configFile, err := os.Open(configFilePath)
		if err != nil {
			glog.Fatalf("Couldn't open configuration %s: %#v",
				configFilePath, err)
			return Config{}, err
		}

		defer configFile.Close()

		err = gcfg.ReadInto(&config, configFile)
		if err != nil {
			glog.Fatalf("Couldn't read configuration: %#v", err)
			return Config{}, err
		}
		return config, nil

	} else {
		// Pass explicit nil so plugins can actually check for nil. See
		// "Why is my nil error value not equal to nil?" in golang.org/doc/faq.
		glog.Fatal("No config file path specified")
		return Config{}, errors.New("Missing configuration")
	}
}

func getKeystoneVolumeService(cfg Config) (*gophercloud.ServiceClient, error) {
	provider, err := openstack.NewClient(cfg.Global.AuthUrl)
	if err != nil {
		return nil, err
	}
	if cfg.Global.CAFile != "" {
		roots, err := certutil.NewPool(cfg.Global.CAFile)
		if err != nil {
			return nil, err
		}
		config := &tls.Config{}
		config.RootCAs = roots
		provider.HTTPClient.Transport = netutil.SetOldTransportDefaults(&http.Transport{TLSClientConfig: config})

	}
	if cfg.Global.TrustId != "" {
		opts := cfg.toAuth3Options()
		authOptsExt := trusts.AuthOptsExt{
			TrustID:            cfg.Global.TrustId,
			AuthOptionsBuilder: &opts,
		}
		err = openstack.AuthenticateV3(provider, authOptsExt, gophercloud.EndpointOpts{})
	} else {
		err = openstack.Authenticate(provider, cfg.toAuthOptions())
	}

	if err != nil {
		return nil, err
	}

	volumeService, err := openstack.NewBlockStorageV2(provider,
		gophercloud.EndpointOpts{
			Region: cfg.Global.Region,
		})
	if err != nil {
		return nil, fmt.Errorf("failed to get volume service: %v", err)
	}
	return volumeService, nil
}

func getVolumeService(configFilePath string) (*gophercloud.ServiceClient, error) {
	config, err := getConfig(configFilePath)
	if err != nil {
		return nil, err
	}

	if config.Global.CinderEndpoint != "" {
		return nil, errors.New("Standalone cinder is not yet supported")
	} else {
		return getKeystoneVolumeService(config)
	}
}
<|endoftext|>


<commit_before><commit_msg>add error handlers.<commit_after><|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"os"
	"path/filepath"

	"github.com/juju/errors"
)

// TODO(ericsnow) lp-1392876
// Pull these from authoritative sources (see
// github.com/juju/juju/juju/paths, etc.):
const (
	dataDir        = "/var/lib/juju"
	startupDir     = "/etc/init"
	loggingConfDir = "/etc/rsyslog.d"
	logsDir        = "/var/log/juju"
	sshDir         = "/home/ubuntu/.ssh"

	machinesConfs = "jujud-machine-*.conf"
	agentsDir     = "agents"
	agentsConfs   = "machine-*"
	jujuInitConfs = "juju-*.conf"
	loggingConfs  = "*juju.conf"
	toolsDir      = "tools"

	sshIdentFile   = "system-identity"
	nonceFile      = "nonce.txt"
	allMachinesLog = "all-machines.log"
	// TODO(ericsnow) It might not be machine 0...
	machine0Log  = "machine-0.log"
	authKeysFile = "authorized_keys"

	dbStartupConf = "juju-db.conf"
	dbPEM         = "server.pem"
	dbSecret      = "shared-secret"
)

// Paths holds the paths that backups needs.
type Paths struct {
	DataDir string
	LogsDir string
}

// GetFilesToBackUp returns the paths that should be included in the
// backup archive.
func GetFilesToBackUp(rootDir string, paths *Paths) ([]string, error) {
	var glob string

	glob = filepath.Join(rootDir, startupDir, machinesConfs)
	initMachineConfs, err := filepath.Glob(glob)
	if err != nil {
		return nil, errors.Annotate(err, "failed to fetch machine init files")
	}

	glob = filepath.Join(rootDir, startupDir, jujuInitConfs)
	initConfs, err := filepath.Glob(glob)
	if err != nil {
		return nil, errors.Annotate(err, "failed to fetch startup conf files")
	}

	glob = filepath.Join(rootDir, paths.DataDir, agentsDir, agentsConfs)
	agentConfs, err := filepath.Glob(glob)
	if err != nil {
		return nil, errors.Annotate(err, "failed to fetch agent config files")
	}

	glob = filepath.Join(rootDir, loggingConfDir, loggingConfs)
	jujuLogConfs, err := filepath.Glob(glob)
	if err != nil {
		return nil, errors.Annotate(err, "failed to fetch juju log conf files")
	}

	backupFiles := []string{
		filepath.Join(rootDir, paths.DataDir, toolsDir),

		filepath.Join(rootDir, paths.DataDir, sshIdentFile),

		filepath.Join(rootDir, paths.DataDir, dbPEM),
		filepath.Join(rootDir, paths.DataDir, dbSecret),
	}
	backupFiles = append(backupFiles, initMachineConfs...)
	backupFiles = append(backupFiles, agentConfs...)
	backupFiles = append(backupFiles, initConfs...)
	backupFiles = append(backupFiles, jujuLogConfs...)

	// Handle logs (might not exist).
	// TODO(ericsnow) We should consider dropping these entirely.
	allmachines := filepath.Join(rootDir, paths.LogsDir, allMachinesLog)
	if _, err := os.Stat(allmachines); err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Trace(err)
		}
	} else {
		backupFiles = append(backupFiles, allmachines)
	}
	machine0 := filepath.Join(rootDir, paths.LogsDir, machine0Log)
	if _, err := os.Stat(machine0); err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Trace(err)
		}
	} else {
		backupFiles = append(backupFiles, machine0)
	}

	// Handle nonce.txt (might not exist).
	nonce := filepath.Join(rootDir, paths.DataDir, nonceFile)
	if _, err := os.Stat(nonce); err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Trace(err)
		}
	} else {
		backupFiles = append(backupFiles, nonce)
	}

	// Handle user SSH files (might not exist).
	SSHDir := filepath.Join(rootDir, sshDir)
	if _, err := os.Stat(SSHDir); err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Trace(err)
		}
	} else {
		backupFiles = append(backupFiles, filepath.Join(SSHDir, authKeysFile))
	}

	return backupFiles, nil
}
<commit_msg>Log missing files.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"os"
	"path/filepath"

	"github.com/juju/errors"
)

// TODO(ericsnow) lp-1392876
// Pull these from authoritative sources (see
// github.com/juju/juju/juju/paths, etc.):
const (
	dataDir        = "/var/lib/juju"
	startupDir     = "/etc/init"
	loggingConfDir = "/etc/rsyslog.d"
	logsDir        = "/var/log/juju"
	sshDir         = "/home/ubuntu/.ssh"

	machinesConfs = "jujud-machine-*.conf"
	agentsDir     = "agents"
	agentsConfs   = "machine-*"
	jujuInitConfs = "juju-*.conf"
	loggingConfs  = "*juju.conf"
	toolsDir      = "tools"

	sshIdentFile   = "system-identity"
	nonceFile      = "nonce.txt"
	allMachinesLog = "all-machines.log"
	// TODO(ericsnow) It might not be machine 0...
	machine0Log  = "machine-0.log"
	authKeysFile = "authorized_keys"

	dbStartupConf = "juju-db.conf"
	dbPEM         = "server.pem"
	dbSecret      = "shared-secret"
)

// Paths holds the paths that backups needs.
type Paths struct {
	DataDir string
	LogsDir string
}

// GetFilesToBackUp returns the paths that should be included in the
// backup archive.
func GetFilesToBackUp(rootDir string, paths *Paths) ([]string, error) {
	var glob string

	glob = filepath.Join(rootDir, startupDir, machinesConfs)
	initMachineConfs, err := filepath.Glob(glob)
	if err != nil {
		return nil, errors.Annotate(err, "failed to fetch machine init files")
	}

	glob = filepath.Join(rootDir, startupDir, jujuInitConfs)
	initConfs, err := filepath.Glob(glob)
	if err != nil {
		return nil, errors.Annotate(err, "failed to fetch startup conf files")
	}

	glob = filepath.Join(rootDir, paths.DataDir, agentsDir, agentsConfs)
	agentConfs, err := filepath.Glob(glob)
	if err != nil {
		return nil, errors.Annotate(err, "failed to fetch agent config files")
	}

	glob = filepath.Join(rootDir, loggingConfDir, loggingConfs)
	jujuLogConfs, err := filepath.Glob(glob)
	if err != nil {
		return nil, errors.Annotate(err, "failed to fetch juju log conf files")
	}

	backupFiles := []string{
		filepath.Join(rootDir, paths.DataDir, toolsDir),

		filepath.Join(rootDir, paths.DataDir, sshIdentFile),

		filepath.Join(rootDir, paths.DataDir, dbPEM),
		filepath.Join(rootDir, paths.DataDir, dbSecret),
	}
	backupFiles = append(backupFiles, initMachineConfs...)
	backupFiles = append(backupFiles, agentConfs...)
	backupFiles = append(backupFiles, initConfs...)
	backupFiles = append(backupFiles, jujuLogConfs...)

	// Handle logs (might not exist).
	// TODO(ericsnow) We should consider dropping these entirely.
	allmachines := filepath.Join(rootDir, paths.LogsDir, allMachinesLog)
	if _, err := os.Stat(allmachines); err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Trace(err)
		}
		logger.Errorf("skipping missing file %q", allmachines)
	} else {
		backupFiles = append(backupFiles, allmachines)
	}
	machine0 := filepath.Join(rootDir, paths.LogsDir, machine0Log)
	if _, err := os.Stat(machine0); err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Trace(err)
		}
		logger.Errorf("skipping missing file %q", machine0)
	} else {
		backupFiles = append(backupFiles, machine0)
	}

	// Handle nonce.txt (might not exist).
	nonce := filepath.Join(rootDir, paths.DataDir, nonceFile)
	if _, err := os.Stat(nonce); err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Trace(err)
		}
		logger.Errorf("skipping missing file %q", nonce)
	} else {
		backupFiles = append(backupFiles, nonce)
	}

	// Handle user SSH files (might not exist).
	SSHDir := filepath.Join(rootDir, sshDir)
	if _, err := os.Stat(SSHDir); err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Trace(err)
		}
		logger.Errorf("skipping missing dir %q", SSHDir)
	} else {
		backupFiles = append(backupFiles, filepath.Join(SSHDir, authKeysFile))
	}

	return backupFiles, nil
}
<|endoftext|>


<commit_before>package kdtree

import (
	"ellipsoid"
	"errors"
	"fmt"
	"geo"
	"graph"
	"math"
	"mm"
	"path"
)

var (
	e             ellipsoid.Ellipsoid
	clusterKdTree ClusterKdTree
)

func init() {
	e = ellipsoid.Init("WGS84", ellipsoid.Degrees, ellipsoid.Meter, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
}

func LoadKdTree(clusterGraph *graph.ClusterGraph, base string) error {
	// Load the k-d tree of all clusters
	clusterKdTrees := make([]*KdTree, len(clusterGraph.Cluster))
	for i, g := range clusterGraph.Cluster {
		clusterDir := path.Join(base, fmt.Sprintf("/cluster%d", i+1))
		var encodedSteps []uint64
		var encodedCoordinates []int32
		err := mm.Open(path.Join(clusterDir, "kdtree.ftf"), &encodedSteps)
		if err != nil {
			return err
		}
		err = mm.Open(path.Join(clusterDir, "coordinates.ftf"), &encodedCoordinates)
		if err != nil {
			return err
		}
		clusterKdTrees[i] = &KdTree{
			Graph:              g,
			EncodedSteps:       encodedSteps,
			EncodedCoordinates: encodedCoordinates,
		}
	}

	// Load the k-d tree of the overlay graph
	var encodedSteps []uint64
	var encodedCoordinates []int32
	err := mm.Open(path.Join(base, "/overlay/kdtree.ftf"), &encodedSteps)
	if err != nil {
		return err
	}
	err = mm.Open(path.Join(base, "/overlay/coordinates.ftf"), &encodedCoordinates)
	if err != nil {
		return err
	}
	overlayKdTree := &KdTree{
		Graph:              clusterGraph.Overlay.GraphFile,
		EncodedSteps:       encodedSteps,
		EncodedCoordinates: encodedCoordinates,
	}

	// Load the bounding boxes of the clusters
	var bboxesFile []int32
	err = mm.Open(path.Join(base, "bboxes.ftf"), &bboxesFile)
	if err != nil {
		return err
	}
	if len(bboxesFile)/4 != clusterGraph.Overlay.ClusterCount() {
		return errors.New("size of bboxes file does not match cluster count")
	}
	bboxes := make([]geo.BBox, len(bboxesFile)/4)
	for i, _ := range bboxes {
		bboxes[i] = geo.DecodeBBox(bboxesFile[4*i : 4*i+4])
	}

	clusterKdTree = ClusterKdTree{
		Overlay: overlayKdTree,
		Cluster: clusterKdTrees,
		BBoxes:  bboxes,
	}
	return nil
}

var linear = 0

// NearestNeighbor returns -1 if the way is on the overlay graph
// No fail strategy: a nearest point on the overlay graph is always returned if no point
// is found in the clusters.
func NearestNeighbor(x geo.Coordinate, trans graph.Transport) Location {
	// first search on the overlay graph
	overlay := clusterKdTree.Overlay
	bestStepIndex, coordOverlay, foundPoint := binarySearch(overlay, x, 0, overlay.EncodedStepLen()-1,
		true /* compareLat */, trans)
	minDistance, _ := e.To(x.Lat, x.Lng, coordOverlay.Lat, coordOverlay.Lng)

	// then search on all clusters where the point is inside the bounding box of the cluster
	clusterIndex := -1
	for i, b := range clusterKdTree.BBoxes {
		if b.Contains(x) {
			kdTree := clusterKdTree.Cluster[i]
			stepIndex, coord, ok := binarySearch(kdTree, x, 0, kdTree.EncodedStepLen()-1, true /* compareLat */, trans)
			dist, _ := e.To(x.Lat, x.Lng, coord.Lat, coord.Lng)

			if ok && (!foundPoint || dist < minDistance) {
				foundPoint = true
				minDistance = dist
				bestStepIndex = stepIndex
				clusterIndex = i
			}
		}
	}

	if clusterIndex >= 0 {
		kdTree := clusterKdTree.Cluster[clusterIndex]
		return Location{
			Graph:   kdTree.Graph,
			EC:      kdTree.EncodedStep(bestStepIndex),
			Cluster: clusterIndex,
		}
	}
	return Location{
		Graph:   overlay.Graph,
		EC:      overlay.EncodedStep(bestStepIndex),
		Cluster: clusterIndex,
	}
}

// binarySearch in one k-d tree. The index, the coordinate, and if the returned step/vertex
// is accessible are returned.
func binarySearch(kdTree *KdTree, x geo.Coordinate, start, end int, compareLat bool,
	trans graph.Transport) (int, geo.Coordinate, bool) {

	if end <= start {
		// end of the recursion
		startCoord, startAccessible := decodeCoordinate(kdTree, start, trans)
		return start, startCoord, startAccessible
	}

	middle := (end-start)/2 + start
	middleCoord, middleAccessible := decodeCoordinate(kdTree, middle, trans)

	// exact hit
	if middleAccessible && x.Lat == middleCoord.Lat && x.Lng == middleCoord.Lng {
		return middle, middleCoord, middleAccessible
	}

	middleDist := math.Inf(1)
	if middleAccessible {
		middleDist = distance(x, middleCoord)
	}

	// recursion one half and if no accessible point is returned also on the other half
	var left bool
	if compareLat {
		left = x.Lat < middleCoord.Lat
	} else {
		left = x.Lng < middleCoord.Lng
	}
	var recIndex int
	var recCoord geo.Coordinate
	var recAccessible bool
	bothHalfs := false
	if left {
		// left
		recIndex, recCoord, recAccessible = binarySearch(kdTree, x, start, middle-1, !compareLat, trans)
		if !recAccessible {
			// other half -> right
			recIndex, recCoord, recAccessible = binarySearch(kdTree, x, middle+1, end, !compareLat, trans)
			bothHalfs = true
		}
	} else {
		// right
		recIndex, recCoord, recAccessible = binarySearch(kdTree, x, middle+1, end, !compareLat, trans)
		if !recAccessible {
			// other half -> left
			recIndex, recCoord, recAccessible = binarySearch(kdTree, x, start, middle-1, !compareLat, trans)
			bothHalfs = true
		}
	}
	bestDistance := distance(x, recCoord)

	// we are finished if both have already been searched
	if bothHalfs {
		if middleDist < bestDistance {
			return middle, middleCoord, middleAccessible
		} else {
			return recIndex, recCoord, recAccessible
		}
	}

	distToPlane := 0.0
	if compareLat {
		distToPlane = (x.Lat - middleCoord.Lat) * (x.Lat - middleCoord.Lat)
	} else {
		distToPlane = (x.Lng - middleCoord.Lng) * (x.Lng - middleCoord.Lng)
	}

	var recIndex2 int
	var recCoord2 geo.Coordinate
	recAccessible2 := false
	// test whether the current best distance circle crosses the plane
	// -10m to account for possible inaccuracies
	if bestDistance >= distToPlane-10 {
		// search on the other half
		if !left {
			// left
			recIndex2, recCoord2, recAccessible2 = binarySearch(kdTree, x, start, middle-1, !compareLat, trans)
		} else {
			// right
			recIndex2, recCoord2, recAccessible2 = binarySearch(kdTree, x, middle+1, end, !compareLat, trans)
		}
	}

	bestDistance2 := math.Inf(1)
	if recAccessible2 {
		bestDistance2 = distance(x, recCoord2)
	}

	if bestDistance < bestDistance2 {
		if middleDist < bestDistance {
			return middle, middleCoord, middleAccessible
		} else {
			return recIndex, recCoord, recAccessible
		}
	}
	if middleDist < bestDistance2 {
		return middle, middleCoord, middleAccessible
	}
	return recIndex2, recCoord2, recAccessible2
}

// decodeCoordinate returns the coordinate of the encoded vertex/step and if it is accessible by the
// given transport mode
func decodeCoordinate(t *KdTree, i int, trans graph.Transport) (geo.Coordinate, bool) {
	g := t.Graph
	ec := t.EncodedStep(i)
	coord := geo.DecodeCoordinate(t.EncodedCoordinates[2*i], t.EncodedCoordinates[2*i+1])

	vertexIndex := ec >> (EdgeOffsetBits + StepOffsetBits)
	edgeOffset := uint32((ec >> StepOffsetBits) & MaxEdgeOffset)
	stepOffset := ec & MaxStepOffset
	vertex := graph.Vertex(vertexIndex)

	if edgeOffset == MaxEdgeOffset && stepOffset == MaxStepOffset {
		// it is a vertex and not a step
		return coord, g.VertexAccessible(vertex, trans)
	}
	edge := graph.Edge(g.FirstOut[vertex] + edgeOffset)
	edgeAccessible := g.EdgeAccessible(edge, trans)
	return coord, edgeAccessible
}

// distance returns the squared euclidian distance
func distance(x, y geo.Coordinate) float64 {
	return (x.Lat-y.Lat)*(x.Lat-y.Lat) + (x.Lng-y.Lng)*(x.Lng-y.Lng)
}
<commit_msg>cleaned up and improved nearest neighbor<commit_after>package kdtree

import (
	"ellipsoid"
	"errors"
	"fmt"
	"geo"
	"graph"
	"math"
	"mm"
	"path"
)

var (
	inf           float64
	e             ellipsoid.Ellipsoid
	clusterKdTree ClusterKdTree
)

type NN struct {
	kdTree *KdTree
	x      geo.Coordinate
	trans  graph.Transport
}

func init() {
	inf = math.Inf(1)
	e = ellipsoid.Init("WGS84", ellipsoid.Degrees, ellipsoid.Meter, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)
}

func LoadKdTree(clusterGraph *graph.ClusterGraph, base string) error {
	// Load the k-d tree of all clusters
	clusterKdTrees := make([]*KdTree, len(clusterGraph.Cluster))
	for i, g := range clusterGraph.Cluster {
		clusterDir := path.Join(base, fmt.Sprintf("/cluster%d", i+1))
		var encodedSteps []uint64
		var encodedCoordinates []int32
		err := mm.Open(path.Join(clusterDir, "kdtree.ftf"), &encodedSteps)
		if err != nil {
			return err
		}
		err = mm.Open(path.Join(clusterDir, "coordinates.ftf"), &encodedCoordinates)
		if err != nil {
			return err
		}
		clusterKdTrees[i] = &KdTree{
			Graph:              g,
			EncodedSteps:       encodedSteps,
			EncodedCoordinates: encodedCoordinates,
		}
	}

	// Load the k-d tree of the overlay graph
	var encodedSteps []uint64
	var encodedCoordinates []int32
	err := mm.Open(path.Join(base, "/overlay/kdtree.ftf"), &encodedSteps)
	if err != nil {
		return err
	}
	err = mm.Open(path.Join(base, "/overlay/coordinates.ftf"), &encodedCoordinates)
	if err != nil {
		return err
	}
	overlayKdTree := &KdTree{
		Graph:              clusterGraph.Overlay.GraphFile,
		EncodedSteps:       encodedSteps,
		EncodedCoordinates: encodedCoordinates,
	}

	// Load the bounding boxes of the clusters
	var bboxesFile []int32
	err = mm.Open(path.Join(base, "bboxes.ftf"), &bboxesFile)
	if err != nil {
		return err
	}
	if len(bboxesFile)/4 != clusterGraph.Overlay.ClusterCount() {
		return errors.New("size of bboxes file does not match cluster count")
	}
	bboxes := make([]geo.BBox, len(bboxesFile)/4)
	for i, _ := range bboxes {
		bboxes[i] = geo.DecodeBBox(bboxesFile[4*i : 4*i+4])
	}

	clusterKdTree = ClusterKdTree{
		Overlay: overlayKdTree,
		Cluster: clusterKdTrees,
		BBoxes:  bboxes,
	}
	return nil
}

var linear = 0

// NearestNeighbor returns -1 if the location is on the overlay graph
// No fail strategy: a nearest point on the overlay graph is always returned if no point
// is found in the clusters.
func NearestNeighbor(x geo.Coordinate, trans graph.Transport) Location {
	// first search on the overlay graph
	t := clusterKdTree.Overlay
	nn := &NN{
		kdTree: t,
		x:      x,
		trans:  trans,
	}
	bestStepIndex, coord, foundPoint := nn.binarySearch(0, t.EncodedStepLen()-1, true /* compareLat */)
	minDistance, _ := e.To(x.Lat, x.Lng, coord.Lat, coord.Lng)

	// then search on all clusters where the point is inside the bounding box of the cluster
	clusterIndex := -1
	for i, b := range clusterKdTree.BBoxes {
		if b.Contains(x) {
			t = clusterKdTree.Cluster[i]
			nn = &NN{
				kdTree: t,
				x:      x,
				trans:  trans,
			}
			stepIndex, coord, ok := nn.binarySearch(0, t.EncodedStepLen()-1, true /* compareLat */)
			dist, _ := e.To(x.Lat, x.Lng, coord.Lat, coord.Lng)

			if ok && (!foundPoint || dist < minDistance) {
				foundPoint = true
				minDistance = dist
				bestStepIndex = stepIndex
				clusterIndex = i
			}
		}
	}

	if clusterIndex >= 0 {
		kdTree := clusterKdTree.Cluster[clusterIndex]
		return Location{
			Graph:   kdTree.Graph,
			EC:      kdTree.EncodedStep(bestStepIndex),
			Cluster: clusterIndex,
		}
	}
	return Location{
		Graph:   clusterKdTree.Overlay.Graph,
		EC:      clusterKdTree.Overlay.EncodedStep(bestStepIndex),
		Cluster: clusterIndex,
	}
}

// binarySearch in one k-d tree. The index, the coordinate, and if the returned step/vertex
// is accessible are returned.
func (nn *NN) binarySearch(start, end int, compareLat bool) (int, geo.Coordinate, bool) {
	if end <= start {
		// end of the recursion
		startCoord, startAccessible := nn.decodeCoordinate(start)
		return start, startCoord, startAccessible
	}

	x := nn.x
	middle := (end-start)/2 + start
	middleCoord, middleAccessible := nn.decodeCoordinate(middle)

	// exact hit
	if middleAccessible && x.Lat == middleCoord.Lat && x.Lng == middleCoord.Lng {
		return middle, middleCoord, middleAccessible
	}

	middleDist := inf
	if middleAccessible {
		middleDist = distance(x, middleCoord)
	}

	// recursion one half and if no accessible point is returned also on the other half
	var left bool
	if compareLat {
		left = x.Lat < middleCoord.Lat
	} else {
		left = x.Lng < middleCoord.Lng
	}
	var recIndex int
	var recCoord geo.Coordinate
	var recAccessible bool
	bothHalfs := false
	if left {
		// left
		recIndex, recCoord, recAccessible = nn.binarySearch(start, middle-1, !compareLat)
		if !recAccessible {
			// other half -> right
			recIndex, recCoord, recAccessible = nn.binarySearch(middle+1, end, !compareLat)
			bothHalfs = true
		}
	} else {
		// right
		recIndex, recCoord, recAccessible = nn.binarySearch(middle+1, end, !compareLat)
		if !recAccessible {
			// other half -> left
			recIndex, recCoord, recAccessible = nn.binarySearch(start, middle-1, !compareLat)
			bothHalfs = true
		}
	}
	bestDistance := distance(x, recCoord)

	// we are finished if both have already been searched
	if bothHalfs {
		if middleDist < bestDistance {
			return middle, middleCoord, middleAccessible
		} else {
			return recIndex, recCoord, recAccessible
		}
	}

	distToPlane := 0.0
	if compareLat {
		distToPlane = (x.Lat - middleCoord.Lat) * (x.Lat - middleCoord.Lat)
	} else {
		distToPlane = (x.Lng - middleCoord.Lng) * (x.Lng - middleCoord.Lng)
	}

	var recIndex2 int
	var recCoord2 geo.Coordinate
	recAccessible2 := false
	// Test whether the current best distance circle crosses the plane.
	// We subtract 10 so that even with possible inaccuracies the nearest neighbor and not
	// some near neighbor is found (10 is about 3.16m because of the squared distance).
	if bestDistance >= distToPlane-10 {
		// search on the other half
		if !left {
			// left
			recIndex2, recCoord2, recAccessible2 = nn.binarySearch(start, middle-1, !compareLat)
		} else {
			// right
			recIndex2, recCoord2, recAccessible2 = nn.binarySearch(middle+1, end, !compareLat)
		}
	}

	bestDistance2 := inf
	if recAccessible2 {
		bestDistance2 = distance(x, recCoord2)
	}

	if bestDistance < bestDistance2 {
		if middleDist < bestDistance {
			return middle, middleCoord, middleAccessible
		} else {
			return recIndex, recCoord, recAccessible
		}
	}
	if middleDist < bestDistance2 {
		return middle, middleCoord, middleAccessible
	}
	return recIndex2, recCoord2, recAccessible2
}

// decodeCoordinate returns the coordinate of the encoded vertex/step and if it is accessible by the
// given transport mode
func (nn *NN) decodeCoordinate(i int) (geo.Coordinate, bool) {
	t := nn.kdTree
	g := t.Graph
	// inlining of geo.DecodeCoordinate as the Go compiler does not do it
	coord := geo.Coordinate{
		Lat: float64(t.EncodedCoordinates[2*i]) / geo.OsmPrecision,
		Lng: float64(t.EncodedCoordinates[2*i+1]) / geo.OsmPrecision,
	}

	// decode the index and the offsets
	ec := t.EncodedStep(i)
	vertexIndex := ec >> (EdgeOffsetBits + StepOffsetBits)
	edgeOffset := uint32((ec >> StepOffsetBits) & MaxEdgeOffset)
	stepOffset := ec & MaxStepOffset
	vertex := graph.Vertex(vertexIndex)

	if edgeOffset == MaxEdgeOffset && stepOffset == MaxStepOffset {
		// it is a vertex and not a step
		return coord, g.VertexAccessible(vertex, nn.trans)
	}
	edge := graph.Edge(g.FirstOut[vertex] + edgeOffset)
	edgeAccessible := g.EdgeAccessible(edge, nn.trans)
	return coord, edgeAccessible
}

// distance returns the squared euclidian distance
func distance(x, y geo.Coordinate) float64 {
	return (x.Lat-y.Lat)*(x.Lat-y.Lat) + (x.Lng-y.Lng)*(x.Lng-y.Lng)
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	jujutxn "github.com/juju/txn"
	"gopkg.in/juju/blobstore.v2"
	"gopkg.in/mgo.v2"

	"github.com/juju/juju/state/binarystorage"
)

var binarystorageNew = binarystorage.New

// ToolsStorage returns a new binarystorage.StorageCloser that stores tools
// metadata in the "juju" database "toolsmetadata" collection.
//
// TODO(axw) remove this, add a constructor function in binarystorage.
func (st *State) ToolsStorage() (binarystorage.StorageCloser, error) {
	return newStorage(st.session.Copy(), st.ModelUUID(), toolsmetadataC), nil
}

// GUIStorage returns a new binarystorage.StorageCloser that stores GUI archive
// metadata in the "juju" database "guimetadata" collection.
func (st *State) GUIStorage() (binarystorage.StorageCloser, error) {
	return newStorage(st.session.Copy(), st.controllerTag.Id(), guimetadataC), nil
}

func newStorage(session *mgo.Session, uuid, metadataCollection string) binarystorage.StorageCloser {
	rs := blobstore.NewGridFS(blobstoreDB, uuid, session)
	db := session.DB(jujuDB)
	c := db.C(metadataCollection)
	txnRunner := jujutxn.NewRunner(jujutxn.RunnerParams{Database: db})
	managedStorage := blobstore.NewManagedStorage(db, rs)
	storage := binarystorageNew(uuid, managedStorage, c, txnRunner)
	return &storageCloser{storage, session}
}

type storageCloser struct {
	binarystorage.Storage
	session *mgo.Session
}

func (t *storageCloser) Close() error {
	t.session.Close()
	return nil
}
<commit_msg>newStorage: provide a state object and use session.Clone instead of Copy.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	jujutxn "github.com/juju/txn"
	"gopkg.in/juju/blobstore.v2"
	"gopkg.in/mgo.v2"

	"github.com/juju/juju/state/binarystorage"
)

var binarystorageNew = binarystorage.New

// ToolsStorage returns a new binarystorage.StorageCloser that stores tools
// metadata in the "juju" database "toolsmetadata" collection.
//
// TODO(axw) remove this, add a constructor function in binarystorage.
func (st *State) ToolsStorage() (binarystorage.StorageCloser, error) {
	return newStorage(st, st.ModelUUID(), toolsmetadataC), nil
}

// GUIStorage returns a new binarystorage.StorageCloser that stores GUI archive
// metadata in the "juju" database "guimetadata" collection.
func (st *State) GUIStorage() (binarystorage.StorageCloser, error) {
	return newStorage(st, st.controllerTag.Id(), guimetadataC), nil
}

func newStorage(st *State, uuid, metadataCollection string) binarystorage.StorageCloser {
	session := st.session.Clone()
	rs := blobstore.NewGridFS(blobstoreDB, uuid, session)
	db := session.DB(jujuDB)
	c := db.C(metadataCollection)
	txnRunner := jujutxn.NewRunner(jujutxn.RunnerParams{Database: db})
	managedStorage := blobstore.NewManagedStorage(db, rs)
	storage := binarystorageNew(uuid, managedStorage, c, txnRunner)
	return &storageCloser{storage, session}
}

type storageCloser struct {
	binarystorage.Storage
	session *mgo.Session
}

func (t *storageCloser) Close() error {
	t.session.Close()
	return nil
}
<|endoftext|>


<commit_before>package states

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"sort"
	"strings"

	ctyjson "github.com/zclconf/go-cty/cty/json"

	"github.com/hashicorp/terraform/addrs"
	"github.com/hashicorp/terraform/config/hcl2shim"
)

// String returns a rather-odd string representation of the entire state.
//
// This is intended to match the behavior of the older terraform.State.String
// method that is used in lots of existing tests. It should not be used in
// new tests: instead, use "cmp" to directly compare the state data structures
// and print out a diff if they do not match.
//
// This method should never be used in non-test code, whether directly by call
// or indirectly via a %s or %q verb in package fmt.
func (s *State) String() string {
	if s == nil {
		return "<nil>"
	}

	// sort the modules by name for consistent output
	modules := make([]string, 0, len(s.Modules))
	for m := range s.Modules {
		modules = append(modules, m)
	}
	sort.Strings(modules)

	var buf bytes.Buffer
	for _, name := range modules {
		m := s.Modules[name]
		mStr := m.testString()

		// If we're the root module, we just write the output directly.
		if m.Addr.IsRoot() {
			buf.WriteString(mStr + "\n")
			continue
		}

		// We need to build out a string that resembles the not-quite-standard
		// format that terraform.State.String used to use, where there's a
		// "module." prefix but then just a chain of all of the module names
		// without any further "module." portions.
		buf.WriteString("module")
		for _, step := range m.Addr {
			buf.WriteByte('.')
			buf.WriteString(step.Name)
			if step.InstanceKey != addrs.NoKey {
				buf.WriteByte('[')
				buf.WriteString(step.InstanceKey.String())
				buf.WriteByte(']')
			}
		}
		buf.WriteString(":\n")

		s := bufio.NewScanner(strings.NewReader(mStr))
		for s.Scan() {
			text := s.Text()
			if text != "" {
				text = "  " + text
			}

			buf.WriteString(fmt.Sprintf("%s\n", text))
		}
	}

	return strings.TrimSpace(buf.String())
}

// testString is used to produce part of the output of State.String. It should
// never be used directly.
func (m *Module) testString() string {
	var buf bytes.Buffer

	if len(m.Resources) == 0 {
		buf.WriteString("<no state>")
	}

	// We use AbsResourceInstance here, even though everything belongs to
	// the same module, just because we have a sorting behavior defined
	// for those but not for just ResourceInstance.
	addrsOrder := make([]addrs.AbsResourceInstance, 0, len(m.Resources))
	for _, rs := range m.Resources {
		for ik := range rs.Instances {
			addrsOrder = append(addrsOrder, rs.Addr.Instance(ik).Absolute(addrs.RootModuleInstance))
		}
	}

	sort.Slice(addrsOrder, func(i, j int) bool {
		return addrsOrder[i].Less(addrsOrder[j])
	})

	for _, fakeAbsAddr := range addrsOrder {
		addr := fakeAbsAddr.Resource
		rs := m.Resource(addr.ContainingResource())
		is := m.ResourceInstance(addr)

		// Here we need to fake up a legacy-style address as the old state
		// types would've used, since that's what our tests against those
		// old types expect. The significant difference is that instancekey
		// is dot-separated rather than using index brackets.
		k := addr.ContainingResource().String()
		if addr.Key != addrs.NoKey {
			switch tk := addr.Key.(type) {
			case addrs.IntKey:
				k = fmt.Sprintf("%s.%d", k, tk)
			default:
				// No other key types existed for the legacy types, so we
				// can do whatever we want here. We'll just use our standard
				// syntax for these.
				k = k + tk.String()
			}
		}

		id := legacyInstanceObjectID(is.Current)

		taintStr := ""
		if is.Current != nil && is.Current.Status == ObjectTainted {
			taintStr = " (tainted)"
		}

		deposedStr := ""
		if len(is.Deposed) > 0 {
			deposedStr = fmt.Sprintf(" (%d deposed)", len(is.Deposed))
		}

		buf.WriteString(fmt.Sprintf("%s:%s%s\n", k, taintStr, deposedStr))
		buf.WriteString(fmt.Sprintf("  ID = %s\n", id))
		buf.WriteString(fmt.Sprintf("  provider = %s\n", rs.ProviderConfig.String()))

		// Attributes were a flatmap before, but are not anymore. To preserve
		// our old output as closely as possible we need to do a conversion
		// to flatmap. Normally we'd want to do this with schema for
		// accuracy, but for our purposes here it only needs to be approximate.
		// This should produce an identical result for most cases, though
		// in particular will differ in a few cases:
		//  - The keys used for elements in a set will be different
		//  - Values for attributes of type cty.DynamicPseudoType will be
		//    misinterpreted (but these weren't possible in old world anyway)
		var attributes map[string]string
		if obj := is.Current; obj != nil {
			switch {
			case obj.AttrsFlat != nil:
				// Easy (but increasingly unlikely) case: the state hasn't
				// actually been upgraded to the new form yet.
				attributes = obj.AttrsFlat
			case obj.AttrsJSON != nil:
				ty, err := ctyjson.ImpliedType(obj.AttrsJSON)
				if err == nil {
					val, err := ctyjson.Unmarshal(obj.AttrsJSON, ty)
					if err == nil {
						attributes = hcl2shim.FlatmapValueFromHCL2(val)
					}
				}
			}
		}
		attrKeys := make([]string, 0, len(attributes))
		for ak, _ := range attributes {
			if ak == "id" {
				continue
			}

			attrKeys = append(attrKeys, ak)
		}

		sort.Strings(attrKeys)

		for _, ak := range attrKeys {
			av := attributes[ak]
			buf.WriteString(fmt.Sprintf("  %s = %s\n", ak, av))
		}

		// CAUTION: Since deposed keys are now random strings instead of
		// incrementing integers, this result will not be deterministic
		// if there is more than one deposed object.
		for k, t := range is.Deposed {
			id := legacyInstanceObjectID(t)
			taintStr := ""
			if t.Status == ObjectTainted {
				taintStr = " (tainted)"
			}
			buf.WriteString(fmt.Sprintf("  Deposed ID %s = %s%s\n", k, id, taintStr))
		}

		if obj := is.Current; obj != nil && len(obj.Dependencies) > 0 {
			buf.WriteString(fmt.Sprintf("\n  Dependencies:\n"))
			for _, dep := range obj.Dependencies {
				buf.WriteString(fmt.Sprintf("    %s\n", dep.String()))
			}
		}
	}

	if len(m.OutputValues) > 0 {
		buf.WriteString("\nOutputs:\n\n")

		ks := make([]string, 0, len(m.OutputValues))
		for k := range m.OutputValues {
			ks = append(ks, k)
		}
		sort.Strings(ks)

		for _, k := range ks {
			v := m.OutputValues[k]
			lv := hcl2shim.ConfigValueFromHCL2(v.Value)
			switch vTyped := lv.(type) {
			case string:
				buf.WriteString(fmt.Sprintf("%s = %s\n", k, vTyped))
			case []interface{}:
				buf.WriteString(fmt.Sprintf("%s = %s\n", k, vTyped))
			case map[string]interface{}:
				var mapKeys []string
				for key := range vTyped {
					mapKeys = append(mapKeys, key)
				}
				sort.Strings(mapKeys)

				var mapBuf bytes.Buffer
				mapBuf.WriteString("{")
				for _, key := range mapKeys {
					mapBuf.WriteString(fmt.Sprintf("%s:%s ", key, vTyped[key]))
				}
				mapBuf.WriteString("}")

				buf.WriteString(fmt.Sprintf("%s = %s\n", k, mapBuf.String()))
			}
		}
	}

	return buf.String()
}

// legacyInstanceObjectID is a helper for extracting an object id value from
// an instance object in a way that approximates how we used to do this
// for the old state types. ID is no longer first-class, so this is preserved
// only for compatibility with old tests that include the id as part of their
// expected value.
func legacyInstanceObjectID(obj *ResourceInstanceObjectSrc) string {
	if obj == nil {
		return "<not created>"
	}

	if obj.AttrsJSON != nil {
		type WithID struct {
			ID string `json:"id"`
		}
		var withID WithID
		err := json.Unmarshal(obj.AttrsJSON, &withID)
		if err == nil {
			return withID.ID
		}
	} else if obj.AttrsFlat != nil {
		if flatID, exists := obj.AttrsFlat["id"]; exists {
			return flatID
		}
	}

	// For resource types created after we removed id as special there may
	// not actually be one at all. This is okay because older tests won't
	// encounter this, and new tests shouldn't be using ids.
	return "<none>"
}
<commit_msg>states: Support non-string primitives in state string representation<commit_after>package states

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"sort"
	"strings"

	ctyjson "github.com/zclconf/go-cty/cty/json"

	"github.com/hashicorp/terraform/addrs"
	"github.com/hashicorp/terraform/config/hcl2shim"
)

// String returns a rather-odd string representation of the entire state.
//
// This is intended to match the behavior of the older terraform.State.String
// method that is used in lots of existing tests. It should not be used in
// new tests: instead, use "cmp" to directly compare the state data structures
// and print out a diff if they do not match.
//
// This method should never be used in non-test code, whether directly by call
// or indirectly via a %s or %q verb in package fmt.
func (s *State) String() string {
	if s == nil {
		return "<nil>"
	}

	// sort the modules by name for consistent output
	modules := make([]string, 0, len(s.Modules))
	for m := range s.Modules {
		modules = append(modules, m)
	}
	sort.Strings(modules)

	var buf bytes.Buffer
	for _, name := range modules {
		m := s.Modules[name]
		mStr := m.testString()

		// If we're the root module, we just write the output directly.
		if m.Addr.IsRoot() {
			buf.WriteString(mStr + "\n")
			continue
		}

		// We need to build out a string that resembles the not-quite-standard
		// format that terraform.State.String used to use, where there's a
		// "module." prefix but then just a chain of all of the module names
		// without any further "module." portions.
		buf.WriteString("module")
		for _, step := range m.Addr {
			buf.WriteByte('.')
			buf.WriteString(step.Name)
			if step.InstanceKey != addrs.NoKey {
				buf.WriteByte('[')
				buf.WriteString(step.InstanceKey.String())
				buf.WriteByte(']')
			}
		}
		buf.WriteString(":\n")

		s := bufio.NewScanner(strings.NewReader(mStr))
		for s.Scan() {
			text := s.Text()
			if text != "" {
				text = "  " + text
			}

			buf.WriteString(fmt.Sprintf("%s\n", text))
		}
	}

	return strings.TrimSpace(buf.String())
}

// testString is used to produce part of the output of State.String. It should
// never be used directly.
func (m *Module) testString() string {
	var buf bytes.Buffer

	if len(m.Resources) == 0 {
		buf.WriteString("<no state>")
	}

	// We use AbsResourceInstance here, even though everything belongs to
	// the same module, just because we have a sorting behavior defined
	// for those but not for just ResourceInstance.
	addrsOrder := make([]addrs.AbsResourceInstance, 0, len(m.Resources))
	for _, rs := range m.Resources {
		for ik := range rs.Instances {
			addrsOrder = append(addrsOrder, rs.Addr.Instance(ik).Absolute(addrs.RootModuleInstance))
		}
	}

	sort.Slice(addrsOrder, func(i, j int) bool {
		return addrsOrder[i].Less(addrsOrder[j])
	})

	for _, fakeAbsAddr := range addrsOrder {
		addr := fakeAbsAddr.Resource
		rs := m.Resource(addr.ContainingResource())
		is := m.ResourceInstance(addr)

		// Here we need to fake up a legacy-style address as the old state
		// types would've used, since that's what our tests against those
		// old types expect. The significant difference is that instancekey
		// is dot-separated rather than using index brackets.
		k := addr.ContainingResource().String()
		if addr.Key != addrs.NoKey {
			switch tk := addr.Key.(type) {
			case addrs.IntKey:
				k = fmt.Sprintf("%s.%d", k, tk)
			default:
				// No other key types existed for the legacy types, so we
				// can do whatever we want here. We'll just use our standard
				// syntax for these.
				k = k + tk.String()
			}
		}

		id := legacyInstanceObjectID(is.Current)

		taintStr := ""
		if is.Current != nil && is.Current.Status == ObjectTainted {
			taintStr = " (tainted)"
		}

		deposedStr := ""
		if len(is.Deposed) > 0 {
			deposedStr = fmt.Sprintf(" (%d deposed)", len(is.Deposed))
		}

		buf.WriteString(fmt.Sprintf("%s:%s%s\n", k, taintStr, deposedStr))
		buf.WriteString(fmt.Sprintf("  ID = %s\n", id))
		buf.WriteString(fmt.Sprintf("  provider = %s\n", rs.ProviderConfig.String()))

		// Attributes were a flatmap before, but are not anymore. To preserve
		// our old output as closely as possible we need to do a conversion
		// to flatmap. Normally we'd want to do this with schema for
		// accuracy, but for our purposes here it only needs to be approximate.
		// This should produce an identical result for most cases, though
		// in particular will differ in a few cases:
		//  - The keys used for elements in a set will be different
		//  - Values for attributes of type cty.DynamicPseudoType will be
		//    misinterpreted (but these weren't possible in old world anyway)
		var attributes map[string]string
		if obj := is.Current; obj != nil {
			switch {
			case obj.AttrsFlat != nil:
				// Easy (but increasingly unlikely) case: the state hasn't
				// actually been upgraded to the new form yet.
				attributes = obj.AttrsFlat
			case obj.AttrsJSON != nil:
				ty, err := ctyjson.ImpliedType(obj.AttrsJSON)
				if err == nil {
					val, err := ctyjson.Unmarshal(obj.AttrsJSON, ty)
					if err == nil {
						attributes = hcl2shim.FlatmapValueFromHCL2(val)
					}
				}
			}
		}
		attrKeys := make([]string, 0, len(attributes))
		for ak, _ := range attributes {
			if ak == "id" {
				continue
			}

			attrKeys = append(attrKeys, ak)
		}

		sort.Strings(attrKeys)

		for _, ak := range attrKeys {
			av := attributes[ak]
			buf.WriteString(fmt.Sprintf("  %s = %s\n", ak, av))
		}

		// CAUTION: Since deposed keys are now random strings instead of
		// incrementing integers, this result will not be deterministic
		// if there is more than one deposed object.
		for k, t := range is.Deposed {
			id := legacyInstanceObjectID(t)
			taintStr := ""
			if t.Status == ObjectTainted {
				taintStr = " (tainted)"
			}
			buf.WriteString(fmt.Sprintf("  Deposed ID %s = %s%s\n", k, id, taintStr))
		}

		if obj := is.Current; obj != nil && len(obj.Dependencies) > 0 {
			buf.WriteString(fmt.Sprintf("\n  Dependencies:\n"))
			for _, dep := range obj.Dependencies {
				buf.WriteString(fmt.Sprintf("    %s\n", dep.String()))
			}
		}
	}

	if len(m.OutputValues) > 0 {
		buf.WriteString("\nOutputs:\n\n")

		ks := make([]string, 0, len(m.OutputValues))
		for k := range m.OutputValues {
			ks = append(ks, k)
		}
		sort.Strings(ks)

		for _, k := range ks {
			v := m.OutputValues[k]
			lv := hcl2shim.ConfigValueFromHCL2(v.Value)
			switch vTyped := lv.(type) {
			case string:
				buf.WriteString(fmt.Sprintf("%s = %s\n", k, vTyped))
			case []interface{}:
				buf.WriteString(fmt.Sprintf("%s = %s\n", k, vTyped))
			case map[string]interface{}:
				var mapKeys []string
				for key := range vTyped {
					mapKeys = append(mapKeys, key)
				}
				sort.Strings(mapKeys)

				var mapBuf bytes.Buffer
				mapBuf.WriteString("{")
				for _, key := range mapKeys {
					mapBuf.WriteString(fmt.Sprintf("%s:%s ", key, vTyped[key]))
				}
				mapBuf.WriteString("}")

				buf.WriteString(fmt.Sprintf("%s = %s\n", k, mapBuf.String()))
			default:
				buf.WriteString(fmt.Sprintf("%s = %#v\n", k, lv))
			}
		}
	}

	return buf.String()
}

// legacyInstanceObjectID is a helper for extracting an object id value from
// an instance object in a way that approximates how we used to do this
// for the old state types. ID is no longer first-class, so this is preserved
// only for compatibility with old tests that include the id as part of their
// expected value.
func legacyInstanceObjectID(obj *ResourceInstanceObjectSrc) string {
	if obj == nil {
		return "<not created>"
	}

	if obj.AttrsJSON != nil {
		type WithID struct {
			ID string `json:"id"`
		}
		var withID WithID
		err := json.Unmarshal(obj.AttrsJSON, &withID)
		if err == nil {
			return withID.ID
		}
	} else if obj.AttrsFlat != nil {
		if flatID, exists := obj.AttrsFlat["id"]; exists {
			return flatID
		}
	}

	// For resource types created after we removed id as special there may
	// not actually be one at all. This is okay because older tests won't
	// encounter this, and new tests shouldn't be using ids.
	return "<none>"
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "strings"
    "strconv"
    "log"
)

func main() {
    var output string
    //next := 0

    for start := strings.Index(input, "("); start != -1; start = strings.Index(input, "(")  {
        end := strings.Index(input, ")");

        if (start + 1)  < (end -1) {
            marker := strings.Split(input[start+1:end], "x")
            var err error
            cnt, rep := 0, 0

            if len(marker) == 2 {
                if cnt, err = strconv.Atoi(marker[0]); err == nil {
                    if rep, err = strconv.Atoi(marker[1]); err == nil {
                        output = output + input[:start]
                        for i := 0; i < rep; i++ {
                            if len(input[end+1:]) >= cnt {
                                output = output + input[end + 1 : end + 1 + cnt]
                            } else {
                                log.Fatal("Invalid input file length")
                            }
                        }
                        input = input[end + 1 + cnt:]
                    } else {
                        log.Fatal("Non-integer marker: ", marker[1])
                    }
                } else {
                    log.Fatal("Non-integer marker: ", marker[0])
                }
            } else {
                log.Fatal("Invalid marker format: ", input[start+1:end])
            }
        } else {
            log.Fatal("Invalid marker format: ", input[start+1:end])
        }
    }
    output = output + input[:]
    fmt.Println("Length of output: ", len(output))
}

<commit_msg>Day 9, Puzzle 2<commit_after>package main

import (
    "fmt"
    "strings"
    "strconv"
    "log"
)

func main() {
    output := recurseDecomp(input)
    fmt.Println("Length :", output)
}

func recurseDecomp(input string) (output int) {
    for start := strings.Index(input, "("); start != -1; start = strings.Index(input, "(")  {
        end := strings.Index(input, ")");

        if (start + 1)  < (end -1) {
            marker := strings.Split(input[start+1:end], "x")
            var err error
            cnt, rep := 0, 0

            if len(marker) == 2 {
                if cnt, err = strconv.Atoi(marker[0]); err == nil {
                    if rep, err = strconv.Atoi(marker[1]); err == nil {
                        output = output + len(input[:start])

                        substr := input[end + 1 : end + 1 + cnt]

                        if strings.Index(substr, "(") != -1 {
                            output = output + (rep * recurseDecomp(substr))
                        } else {
                            output = output + (rep * cnt)
                        }
                        input = input[end + 1 + cnt:]
                    } else {
                        log.Fatal("Non-integer marker: ", marker[1])
                    }
                } else {
                    log.Fatal("Non-integer marker: ", marker[0])
                }
            } else {
                log.Fatal("Invalid marker format: ", input[start+1:end])
            }
        } else {
            log.Fatal("Invalid marker format: ", input[start+1:end])
        }
    }
    output = output + len(input[:])
    return output
}
<|endoftext|>