<commit_before>package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
	awscommon "github.com/mitchellh/packer/builder/amazon/common"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/packer"
)

type Config struct {
	Bucket       string `mapstructure:"bucket"`
	ManifestPath string `mapstructure:"manifest"`
	BoxName      string `mapstructure:"box_name"`
	BoxDir       string `mapstructure:"box_dir"`
	Version      string `mapstructure:"version"`

	common.PackerConfig    `mapstructure:",squash"`
	awscommon.AccessConfig `mapstructure:",squash"`
	tpl                    *packer.ConfigTemplate
}

type PostProcessor struct {
	config Config
	s3     *s3.Bucket
}

func (p *PostProcessor) Configure(raws ...interface{}) error {
	_, err := common.DecodeConfig(&p.config, raws...)
	if err != nil {
		return err
	}

	p.config.tpl, err = packer.NewConfigTemplate()
	if err != nil {
		return err
	}
	p.config.tpl.UserVars = p.config.PackerUserVars

	errs := &packer.MultiError{}

	errs = packer.MultiErrorAppend(errs, p.config.AccessConfig.Prepare(p.config.tpl)...)

	// required configuration
	templates := map[string]*string{
		"bucket":   &p.config.Bucket,
		"manifest": &p.config.ManifestPath,
		"box_name": &p.config.BoxName,
		"box_dir":  &p.config.BoxDir,
		"version":  &p.config.Version,
	}

	for key, ptr := range templates {
		if *ptr == "" {
			errs = packer.MultiErrorAppend(errs, fmt.Errorf("vagrant-s3 %s must be set", key))
		}
	}

	// Template process
	for key, ptr := range templates {
		*ptr, err = p.config.tpl.Process(*ptr, nil)
		if err != nil {
			errs = packer.MultiErrorAppend(errs, fmt.Errorf("Error processing %s: %s", key, err))
		}
	}

	// setup the s3 bucket
	auth, err := p.config.AccessConfig.Auth()
	if err != nil {
		errs = packer.MultiErrorAppend(errs, err)
	}
	p.s3 = s3.New(auth, aws.USEast).Bucket(p.config.Bucket)

	if len(errs.Errors) > 0 {
		return errs
	}

	return nil
}

func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
	// Only accept input from the vagrant post-processor
	if artifact.BuilderId() != "mitchellh.post-processor.vagrant" {
		return nil, false, fmt.Errorf("Unknown artifact type, requires box from vagrant post-processor: %s", artifact.BuilderId())
	}

	// Assume there is only one .box file to upload
	box := artifact.Files()[0]
	if !strings.HasSuffix(box, ".box") {
		return nil, false, fmt.Errorf("Unknown files in artifact from vagrant post-processor: %s", artifact.Files())
	}

	provider := providerFromBuilderName(artifact.Id())
	ui.Say(fmt.Sprintf("Preparing to upload box for '%s' provider to S3 bucket '%s'", provider, p.config.Bucket))

	// open the box so we can upload to S3 and calculate checksum for the manifest
	file, err := os.Open(box)
	if err != nil {
		return nil, false, err
	}
	defer file.Close()

	// get the file's size
	info, err := file.Stat()
	if err != nil {
		return nil, false, err
	}
	size := info.Size()
	ui.Message(fmt.Sprintf("Box to upload: %s (%d bytes)", box, size))

	// get the latest manifest so we can add to it
	ui.Message("Fetching latest manifest")
	manifest, err := p.getManifest()
	if err != nil {
		return nil, false, err
	}

	// generate the path to store the box in S3
	boxPath := fmt.Sprintf("%s/%s/%s", p.config.BoxDir, p.config.Version, path.Base(box))

	ui.Message("Generating checksum")
	checksum, err := sum256(file)
	if err != nil {
		return nil, false, err
	}
	ui.Message(fmt.Sprintf("Checksum is %s", checksum))

	ui.Message(fmt.Sprintf("Adding %s %s box to manifest", provider, p.config.Version))
	if err := manifest.add(p.config.Version, &Provider{
		Name:         provider,
		Url:          p.s3.URL(boxPath),
		ChecksumType: "sha256",
		Checksum:     checksum,
	}); err != nil {
		return nil, false, err
	}

	// upload the box to S3 (rewinding as we already read the file to generate the checksum)
	ui.Message(fmt.Sprintf("Uploading box to S3: %s", boxPath))
	if _, err := file.Seek(0, 0); err != nil {
		return nil, false, err
	}
	if size > 100*1024*1024 {
		ui.Message("File size > 100MB. Initiating multipart upload")
		multi, err := p.s3.Multi(boxPath, "application/octet-stream", "public-read")
		if err != nil {
			return nil, false, err
		}
		if _, err := multi.PutAll(file, 5*1024*1024); err != nil {
			return nil, false, err
		}
	} else {
		if err := p.s3.PutReader(boxPath, file, size, "application/octet-stream", "public-read"); err != nil {
			return nil, false, err
		}
	}

	ui.Message(fmt.Sprintf("Uploading the manifest: %s", p.config.ManifestPath))
	if err := p.putManifest(manifest); err != nil {
		return nil, false, err
	}

	return &Artifact{p.s3.URL(p.config.ManifestPath)}, true, nil
}

func (p *PostProcessor) getManifest() (*Manifest, error) {
	body, err := p.s3.GetReader(p.config.ManifestPath)
	if err != nil {
		if s3Err, ok := err.(*s3.Error); ok && s3Err.Code == "NoSuchKey" {
			return &Manifest{Name: p.config.BoxName}, nil
		}
		return nil, err
	}
	defer body.Close()

	manifest := &Manifest{}
	if err := json.NewDecoder(body).Decode(manifest); err != nil {
		return nil, err
	}
	return manifest, nil
}

func (p *PostProcessor) putManifest(manifest *Manifest) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(manifest); err != nil {
		return err
	}
	if err := p.s3.Put(p.config.ManifestPath, buf.Bytes(), "application/json", "public-read"); err != nil {
		return err
	}
	return nil
}

// calculates a sha256 checksum of the file
func sum256(file *os.File) (string, error) {
	h := sha256.New()
	if _, err := io.Copy(h, file); err != nil {
		return "", err
	}
	return hex.EncodeToString(h.Sum(nil)), nil
}

// converts a packer builder name to the corresponding vagrant provider
func providerFromBuilderName(name string) string {
	switch name {
	case "aws":
		return "aws"
	case "digitalocean":
		return "digitalocean"
	case "virtualbox":
		return "virtualbox"
	case "vmware":
		return "vmware_desktop"
	case "parallels":
		return "parallels"
	default:
		return name
	}
}
<commit_msg>Fix multi part upload<commit_after>package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
	awscommon "github.com/mitchellh/packer/builder/amazon/common"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/packer"
)

type Config struct {
	Bucket       string `mapstructure:"bucket"`
	ManifestPath string `mapstructure:"manifest"`
	BoxName      string `mapstructure:"box_name"`
	BoxDir       string `mapstructure:"box_dir"`
	Version      string `mapstructure:"version"`

	common.PackerConfig    `mapstructure:",squash"`
	awscommon.AccessConfig `mapstructure:",squash"`
	tpl                    *packer.ConfigTemplate
}

type PostProcessor struct {
	config Config
	s3     *s3.Bucket
}

func (p *PostProcessor) Configure(raws ...interface{}) error {
	_, err := common.DecodeConfig(&p.config, raws...)
	if err != nil {
		return err
	}

	p.config.tpl, err = packer.NewConfigTemplate()
	if err != nil {
		return err
	}
	p.config.tpl.UserVars = p.config.PackerUserVars

	errs := &packer.MultiError{}

	errs = packer.MultiErrorAppend(errs, p.config.AccessConfig.Prepare(p.config.tpl)...)

	// required configuration
	templates := map[string]*string{
		"bucket":   &p.config.Bucket,
		"manifest": &p.config.ManifestPath,
		"box_name": &p.config.BoxName,
		"box_dir":  &p.config.BoxDir,
		"version":  &p.config.Version,
	}

	for key, ptr := range templates {
		if *ptr == "" {
			errs = packer.MultiErrorAppend(errs, fmt.Errorf("vagrant-s3 %s must be set", key))
		}
	}

	// Template process
	for key, ptr := range templates {
		*ptr, err = p.config.tpl.Process(*ptr, nil)
		if err != nil {
			errs = packer.MultiErrorAppend(errs, fmt.Errorf("Error processing %s: %s", key, err))
		}
	}

	// setup the s3 bucket
	auth, err := p.config.AccessConfig.Auth()
	if err != nil {
		errs = packer.MultiErrorAppend(errs, err)
	}
	p.s3 = s3.New(auth, aws.USEast).Bucket(p.config.Bucket)

	if len(errs.Errors) > 0 {
		return errs
	}

	return nil
}

func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
	// Only accept input from the vagrant post-processor
	if artifact.BuilderId() != "mitchellh.post-processor.vagrant" {
		return nil, false, fmt.Errorf("Unknown artifact type, requires box from vagrant post-processor: %s", artifact.BuilderId())
	}

	// Assume there is only one .box file to upload
	box := artifact.Files()[0]
	if !strings.HasSuffix(box, ".box") {
		return nil, false, fmt.Errorf("Unknown files in artifact from vagrant post-processor: %s", artifact.Files())
	}

	provider := providerFromBuilderName(artifact.Id())
	ui.Say(fmt.Sprintf("Preparing to upload box for '%s' provider to S3 bucket '%s'", provider, p.config.Bucket))

	// open the box so we can upload to S3 and calculate checksum for the manifest
	file, err := os.Open(box)
	if err != nil {
		return nil, false, err
	}
	defer file.Close()

	// get the file's size
	info, err := file.Stat()
	if err != nil {
		return nil, false, err
	}
	size := info.Size()
	ui.Message(fmt.Sprintf("Box to upload: %s (%d bytes)", box, size))

	// get the latest manifest so we can add to it
	ui.Message("Fetching latest manifest")
	manifest, err := p.getManifest()
	if err != nil {
		return nil, false, err
	}

	// generate the path to store the box in S3
	boxPath := fmt.Sprintf("%s/%s/%s", p.config.BoxDir, p.config.Version, path.Base(box))

	ui.Message("Generating checksum")
	checksum, err := sum256(file)
	if err != nil {
		return nil, false, err
	}
	ui.Message(fmt.Sprintf("Checksum is %s", checksum))

	ui.Message(fmt.Sprintf("Adding %s %s box to manifest", provider, p.config.Version))
	if err := manifest.add(p.config.Version, &Provider{
		Name:         provider,
		Url:          p.s3.URL(boxPath),
		ChecksumType: "sha256",
		Checksum:     checksum,
	}); err != nil {
		return nil, false, err
	}

	// upload the box to S3 (rewinding as we already read the file to generate the checksum)
	ui.Message(fmt.Sprintf("Uploading box to S3: %s", boxPath))
	if _, err := file.Seek(0, 0); err != nil {
		return nil, false, err
	}
	if size > 100*1024*1024 {
		ui.Message("File size > 100MB. Initiating multipart upload")
		multi, err := p.s3.Multi(boxPath, "application/octet-stream", "public-read")
		if err != nil {
			return nil, false, err
		}
		parts, err := multi.PutAll(file, 5*1024*1024)
		if err != nil {
			return nil, false, err
		}
		if err := multi.Complete(parts); err != nil {
			return nil, false, err
		}
	} else {
		if err := p.s3.PutReader(boxPath, file, size, "application/octet-stream", "public-read"); err != nil {
			return nil, false, err
		}
	}

	ui.Message(fmt.Sprintf("Uploading the manifest: %s", p.config.ManifestPath))
	if err := p.putManifest(manifest); err != nil {
		return nil, false, err
	}

	return &Artifact{p.s3.URL(p.config.ManifestPath)}, true, nil
}

func (p *PostProcessor) getManifest() (*Manifest, error) {
	body, err := p.s3.GetReader(p.config.ManifestPath)
	if err != nil {
		if s3Err, ok := err.(*s3.Error); ok && s3Err.Code == "NoSuchKey" {
			return &Manifest{Name: p.config.BoxName}, nil
		}
		return nil, err
	}
	defer body.Close()

	manifest := &Manifest{}
	if err := json.NewDecoder(body).Decode(manifest); err != nil {
		return nil, err
	}
	return manifest, nil
}

func (p *PostProcessor) putManifest(manifest *Manifest) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(manifest); err != nil {
		return err
	}
	if err := p.s3.Put(p.config.ManifestPath, buf.Bytes(), "application/json", "public-read"); err != nil {
		return err
	}
	return nil
}

// calculates a sha256 checksum of the file
func sum256(file *os.File) (string, error) {
	h := sha256.New()
	if _, err := io.Copy(h, file); err != nil {
		return "", err
	}
	return hex.EncodeToString(h.Sum(nil)), nil
}

// converts a packer builder name to the corresponding vagrant provider
func providerFromBuilderName(name string) string {
	switch name {
	case "aws":
		return "aws"
	case "digitalocean":
		return "digitalocean"
	case "virtualbox":
		return "virtualbox"
	case "vmware":
		return "vmware_desktop"
	case "parallels":
		return "parallels"
	default:
		return name
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"regexp"

	"github.com/mitchellh/packer/helper/config"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/template/interpolate"
)

// Config is the post-processor configuration with interpolation supported.
// Currently only supports AWS AMI ID.
type Config struct {
	AmiID map[string][]string `mapstructure:"ami_id"`

	ctx interpolate.Context
}

// PostProcessor holds the Config object.
type PostProcessor struct {
	config Config
}

// Configure sets the Config object with configuration values from the Packer
// template.
func (p *PostProcessor) Configure(raws ...interface{}) error {
	err := config.Decode(&p.config, &config.DecodeOpts{
		Interpolate: true,
		InterpolateContext: &p.config.ctx,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{},
		},
	}, raws...)

	if err != nil {
		return err
	}

	if p.config.AmiID == nil {
		p.config.AmiID = make(map[string][]string, 0)
	}

	return nil
}

// PostProcess parses the AMI ID from the artifact ID, and then passes the AMI ID
// to UpdateJSONFile to be set as the new value of the JSON paths properties in
// Packer template.
func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {

	ui.Say(fmt.Sprintf("[DEBUG] Artifact ID: %s", artifact.Id()))
	ui.Say(fmt.Sprintf("[DEBUG] Artifact String: %s", artifact.String()))

	if p.config.AmiID != nil {
		r, _ := regexp.Compile("ami-[a-z0-9]+")
		amiID := r.FindString(artifact.Id())

		for file, properties := range p.config.AmiID {
			err := UpdateJSONFile(file, properties, amiID, ui)
			if err != nil {
				return artifact, false, err
			}
		}
	}

	return artifact, true, nil
}
<commit_msg>Add note on artifact ID output. Update logging with artifact string instead.<commit_after>package main

import (
	"fmt"
	"regexp"

	"github.com/mitchellh/packer/helper/config"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/template/interpolate"
)

// Config is the post-processor configuration with interpolation supported.
// Currently only supports AWS AMI ID.
type Config struct {
	AmiID map[string][]string `mapstructure:"ami_id"`

	ctx interpolate.Context
}

// PostProcessor holds the Config object.
type PostProcessor struct {
	config Config
}

// Configure sets the Config object with configuration values from the Packer
// template.
func (p *PostProcessor) Configure(raws ...interface{}) error {
	err := config.Decode(&p.config, &config.DecodeOpts{
		Interpolate: true,
		InterpolateContext: &p.config.ctx,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{},
		},
	}, raws...)

	if err != nil {
		return err
	}

	if p.config.AmiID == nil {
		p.config.AmiID = make(map[string][]string, 0)
	}

	return nil
}

// PostProcess parses the AMI ID from the artifact ID, and then passes the AMI ID
// to UpdateJSONFile to be set as the new value of the JSON paths properties in
// Packer template.
// AWS artifact ID output has the format of <region>:<ami_id>,
// for example: ap-southeast-2:ami-4f8fae2c
func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {

  ui.Say(fmt.Sprintf("%s", artifact.String()))

	if p.config.AmiID != nil {
		r, _ := regexp.Compile("ami-[a-z0-9]+")
		amiID := r.FindString(artifact.Id())

		for file, properties := range p.config.AmiID {
			err := UpdateJSONFile(file, properties, amiID, ui)
			if err != nil {
				return artifact, false, err
			}
		}
	}

	return artifact, true, nil
}
<|endoftext|>


<commit_before>package core

import (
	"fmt"
	"strings"
	"time"

	"github.com/af83/edwig/audit"
	"github.com/af83/edwig/logger"
	"github.com/af83/edwig/model"
	"github.com/af83/edwig/siri"
)

type SIRIStopMonitoringSubscriber interface {
	model.Stopable
	model.Startable
}

type SMSubscriber struct {
	model.ClockConsumer

	connector *SIRIStopMonitoringSubscriptionCollector
}

type StopMonitoringSubscriber struct {
	SMSubscriber

	stop chan struct{}
}

type FakeStopMonitoringSubscriber struct {
	SMSubscriber
}

func NewFakeStopMonitoringSubscriber(connector *SIRIStopMonitoringSubscriptionCollector) SIRIStopMonitoringSubscriber {
	subscriber := &FakeStopMonitoringSubscriber{}
	subscriber.connector = connector
	return subscriber
}

func (subscriber *FakeStopMonitoringSubscriber) Start() {
	subscriber.prepareSIRIStopMonitoringSubscriptionRequest()
}

func (subscriber *FakeStopMonitoringSubscriber) Stop() {}

func NewSIRIStopMonitoringSubscriber(connector *SIRIStopMonitoringSubscriptionCollector) SIRIStopMonitoringSubscriber {
	subscriber := &StopMonitoringSubscriber{}
	subscriber.connector = connector
	return subscriber
}

func (subscriber *StopMonitoringSubscriber) Start() {
	logger.Log.Debugf("Start StopMonitoringSubscriber")

	subscriber.stop = make(chan struct{})
	go subscriber.run()
}

func (subscriber *StopMonitoringSubscriber) run() {
	c := subscriber.Clock().After(5 * time.Second)

	for {
		select {
		case <-subscriber.stop:
			return
		case <-c:
			logger.Log.Debugf("SIRIStopMonitoringSubscriber visit")

			subscriber.prepareSIRIStopMonitoringSubscriptionRequest()

			c = subscriber.Clock().After(5 * time.Second)
		}
	}
}

func (subscriber *StopMonitoringSubscriber) Stop() {
	if subscriber.stop != nil {
		close(subscriber.stop)
	}
}

func (subscriber *SMSubscriber) prepareSIRIStopMonitoringSubscriptionRequest() {
	subscription, _ := subscriber.connector.partner.Subscriptions().FindOrCreateByKind("StopMonitoring")
	monitoringRefList := []string{}

	stopAreasToRequest := make(map[string]*model.ObjectID)
	for _, resource := range subscription.ResourcesByObjectID() {
		logger.Log.Debugf("resource == %v", resource.Reference.ObjectId.String())
		if resource.SubscribedAt.IsZero() && resource.RetryCount <= 10 {
			messageIdentifier := subscriber.connector.SIRIPartner().IdentifierGenerator("message_identifier").NewMessageIdentifier()
			stopAreasToRequest[messageIdentifier] = resource.Reference.ObjectId
		}
	}

	if len(stopAreasToRequest) == 0 {
		return
	}

	logStashEvent := make(audit.LogStashEvent)
	defer audit.CurrentLogStash().WriteEvent(logStashEvent)

	siriStopMonitoringSubscriptionRequest := &siri.SIRIStopMonitoringSubscriptionRequest{
		ConsumerAddress:   subscriber.connector.Partner().Address(),
		MessageIdentifier: subscriber.connector.SIRIPartner().IdentifierGenerator("message_identifier").NewMessageIdentifier(),
		RequestorRef:      subscriber.connector.SIRIPartner().RequestorRef(),
		RequestTimestamp:  subscriber.Clock().Now(),
	}

	for messageIdentifier, stopAreaObjectid := range stopAreasToRequest {
		entry := &siri.SIRIStopMonitoringSubscriptionRequestEntry{
			MessageIdentifier:      messageIdentifier,
			RequestTimestamp:       subscriber.Clock().Now(),
			MonitoringRef:          stopAreaObjectid.Value(),
			SubscriberRef:          subscriber.connector.SIRIPartner().RequestorRef(),
			SubscriptionIdentifier: string(subscription.Id()),
			InitialTerminationTime: subscriber.Clock().Now().Add(48 * time.Hour),
		}

		monitoringRefList = append(monitoringRefList, stopAreaObjectid.Value())
		siriStopMonitoringSubscriptionRequest.Entries = append(siriStopMonitoringSubscriptionRequest.Entries, entry)
	}

	logStashEvent["MonitoringRef"] = strings.Join(monitoringRefList, ", ")
	logSIRIStopMonitoringSubscriptionRequest(logStashEvent, siriStopMonitoringSubscriptionRequest, monitoringRefList)

	response, err := subscriber.connector.SIRIPartner().SOAPClient().StopMonitoringSubscription(siriStopMonitoringSubscriptionRequest)
	if err != nil {
		logger.Log.Debugf("Error while subscribing: %v", err)
		for _, stopAreaObjectid := range stopAreasToRequest {
			logger.Log.Debugf("%v", *stopAreaObjectid)
			resource := subscription.Resource(*stopAreaObjectid)
			resource.RetryCount++
		}
		return
	}

	logStashEvent["response"] = response.RawXML()

	for _, responseStatus := range response.ResponseStatus() {
		stopAreaObjectid, ok := stopAreasToRequest[responseStatus.RequestMessageRef()]
		if !ok {
			logger.Log.Debugf("ResponseStatus RequestMessageRef unknown: %v", responseStatus.RequestMessageRef())
			continue
		}
		resource := subscription.Resource(*stopAreaObjectid)
		if resource == nil { // Should never happen
			logger.Log.Debugf("Response for unknown subscription resource %v", stopAreaObjectid.String())
			continue
		}
		if !responseStatus.Status() {
			logger.Log.Debugf("Subscription status false for stopArea %v: %v %v ", stopAreaObjectid.Value(), responseStatus.ErrorType(), responseStatus.ErrorText())
			resource.RetryCount++
			continue
		}
		resource.SubscribedAt = subscriber.Clock().Now()
		resource.RetryCount = 0
	}
}

func logSIRIStopMonitoringSubscriptionRequest(logStashEvent audit.LogStashEvent, request *siri.SIRIStopMonitoringSubscriptionRequest, monitorinRefList []string) {
	logStashEvent["Connector"] = "SIRIStopMonitoringSubscriptionCollector"
	logStashEvent["messageIdentifier"] = request.MessageIdentifier
	logStashEvent["requestorRef"] = request.RequestorRef
	logStashEvent["requestTimestamp"] = request.RequestTimestamp.String()
	xml, err := request.BuildXML()
	if err != nil {
		logStashEvent["requestXML"] = fmt.Sprintf("%v", err)
		return
	}
	logStashEvent["requestXML"] = xml
}
<commit_msg>Debug message clean<commit_after>package core

import (
	"fmt"
	"strings"
	"time"

	"github.com/af83/edwig/audit"
	"github.com/af83/edwig/logger"
	"github.com/af83/edwig/model"
	"github.com/af83/edwig/siri"
)

type SIRIStopMonitoringSubscriber interface {
	model.Stopable
	model.Startable
}

type SMSubscriber struct {
	model.ClockConsumer

	connector *SIRIStopMonitoringSubscriptionCollector
}

type StopMonitoringSubscriber struct {
	SMSubscriber

	stop chan struct{}
}

type FakeStopMonitoringSubscriber struct {
	SMSubscriber
}

func NewFakeStopMonitoringSubscriber(connector *SIRIStopMonitoringSubscriptionCollector) SIRIStopMonitoringSubscriber {
	subscriber := &FakeStopMonitoringSubscriber{}
	subscriber.connector = connector
	return subscriber
}

func (subscriber *FakeStopMonitoringSubscriber) Start() {
	subscriber.prepareSIRIStopMonitoringSubscriptionRequest()
}

func (subscriber *FakeStopMonitoringSubscriber) Stop() {}

func NewSIRIStopMonitoringSubscriber(connector *SIRIStopMonitoringSubscriptionCollector) SIRIStopMonitoringSubscriber {
	subscriber := &StopMonitoringSubscriber{}
	subscriber.connector = connector
	return subscriber
}

func (subscriber *StopMonitoringSubscriber) Start() {
	logger.Log.Debugf("Start StopMonitoringSubscriber")

	subscriber.stop = make(chan struct{})
	go subscriber.run()
}

func (subscriber *StopMonitoringSubscriber) run() {
	c := subscriber.Clock().After(5 * time.Second)

	for {
		select {
		case <-subscriber.stop:
			return
		case <-c:
			logger.Log.Debugf("SIRIStopMonitoringSubscriber visit")

			subscriber.prepareSIRIStopMonitoringSubscriptionRequest()

			c = subscriber.Clock().After(5 * time.Second)
		}
	}
}

func (subscriber *StopMonitoringSubscriber) Stop() {
	if subscriber.stop != nil {
		close(subscriber.stop)
	}
}

func (subscriber *SMSubscriber) prepareSIRIStopMonitoringSubscriptionRequest() {
	subscription, _ := subscriber.connector.partner.Subscriptions().FindOrCreateByKind("StopMonitoring")
	monitoringRefList := []string{}

	stopAreasToRequest := make(map[string]*model.ObjectID)
	for _, resource := range subscription.ResourcesByObjectID() {
		if resource.SubscribedAt.IsZero() && resource.RetryCount <= 10 {
			messageIdentifier := subscriber.connector.SIRIPartner().IdentifierGenerator("message_identifier").NewMessageIdentifier()
			stopAreasToRequest[messageIdentifier] = resource.Reference.ObjectId
		}
	}

	if len(stopAreasToRequest) == 0 {
		return
	}

	logStashEvent := make(audit.LogStashEvent)
	defer audit.CurrentLogStash().WriteEvent(logStashEvent)

	siriStopMonitoringSubscriptionRequest := &siri.SIRIStopMonitoringSubscriptionRequest{
		ConsumerAddress:   subscriber.connector.Partner().Address(),
		MessageIdentifier: subscriber.connector.SIRIPartner().IdentifierGenerator("message_identifier").NewMessageIdentifier(),
		RequestorRef:      subscriber.connector.SIRIPartner().RequestorRef(),
		RequestTimestamp:  subscriber.Clock().Now(),
	}

	for messageIdentifier, stopAreaObjectid := range stopAreasToRequest {
		entry := &siri.SIRIStopMonitoringSubscriptionRequestEntry{
			MessageIdentifier:      messageIdentifier,
			RequestTimestamp:       subscriber.Clock().Now(),
			MonitoringRef:          stopAreaObjectid.Value(),
			SubscriberRef:          subscriber.connector.SIRIPartner().RequestorRef(),
			SubscriptionIdentifier: string(subscription.Id()),
			InitialTerminationTime: subscriber.Clock().Now().Add(48 * time.Hour),
		}

		monitoringRefList = append(monitoringRefList, stopAreaObjectid.Value())
		siriStopMonitoringSubscriptionRequest.Entries = append(siriStopMonitoringSubscriptionRequest.Entries, entry)
	}

	logStashEvent["MonitoringRef"] = strings.Join(monitoringRefList, ", ")
	logSIRIStopMonitoringSubscriptionRequest(logStashEvent, siriStopMonitoringSubscriptionRequest, monitoringRefList)

	response, err := subscriber.connector.SIRIPartner().SOAPClient().StopMonitoringSubscription(siriStopMonitoringSubscriptionRequest)
	if err != nil {
		logger.Log.Debugf("Error while subscribing: %v", err)
		for _, stopAreaObjectid := range stopAreasToRequest {
			logger.Log.Debugf("%v", *stopAreaObjectid)
			resource := subscription.Resource(*stopAreaObjectid)
			resource.RetryCount++
		}
		return
	}

	logStashEvent["response"] = response.RawXML()

	for _, responseStatus := range response.ResponseStatus() {
		stopAreaObjectid, ok := stopAreasToRequest[responseStatus.RequestMessageRef()]
		if !ok {
			logger.Log.Debugf("ResponseStatus RequestMessageRef unknown: %v", responseStatus.RequestMessageRef())
			continue
		}
		resource := subscription.Resource(*stopAreaObjectid)
		if resource == nil { // Should never happen
			logger.Log.Debugf("Response for unknown subscription resource %v", stopAreaObjectid.String())
			continue
		}
		if !responseStatus.Status() {
			logger.Log.Debugf("Subscription status false for stopArea %v: %v %v ", stopAreaObjectid.Value(), responseStatus.ErrorType(), responseStatus.ErrorText())
			resource.RetryCount++
			continue
		}
		resource.SubscribedAt = subscriber.Clock().Now()
		resource.RetryCount = 0
	}
}

func logSIRIStopMonitoringSubscriptionRequest(logStashEvent audit.LogStashEvent, request *siri.SIRIStopMonitoringSubscriptionRequest, monitorinRefList []string) {
	logStashEvent["Connector"] = "SIRIStopMonitoringSubscriptionCollector"
	logStashEvent["messageIdentifier"] = request.MessageIdentifier
	logStashEvent["requestorRef"] = request.RequestorRef
	logStashEvent["requestTimestamp"] = request.RequestTimestamp.String()
	xml, err := request.BuildXML()
	if err != nil {
		logStashEvent["requestXML"] = fmt.Sprintf("%v", err)
		return
	}
	logStashEvent["requestXML"] = xml
}
<|endoftext|>


<commit_before>package projects

import "time"

const (
	// GetSbomEndpoint is the current endpoint for retrieving an organization's SBOMs
	GetSbomEndpoint = "v1/project/getSBOM"
	// GetSbomsEndpoint is the current endpoint for retrieving an organization's SBOMs
	GetSbomsEndpoint = "v1/project/getSBOMs"
)

// SourceDetails contains the original package information retrieved directly from an uploaded SBOM.
type SourceDetails struct {
	Name    string `json:"sbom_name"`
	Org     string `json:"sbom_org"`
	Version string `json:"sbom_version"`
}

// SBOMEntry represents a single entry within an SBOM.
type SBOMEntry struct {
	ID             string        `json:"id"`
	SBOMID         string        `json:"sbom_id"`
	Confidence     float32       `json:"confidence"`
	Name           string        `json:"name"`
	Org            string        `json:"org"`
	Version        string        `json:"version"`
	IonID          string        `json:"ion_id"`
	Selected       bool          `json:"selected"`
	LocationInSBOM int           `json:"location_in_sbom"`
	Source         SourceDetails `json:"source"`
	ErrMsg         string        `json:"error_message"`
}

// SBOM represents a software list containing zero or more SBOMEntry objects.
type SBOM struct {
	ID         string      `json:"id"`
	Name       string      `json:"sbom_name"`
	Version    string      `json:"sbom_version"`
	Supplier   string      `json:"supplier_name"`
	SbomType   string      `json:"sbom_type"`
	SbomStatus string      `json:"sbom_status"`
	CreatedAt  time.Time   `json:"created_at"`
	UpdatedAt  time.Time   `json:"updated_at"`
	EntryCount int         `json:"entry_count"`
	Entries    []SBOMEntry `json:"entries"`
	TeamID     string      `json:"team_id"`
	OrgID      string      `json:"org_id"`
	RulesetID  string      `json:"ruleset_id"`
}
<commit_msg>New fields for SBOM entries.<commit_after>package projects

import "time"

const (
	// GetSbomEndpoint is the current endpoint for retrieving an organization's SBOMs
	GetSbomEndpoint = "v1/project/getSBOM"
	// GetSbomsEndpoint is the current endpoint for retrieving an organization's SBOMs
	GetSbomsEndpoint = "v1/project/getSBOMs"
)

// SourceDetails contains the original package information retrieved directly from an uploaded SBOM.
type SourceDetails struct {
	Name    string `json:"sbom_name"`
	Org     string `json:"sbom_org"`
	Version string `json:"sbom_version"`
}

// SBOMEntry represents a single entry within an SBOM.
type SBOMEntry struct {
	ID             string        `json:"id"`
	SBOMID         string        `json:"sbom_id"`
	Confidence     float32       `json:"confidence"`
	Name           string        `json:"name"`
	Org            string        `json:"org"`
	Version        string        `json:"version"`
	IonID          string        `json:"ion_id"`
	Selected       bool          `json:"selected"`
	LocationInSBOM int           `json:"location_in_sbom"`
	Source         SourceDetails `json:"source"`
	ErrMsg         string        `json:"error_message"`
	ProductID      string        `json:"product_id"` // CPE
	PackageID      string        `json:"package_id"` // PURL
	Repo           string        `json:"repo"`
}

// SBOM represents a software list containing zero or more SBOMEntry objects.
type SBOM struct {
	ID         string      `json:"id"`
	Name       string      `json:"sbom_name"`
	Version    string      `json:"sbom_version"`
	Supplier   string      `json:"supplier_name"`
	SbomType   string      `json:"sbom_type"`
	SbomStatus string      `json:"sbom_status"`
	CreatedAt  time.Time   `json:"created_at"`
	UpdatedAt  time.Time   `json:"updated_at"`
	EntryCount int         `json:"entry_count"`
	Entries    []SBOMEntry `json:"entries"`
	TeamID     string      `json:"team_id"`
	OrgID      string      `json:"org_id"`
	RulesetID  string      `json:"ruleset_id"`
}
<|endoftext|>


<commit_before>package drax

import (
	"bytes"
	"fmt"
	"io"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/cpuguy83/drax/api"
	"github.com/cpuguy83/drax/api/client"
	libkvstore "github.com/docker/libkv/store"
	"github.com/hashicorp/raft"
)

type store struct {
	mu      sync.RWMutex
	ttlLock sync.Mutex
	data    *db
	r       *Raft
}

type ttl struct {
	TTL         time.Duration
	CreateTime  time.Time
	CreateIndex uint64
}

type db struct {
	KV   map[string]*libkvstore.KVPair
	TTLs map[string]*ttl
}

func newDB() *db {
	return &db{KV: make(map[string]*libkvstore.KVPair), TTLs: make(map[string]*ttl)}
}

func (s *store) newClient() client.Client {
	leader := s.r.getLeader()
	return client.New(leader, s.r.tlsConfig, defaultTimeout)
}

func newStore() *store {
	return &store{data: newDB()}
}

func (s *store) Get(key string) (*libkvstore.KVPair, error) {
	if s.r.IsLeader() {
		s.mu.RLock()
		defer s.mu.RUnlock()
		return s.get(key)
	}
	return s.newClient().Get(key)
}

func (s *store) get(key string) (*libkvstore.KVPair, error) {
	kv, ok := s.data.KV[key]
	if !ok {
		return nil, libkvstore.ErrKeyNotFound
	}
	return kv, nil
}

// TODO: support TTL from libkvstore.WriteOptions
func (s *store) Put(key string, value []byte, options *libkvstore.WriteOptions) error {
	if !s.r.IsLeader() {
		return s.newClient().Put(key, value, options)
	}
	s.mu.Lock()
	defer s.mu.Unlock()

	req := &api.Request{
		Action: api.Put,
		Key:    key,
		Value:  value,
	}
	if options != nil {
		req.TTL = options.TTL
	}
	return s.apply(req)
}

func (s *store) Delete(key string) error {
	if !s.r.IsLeader() {
		return s.newClient().Delete(key)
	}

	s.mu.Lock()
	defer s.mu.Unlock()
	return s.apply(&api.Request{
		Action: api.Delete,
		Key:    key,
	})
}

func (s *store) Exists(key string) (bool, error) {
	if !s.r.IsLeader() {
		return s.newClient().Exists(key)
	}
	s.mu.RLock()
	defer s.mu.RUnlock()
	_, ok := s.data.KV[key]
	return ok, nil
}

func (s *store) List(prefix string) ([]*libkvstore.KVPair, error) {
	if !s.r.IsLeader() {
		return s.newClient().List(prefix)
	}
	s.mu.RLock()
	defer s.mu.RUnlock()

	var out []*libkvstore.KVPair

	for k, v := range s.data.KV {
		if !strings.HasPrefix(k, prefix) {
			continue
		}
		var kv libkvstore.KVPair
		kv = *v
		out = append(out, &kv)
	}

	if len(out) == 0 {
		return nil, libkvstore.ErrKeyNotFound
	}
	return out, nil
}

func (s *store) DeleteTree(dir string) error {
	if !s.r.IsLeader() {
		return s.newClient().DeleteTree(dir)
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.apply(&api.Request{
		Action: api.DeleteTree,
		Key:    dir,
	})
}

func (s *store) Watch(key string, stopCh <-chan struct{}) (<-chan *libkvstore.KVPair, error) {
	if !s.r.IsLeader() {
		return s.newClient().Watch(key, stopCh)
	}
	return nil, libkvstore.ErrCallNotSupported
}

func (s *store) WatchTree(dir string, stopCh <-chan struct{}) (<-chan []*libkvstore.KVPair, error) {
	if !s.r.IsLeader() {
		return s.newClient().WatchTree(dir, stopCh)
	}
	return nil, libkvstore.ErrCallNotSupported
}

func (s *store) NewLock(key string, options *libkvstore.LockOptions) (libkvstore.Locker, error) {
	if !s.r.IsLeader() {
		return s.newClient().NewLock(key, options)
	}
	return nil, libkvstore.ErrCallNotSupported
}

func (s *store) AtomicPut(key string, value []byte, previous *libkvstore.KVPair, options *libkvstore.WriteOptions) (bool, *libkvstore.KVPair, error) {
	if !s.r.IsLeader() {
		return s.newClient().AtomicPut(key, value, previous, options)
	}
	s.mu.Lock()
	defer s.mu.Unlock()

	kv, err := s.get(key)
	if err != nil {
		if previous != nil && err == libkvstore.ErrKeyNotFound {
			return false, nil, libkvstore.ErrKeyModified
		}
		return false, nil, err
	}

	if previous != nil && kv.LastIndex != previous.LastIndex {
		return false, nil, libkvstore.ErrKeyModified
	}

	req = &api.Request{
		Action: api.Put,
		Key:    key,
		Value:  value,
	}
	if options != nil {
		req.TTL = options.TTL
	}
	if err := s.apply(req); err != nil {
		return false, nil, err
	}

	kv, err = s.get(key)
	if err != nil {
		return false, nil, libkvstore.ErrKeyNotFound
	}
	return true, kv, nil
}

func (s *store) AtomicDelete(key string, previous *libkvstore.KVPair) (bool, error) {
	if !s.r.IsLeader() {
		return s.newClient().AtomicDelete(key, previous)
	}
	s.mu.Lock()
	defer s.mu.Unlock()

	if previous == nil {
		return false, libkvstore.ErrPreviousNotSpecified
	}

	kv, err := s.get(key)
	if err != nil {
		if err == libkvstore.ErrKeyModified {
			return false, err
		}
	}
	if kv.LastIndex != previous.LastIndex {
		return false, libkvstore.ErrKeyModified
	}
	if err := s.apply(&api.Request{
		Action: api.Delete,
		Key:    key,
	}); err != nil {
		return false, err
	}
	return true, nil
}

func (s *store) Close() {
	return
}

func (s *store) forwardToLeader(ax *api.Request) error {
	leader := s.r.Leader()
	if leader == "" {
		return raft.ErrNotLeader
	}

	return raft.ErrNotLeader
}

func (s *store) apply(ax *api.Request) error {
	if !s.r.IsLeader() {
		return s.forwardToLeader(ax)
	}
	buf := bytes.NewBuffer(nil)
	if err := api.Encode(ax, buf); err != nil {
		return err
	}
	return s.r.Apply(buf.Bytes())
}

func (s *store) waitLeader() {
	leaderCh := s.r.LeaderCh()
	logrus.Debug("store: waiting for leader")
	var state raft.RaftState
	for {
		select {
		case si := <-leaderCh:
			state = si.(raft.RaftState)
		case <-s.r.ShutdownCh():
			return
		}

		if state != raft.Leader {
			continue
		}
		logrus.Debug("store: handling leader")
		s.handleLeader(leaderCh)
		logrus.Debugf("store: waiting for leader")
	}
}

func (s *store) handleLeader(leaderCh <-chan interface{}) {
	for {
		select {
		case state := <-leaderCh:
			if state != raft.Leader {
				return
			}
		case <-s.r.ShutdownCh():
			return
		default:
		}
		time.Sleep(1 * time.Second)

		s.ttlLock.Lock()
		var keys []string
		for k, ttl := range s.data.TTLs {
			if ttlDue(ttl) {
				keys = append(keys, k)
			}
		}
		if len(keys) > 0 {
			logrus.Debugf("reaping TTL's for %v", keys)
			s.reapKeys(keys)
		}
		s.ttlLock.Unlock()
	}
}

func (s *store) reapKeys(keys []string) {
	if err := s.apply(&api.Request{
		Action: reapKeys,
		Args:   keys,
	}); err != nil {
		logrus.Debugf("error reaping keys: %v", err)
	}
}

func ttlDue(t *ttl) bool {
	now := time.Now()
	return now.After(t.CreateTime.Add(t.TTL))
}

type storeFSM store

// TODO: handle watches
func (s *storeFSM) Apply(l *raft.Log) interface{} {
	var ax api.Request
	if err := api.Decode(&ax, bytes.NewBuffer(l.Data)); err != nil {
		return err
	}

	switch ax.Action {
	case api.Delete:
		delete(s.data.KV, ax.Key)
		delete(s.data.TTLs, ax.Key)
	case api.Put:
		s.data.KV[ax.Key] = &libkvstore.KVPair{Key: ax.Key, Value: ax.Value, LastIndex: l.Index}
		if ax.TTL != 0 {
			s.ttlLock.Lock()
			s.data.TTLs[ax.Key] = &ttl{CreateTime: time.Now(), TTL: ax.TTL, CreateIndex: l.Index}
			s.ttlLock.Unlock()
		}
	case api.DeleteTree:
		for k := range s.data.KV {
			if !strings.HasPrefix(k, ax.Key) {
				continue
			}
			delete(s.data.KV, k)
			delete(s.data.TTLs, k)
		}
	case reapKeys:
		s.mu.Lock()
		for _, k := range ax.Args {
			delete(s.data.KV, k)
			delete(s.data.TTLs, k)
		}
		s.mu.Unlock()
	default:
		return fmt.Errorf("unknown api.Request")
	}
	return nil
}

func (s *storeFSM) Snapshot() (raft.FSMSnapshot, error) {
	return s, nil
}

func (s *storeFSM) Restore(r io.ReadCloser) error {
	defer r.Close()
	s.data = newDB()
	return api.Decode(s.data, r)
}

func (s *storeFSM) Persist(sink raft.SnapshotSink) error {
	defer sink.Close()
	s.mu.Lock()
	defer s.mu.Unlock()
	return api.Encode(s.data, sink)
}

func (*storeFSM) Release() {}
<commit_msg>ttls are supported<commit_after>package drax

import (
	"bytes"
	"fmt"
	"io"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/cpuguy83/drax/api"
	"github.com/cpuguy83/drax/api/client"
	libkvstore "github.com/docker/libkv/store"
	"github.com/hashicorp/raft"
)

type store struct {
	mu      sync.RWMutex
	ttlLock sync.Mutex
	data    *db
	r       *Raft
}

type ttl struct {
	TTL         time.Duration
	CreateTime  time.Time
	CreateIndex uint64
}

type db struct {
	KV   map[string]*libkvstore.KVPair
	TTLs map[string]*ttl
}

func newDB() *db {
	return &db{KV: make(map[string]*libkvstore.KVPair), TTLs: make(map[string]*ttl)}
}

func (s *store) newClient() client.Client {
	leader := s.r.getLeader()
	return client.New(leader, s.r.tlsConfig, defaultTimeout)
}

func newStore() *store {
	return &store{data: newDB()}
}

func (s *store) Get(key string) (*libkvstore.KVPair, error) {
	if s.r.IsLeader() {
		s.mu.RLock()
		defer s.mu.RUnlock()
		return s.get(key)
	}
	return s.newClient().Get(key)
}

func (s *store) get(key string) (*libkvstore.KVPair, error) {
	kv, ok := s.data.KV[key]
	if !ok {
		return nil, libkvstore.ErrKeyNotFound
	}
	return kv, nil
}

func (s *store) Put(key string, value []byte, options *libkvstore.WriteOptions) error {
	if !s.r.IsLeader() {
		return s.newClient().Put(key, value, options)
	}
	s.mu.Lock()
	defer s.mu.Unlock()

	req := &api.Request{
		Action: api.Put,
		Key:    key,
		Value:  value,
	}
	if options != nil {
		req.TTL = options.TTL
	}
	return s.apply(req)
}

func (s *store) Delete(key string) error {
	if !s.r.IsLeader() {
		return s.newClient().Delete(key)
	}

	s.mu.Lock()
	defer s.mu.Unlock()
	return s.apply(&api.Request{
		Action: api.Delete,
		Key:    key,
	})
}

func (s *store) Exists(key string) (bool, error) {
	if !s.r.IsLeader() {
		return s.newClient().Exists(key)
	}
	s.mu.RLock()
	defer s.mu.RUnlock()
	_, ok := s.data.KV[key]
	return ok, nil
}

func (s *store) List(prefix string) ([]*libkvstore.KVPair, error) {
	if !s.r.IsLeader() {
		return s.newClient().List(prefix)
	}
	s.mu.RLock()
	defer s.mu.RUnlock()

	var out []*libkvstore.KVPair

	for k, v := range s.data.KV {
		if !strings.HasPrefix(k, prefix) {
			continue
		}
		var kv libkvstore.KVPair
		kv = *v
		out = append(out, &kv)
	}

	if len(out) == 0 {
		return nil, libkvstore.ErrKeyNotFound
	}
	return out, nil
}

func (s *store) DeleteTree(dir string) error {
	if !s.r.IsLeader() {
		return s.newClient().DeleteTree(dir)
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.apply(&api.Request{
		Action: api.DeleteTree,
		Key:    dir,
	})
}

func (s *store) Watch(key string, stopCh <-chan struct{}) (<-chan *libkvstore.KVPair, error) {
	if !s.r.IsLeader() {
		return s.newClient().Watch(key, stopCh)
	}
	return nil, libkvstore.ErrCallNotSupported
}

func (s *store) WatchTree(dir string, stopCh <-chan struct{}) (<-chan []*libkvstore.KVPair, error) {
	if !s.r.IsLeader() {
		return s.newClient().WatchTree(dir, stopCh)
	}
	return nil, libkvstore.ErrCallNotSupported
}

func (s *store) NewLock(key string, options *libkvstore.LockOptions) (libkvstore.Locker, error) {
	if !s.r.IsLeader() {
		return s.newClient().NewLock(key, options)
	}
	return nil, libkvstore.ErrCallNotSupported
}

func (s *store) AtomicPut(key string, value []byte, previous *libkvstore.KVPair, options *libkvstore.WriteOptions) (bool, *libkvstore.KVPair, error) {
	if !s.r.IsLeader() {
		return s.newClient().AtomicPut(key, value, previous, options)
	}
	s.mu.Lock()
	defer s.mu.Unlock()

	kv, err := s.get(key)
	if err != nil {
		if previous != nil && err == libkvstore.ErrKeyNotFound {
			return false, nil, libkvstore.ErrKeyModified
		}
		return false, nil, err
	}

	if previous != nil && kv.LastIndex != previous.LastIndex {
		return false, nil, libkvstore.ErrKeyModified
	}

	req = &api.Request{
		Action: api.Put,
		Key:    key,
		Value:  value,
	}
	if options != nil {
		req.TTL = options.TTL
	}
	if err := s.apply(req); err != nil {
		return false, nil, err
	}

	kv, err = s.get(key)
	if err != nil {
		return false, nil, libkvstore.ErrKeyNotFound
	}
	return true, kv, nil
}

func (s *store) AtomicDelete(key string, previous *libkvstore.KVPair) (bool, error) {
	if !s.r.IsLeader() {
		return s.newClient().AtomicDelete(key, previous)
	}
	s.mu.Lock()
	defer s.mu.Unlock()

	if previous == nil {
		return false, libkvstore.ErrPreviousNotSpecified
	}

	kv, err := s.get(key)
	if err != nil {
		if err == libkvstore.ErrKeyModified {
			return false, err
		}
	}
	if kv.LastIndex != previous.LastIndex {
		return false, libkvstore.ErrKeyModified
	}
	if err := s.apply(&api.Request{
		Action: api.Delete,
		Key:    key,
	}); err != nil {
		return false, err
	}
	return true, nil
}

func (s *store) Close() {
	return
}

func (s *store) forwardToLeader(ax *api.Request) error {
	leader := s.r.Leader()
	if leader == "" {
		return raft.ErrNotLeader
	}

	return raft.ErrNotLeader
}

func (s *store) apply(ax *api.Request) error {
	if !s.r.IsLeader() {
		return s.forwardToLeader(ax)
	}
	buf := bytes.NewBuffer(nil)
	if err := api.Encode(ax, buf); err != nil {
		return err
	}
	return s.r.Apply(buf.Bytes())
}

func (s *store) waitLeader() {
	leaderCh := s.r.LeaderCh()
	logrus.Debug("store: waiting for leader")
	var state raft.RaftState
	for {
		select {
		case si := <-leaderCh:
			state = si.(raft.RaftState)
		case <-s.r.ShutdownCh():
			return
		}

		if state != raft.Leader {
			continue
		}
		logrus.Debug("store: handling leader")
		s.handleLeader(leaderCh)
		logrus.Debugf("store: waiting for leader")
	}
}

func (s *store) handleLeader(leaderCh <-chan interface{}) {
	for {
		select {
		case state := <-leaderCh:
			if state != raft.Leader {
				return
			}
		case <-s.r.ShutdownCh():
			return
		default:
		}
		time.Sleep(1 * time.Second)

		s.ttlLock.Lock()
		var keys []string
		for k, ttl := range s.data.TTLs {
			if ttlDue(ttl) {
				keys = append(keys, k)
			}
		}
		if len(keys) > 0 {
			logrus.Debugf("reaping TTL's for %v", keys)
			s.reapKeys(keys)
		}
		s.ttlLock.Unlock()
	}
}

func (s *store) reapKeys(keys []string) {
	if err := s.apply(&api.Request{
		Action: reapKeys,
		Args:   keys,
	}); err != nil {
		logrus.Debugf("error reaping keys: %v", err)
	}
}

func ttlDue(t *ttl) bool {
	now := time.Now()
	return now.After(t.CreateTime.Add(t.TTL))
}

type storeFSM store

// TODO: handle watches
func (s *storeFSM) Apply(l *raft.Log) interface{} {
	var ax api.Request
	if err := api.Decode(&ax, bytes.NewBuffer(l.Data)); err != nil {
		return err
	}

	switch ax.Action {
	case api.Delete:
		delete(s.data.KV, ax.Key)
		delete(s.data.TTLs, ax.Key)
	case api.Put:
		s.data.KV[ax.Key] = &libkvstore.KVPair{Key: ax.Key, Value: ax.Value, LastIndex: l.Index}
		if ax.TTL != 0 {
			s.ttlLock.Lock()
			s.data.TTLs[ax.Key] = &ttl{CreateTime: time.Now(), TTL: ax.TTL, CreateIndex: l.Index}
			s.ttlLock.Unlock()
		}
	case api.DeleteTree:
		for k := range s.data.KV {
			if !strings.HasPrefix(k, ax.Key) {
				continue
			}
			delete(s.data.KV, k)
			delete(s.data.TTLs, k)
		}
	case reapKeys:
		s.mu.Lock()
		for _, k := range ax.Args {
			delete(s.data.KV, k)
			delete(s.data.TTLs, k)
		}
		s.mu.Unlock()
	default:
		return fmt.Errorf("unknown api.Request")
	}
	return nil
}

func (s *storeFSM) Snapshot() (raft.FSMSnapshot, error) {
	return s, nil
}

func (s *storeFSM) Restore(r io.ReadCloser) error {
	defer r.Close()
	s.data = newDB()
	return api.Decode(s.data, r)
}

func (s *storeFSM) Persist(sink raft.SnapshotSink) error {
	defer sink.Close()
	s.mu.Lock()
	defer s.mu.Unlock()
	return api.Encode(s.data, sink)
}

func (*storeFSM) Release() {}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"sync"
)

type StatusStore struct {
	status       map[string]string
	mu           sync.RWMutex
	file         *os.File
	isFileLoaded bool
	newWorld     map[string]string
}
type record struct {
	Key, Status string
}

func NewStatusStore(filename string) *StatusStore {
	s := &StatusStore{status: make(map[string]string)}
	f, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		log.Fatal("StatusStore:", err)
	}
	s.file = f
	if err := s.load(); err != nil {
		log.Println("StatusStore:", err)
	}
	s.isFileLoaded = true
	return s
}

func (s *StatusStore) Get(key string) string {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.status[key]
}

func (s *StatusStore) Set(key, status string) bool {
	s.mu.Lock()
	defer s.mu.Unlock()
	if _, present := s.status[key]; present {
		return false
	}
	s.status[key] = status
	return true
}

func (s *StatusStore) Count() int {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return len(s.status)
}

func (s *StatusStore) Check(key string) bool {

	if _, ok := s.status[key]; ok {
		fmt.Println(s.status[key])
		fmt.Println(ok)
		return true
	} else {
		fmt.Println(s.status[key])
		fmt.Println(ok)

		return false
	}

}
func (s *StatusStore) Put(status string) string {
	for {
		key := genKey(s.Count())
		if ok := s.Set(key, status); ok {
			if err := s.save(key, status); err != nil {
				log.Println("StatusStore:", err)
			}
			return key
		}
	}
	panic("shouldn't get here")
}

func (s *StatusStore) RemoveOldRecords(ec2Map map[string]string, status string) {
	for k, v := range s.status {
		if v == status {
			if ec2Map[k] == status {
			} else {
				delete(s.status, k)
			}
		}
	}
}

func (s *StatusStore) DataToFile(status string, c *Conn) {

	for k, v := range c.data {
		if v == status {
			if _, ok := s.status[k]; ok {
			} else {
				err := s.save(k, v)
				if err != nil {
					log.Printf("something went wrong save %s", k)
				}
			}
		}
	}
}
func (s *StatusStore) load() error {
	if _, err := s.file.Seek(0, 0); err != nil {
		return err
	}
	d := json.NewDecoder(s.file)
	var err error
	for err == nil {
		var r record
		if err = d.Decode(&r); err == nil {
			s.Set(r.Key, r.Status)
		}
	}
	if err == io.EOF {
		return nil
	}
	s.isFileLoaded = true
	return err
}

func (s *StatusStore) save(key, status string) error {
	e := json.NewEncoder(s.file)
	return e.Encode(record{key, status})
}
<commit_msg>{WIP}<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"sync"
)

type StatusStore struct {
	status       map[string]string
	mu           sync.RWMutex
	file         *os.File
	isFileLoaded bool
	newWorld     map[string]string
}
type record struct {
	Key, Status string
}

func NewStatusStore(filename string) *StatusStore {
	s := &StatusStore{status: make(map[string]string)}
	f, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		log.Fatal("StatusStore:", err)
	}
	s.file = f
	if err := s.load(); err != nil {
		log.Println("StatusStore:", err)
	}
	s.isFileLoaded = true
	return s
}

func (s *StatusStore) Get(key string) string {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.status[key]
}

func (s *StatusStore) Set(key, status string) bool {
	s.mu.Lock()
	defer s.mu.Unlock()
	if _, present := s.status[key]; present {
		return false
	}
	s.status[key] = status
	return true
}

func (s *StatusStore) Count() int {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return len(s.status)
}

func (s *StatusStore) Check(key string) bool {

	if _, ok := s.status[key]; ok {
		fmt.Println(s.status[key])
		fmt.Println(ok)
		return true
	} else {
		fmt.Println(s.status[key])
		fmt.Println(ok)

		return false
	}

}
func (s *StatusStore) Put(status string) string {
	for {
		key := genKey(s.Count())
		if ok := s.Set(key, status); ok {
			if err := s.save(key, status); err != nil {
				log.Println("StatusStore:", err)
			}
			return key
		}
	}
	panic("shouldn't get here")
}

func (s *StatusStore) RemoveOldRecords(ec2Map map[string]string, status string) {
	for k, v := range s.status {
		if v == status {
			if ec2Map[k] == status {
			} else {
				delete(s.status, k)
			}
		}
	}
}

func (s *StatusStore) DataToFile(status string, c *Conn) {

	for k, v := range c.data {
		if v == status {
			if _, ok := s.status[k]; ok {
				err := s.save(k, v) // a little redundant but needed
				if err != nil {
					log.Printf("something went wrong saving %s", k)
				}
			} else {
				err := s.save(k, v)
				if err != nil {
					log.Printf("something went wrong save %s", k)
				}
			}
		}
	}
}
func (s *StatusStore) load() error {
	if _, err := s.file.Seek(0, 0); err != nil {
		return err
	}
	d := json.NewDecoder(s.file)
	var err error
	for err == nil {
		var r record
		if err = d.Decode(&r); err == nil {
			s.Set(r.Key, r.Status)
		}
	}
	if err == io.EOF {
		return nil
	}
	s.isFileLoaded = true
	return err
}

func (s *StatusStore) save(key, status string) error {
	e := json.NewEncoder(s.file)
	return e.Encode(record{key, status})
}
<|endoftext|>


<commit_before>package store

// TODO The protocol could be modified so that the client checks if its query
// is in the set before actually requesting it from the server. I should
// evaluate experimentally how frequently the server ends up sending a
// ciphertext needless for an incorrect query.

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/sha256"
	"encoding/binary"
	"encoding/hex"
	"fmt"

	"github.com/cjpatton/store/pb"
	"golang.org/x/crypto/pbkdf2"
)

const SealKeyBytes = 16
const KeyBytes = DictKeyBytes + SealKeyBytes

const ErrorMapTooLarge = Error("input map is too large")

// GenerateKey generates a fresh, random key and returns it.
func GenerateKey() []byte {
	K := make([]byte, KeyBytes)
	_, err := rand.Read(K)
	if err != nil {
		return nil
	}
	return K
}

// DeriveKeyFromPassword derives a key from a password and (optional) salt and
// returns it.
//
// If salt == nil, then no salt is used. Note that the salt is not the same as
// pb.Params.Salt. pb.Params.Salt is generated by NewDict(), which in turn
// depends on the key.
func DeriveKeyFromPassword(password, salt []byte) []byte {
	return pbkdf2.Key(password, salt, 4096, KeyBytes, sha256.New)
}

type PubStore struct {
	dict   *PubDict
	nonce  []byte
	sealed [][]byte
	graph  Graph
}

type PrivStore struct {
	dict *PrivDict
	aead cipher.AEAD
}

func NewStore(K []byte, M map[string]string) (*PubStore, *PrivStore, error) {

	cM := newCMap(M)
	defer cM.free()

	pub := new(PubStore)
	priv := new(PrivStore)

	// Set up context for AEAD.
	block, err := aes.NewCipher(K[:DictKeyBytes])
	if err != nil {
		return nil, nil, err
	}
	priv.aead, err = cipher.NewGCM(block)
	if err != nil {
		return nil, nil, err
	}

	// AEAD nonce is the dictionary salt plus a counter.
	//
	// Compute the number of bytes of the nonce allocated for the counter and
	// ensure that it is long enough to uniquely encode each input/output pair
	// in the map.
	ctrBytes := priv.aead.NonceSize() - SaltBytes
	if len(M) > (1 << (8 * uint(ctrBytes))) {
		return nil, nil, ErrorMapTooLarge
	}

	// Create a *cMap for inputs to counters. This is what will actually be
	// stored by pub.dict.
	cN := cM.getCtrMap(ctrBytes)
	defer cN.free()

	// Construct the graph.
	pub.dict, priv.dict, pub.graph, err = newDictAndGraph(
		K[DictKeyBytes:], cN, 0, false)
	if err != nil {
		return nil, nil, err
	}

	// Encrypt each output and store in pub.sealed.
	nonce := cBytesToBytes(priv.dict.params.salt, priv.dict.params.salt_bytes)
	ctr := make([]byte, ctrBytes)
	pub.sealed = make([][]byte, len(M))
	for i := 0; i < len(M); i++ {
		// TODO M and A copied twice unnecessarily
		cMessage, cMessageBytes := cM.getOutput(i)
		cAssociatedData, cAssociatedDataBytes := cM.getInput(i)
		M := cBytesToBytes(cMessage, cMessageBytes)
		A := cBytesToBytes(cAssociatedData, cAssociatedDataBytes)
		binary.LittleEndian.PutUint32(ctr, uint32(i))
		pub.sealed[i] = priv.aead.Seal(nil, append(nonce, ctr...), M, A)
	}

	return pub, priv, nil
}

func (priv *PrivStore) GetIdx(input string) (int, int, error) {
	return priv.dict.GetIdx(input)
}

func (pub *PubStore) GetShare(x, y int) ([]byte, error) {

	// Get counter share.
	ctrShare, err := pub.dict.GetShare(x, y)
	if err != nil {
		return nil, err
	}

	// Look up sealed output.
	for i := 0; i < len(pub.graph[x]); i++ {
		e := pub.graph[x][i]
		for j := 0; j < len(pub.graph[y]); j++ {
			if pub.graph[y][j] == e {
				return append(ctrShare, pub.sealed[e]...), nil
			}
		}
	}

	return nil, ItemNotFound
}

func (priv *PrivStore) GetOutput(input string, pubShare []byte) (string, error) {
	ctrShareBytes := priv.dict.params.row_bytes
	ctr, err := priv.dict.GetOutput(input, pubShare[:ctrShareBytes])
	if err != nil {
		return "", err
	}

	nonce := cBytesToBytes(priv.dict.params.salt, priv.dict.params.salt_bytes)
	output, err := priv.aead.Open(
		nil, append(nonce, []byte(ctr)...), pubShare[ctrShareBytes:], []byte(input))
	if err != nil {
		return "", ItemNotFound
	}
	return string(output), nil
}

func (pub *PubStore) Free() {
	pub.dict.Free()
}

func (priv *PrivStore) Free() {
	priv.dict.Free()
}

func NewPubStoreFromProto(table *pb.Store) *PubStore {
	pub := new(PubStore)
	pub.dict = NewPubDictFromProto(table.GetDict())
	pub.sealed = table.GetSealed()
	pub.graph = make(Graph, table.GetNodeCt())
	for i := 0; i < len(table.Node); i++ {
		pub.graph[table.Node[i]] = table.AdjList[i].Edge
	}
	return pub
}

func (pub *PubStore) GetProto() *pb.Store {
	adjList := make([]*pb.Store_AdjList, 0)
	node := make([]int32, 0)
	for i := 0; i < len(pub.graph); i++ {
		if len(pub.graph[i]) > 0 {
			node = append(node, int32(i))
			adjList = append(adjList,
				&pb.Store_AdjList{Edge: pub.graph[i]})
		}
	}
	return &pb.Store{
		Dict:    pub.dict.GetProto(),
		Sealed:  pub.sealed,
		Node:    node,
		AdjList: adjList,
		NodeCt:  int32(len(pub.graph)),
	}
}

func (pub *PubStore) String() string {
	str := pub.dict.String()
	for i := 0; i < len(pub.sealed); i++ {
		str += fmt.Sprintf("%s\n", hex.EncodeToString(pub.sealed[i]))
	}
	return str
}

func NewPrivStore(K []byte, params *pb.Params) (*PrivStore, error) {
	priv := new(PrivStore)

	block, err := aes.NewCipher(K[:DictKeyBytes])
	if err != nil {
		return nil, err
	}
	priv.aead, err = cipher.NewGCM(block)
	if err != nil {
		return nil, err
	}

	priv.dict, err = NewPrivDict(K[DictKeyBytes:], params)
	if err != nil {
		return nil, err
	}
	return priv, nil
}

func (priv *PrivStore) GetParams() *pb.Params {
	return priv.dict.GetParams()
}
<commit_msg>Collect TOODs.<commit_after>package store

// TODO Clean up documentation.
// TODO The protocol could be modified so that the client checks if its query
// is in the set before actually requesting it from the server. I should
// evaluate experimentally how frequently the server ends up sending a
// ciphertext needless for an incorrect query.
// TODO store.go:104
// TODO Should Store and Dict implement the same interface?

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/sha256"
	"encoding/binary"
	"encoding/hex"
	"fmt"

	"github.com/cjpatton/store/pb"
	"golang.org/x/crypto/pbkdf2"
)

const SealKeyBytes = 16
const KeyBytes = DictKeyBytes + SealKeyBytes

const ErrorMapTooLarge = Error("input map is too large")

// GenerateKey generates a fresh, random key and returns it.
func GenerateKey() []byte {
	K := make([]byte, KeyBytes)
	_, err := rand.Read(K)
	if err != nil {
		return nil
	}
	return K
}

// DeriveKeyFromPassword derives a key from a password and (optional) salt and
// returns it.
//
// If salt == nil, then no salt is used. Note that the salt is not the same as
// pb.Params.Salt. pb.Params.Salt is generated by NewDict(), which in turn
// depends on the key.
func DeriveKeyFromPassword(password, salt []byte) []byte {
	return pbkdf2.Key(password, salt, 4096, KeyBytes, sha256.New)
}

type PubStore struct {
	dict   *PubDict
	nonce  []byte
	sealed [][]byte
	graph  Graph
}

type PrivStore struct {
	dict *PrivDict
	aead cipher.AEAD
}

func NewStore(K []byte, M map[string]string) (*PubStore, *PrivStore, error) {

	cM := newCMap(M)
	defer cM.free()

	pub := new(PubStore)
	priv := new(PrivStore)

	// Set up context for AEAD.
	block, err := aes.NewCipher(K[:DictKeyBytes])
	if err != nil {
		return nil, nil, err
	}
	priv.aead, err = cipher.NewGCM(block)
	if err != nil {
		return nil, nil, err
	}

	// AEAD nonce is the dictionary salt plus a counter.
	//
	// Compute the number of bytes of the nonce allocated for the counter and
	// ensure that it is long enough to uniquely encode each input/output pair
	// in the map.
	ctrBytes := priv.aead.NonceSize() - SaltBytes
	if len(M) > (1 << (8 * uint(ctrBytes))) {
		return nil, nil, ErrorMapTooLarge
	}

	// Create a *cMap for inputs to counters. This is what will actually be
	// stored by pub.dict.
	cN := cM.getCtrMap(ctrBytes)
	defer cN.free()

	// Construct the graph.
	pub.dict, priv.dict, pub.graph, err = newDictAndGraph(
		K[DictKeyBytes:], cN, 0, false)
	if err != nil {
		return nil, nil, err
	}

	// Encrypt each output and store in pub.sealed.
	nonce := cBytesToBytes(priv.dict.params.salt, priv.dict.params.salt_bytes)
	ctr := make([]byte, ctrBytes)
	pub.sealed = make([][]byte, len(M))
	for i := 0; i < len(M); i++ {
		// TODO M and A copied twice unnecessarily
		cMessage, cMessageBytes := cM.getOutput(i)
		cAssociatedData, cAssociatedDataBytes := cM.getInput(i)
		M := cBytesToBytes(cMessage, cMessageBytes)
		A := cBytesToBytes(cAssociatedData, cAssociatedDataBytes)
		binary.LittleEndian.PutUint32(ctr, uint32(i))
		pub.sealed[i] = priv.aead.Seal(nil, append(nonce, ctr...), M, A)
	}

	return pub, priv, nil
}

func (priv *PrivStore) GetIdx(input string) (int, int, error) {
	return priv.dict.GetIdx(input)
}

func (pub *PubStore) GetShare(x, y int) ([]byte, error) {

	// Get counter share.
	ctrShare, err := pub.dict.GetShare(x, y)
	if err != nil {
		return nil, err
	}

	// Look up sealed output.
	for i := 0; i < len(pub.graph[x]); i++ {
		e := pub.graph[x][i]
		for j := 0; j < len(pub.graph[y]); j++ {
			if pub.graph[y][j] == e {
				return append(ctrShare, pub.sealed[e]...), nil
			}
		}
	}

	return nil, ItemNotFound
}

func (priv *PrivStore) GetOutput(input string, pubShare []byte) (string, error) {
	ctrShareBytes := priv.dict.params.row_bytes
	ctr, err := priv.dict.GetOutput(input, pubShare[:ctrShareBytes])
	if err != nil {
		return "", err
	}

	nonce := cBytesToBytes(priv.dict.params.salt, priv.dict.params.salt_bytes)
	output, err := priv.aead.Open(
		nil, append(nonce, []byte(ctr)...), pubShare[ctrShareBytes:], []byte(input))
	if err != nil {
		return "", ItemNotFound
	}
	return string(output), nil
}

func (pub *PubStore) Free() {
	pub.dict.Free()
}

func (priv *PrivStore) Free() {
	priv.dict.Free()
}

func NewPubStoreFromProto(table *pb.Store) *PubStore {
	pub := new(PubStore)
	pub.dict = NewPubDictFromProto(table.GetDict())
	pub.sealed = table.GetSealed()
	pub.graph = make(Graph, table.GetNodeCt())
	for i := 0; i < len(table.Node); i++ {
		pub.graph[table.Node[i]] = table.AdjList[i].Edge
	}
	return pub
}

func (pub *PubStore) GetProto() *pb.Store {
	adjList := make([]*pb.Store_AdjList, 0)
	node := make([]int32, 0)
	for i := 0; i < len(pub.graph); i++ {
		if len(pub.graph[i]) > 0 {
			node = append(node, int32(i))
			adjList = append(adjList,
				&pb.Store_AdjList{Edge: pub.graph[i]})
		}
	}
	return &pb.Store{
		Dict:    pub.dict.GetProto(),
		Sealed:  pub.sealed,
		Node:    node,
		AdjList: adjList,
		NodeCt:  int32(len(pub.graph)),
	}
}

func (pub *PubStore) String() string {
	str := pub.dict.String()
	for i := 0; i < len(pub.sealed); i++ {
		str += fmt.Sprintf("%s\n", hex.EncodeToString(pub.sealed[i]))
	}
	return str
}

func NewPrivStore(K []byte, params *pb.Params) (*PrivStore, error) {
	priv := new(PrivStore)

	block, err := aes.NewCipher(K[:DictKeyBytes])
	if err != nil {
		return nil, err
	}
	priv.aead, err = cipher.NewGCM(block)
	if err != nil {
		return nil, err
	}

	priv.dict, err = NewPrivDict(K[DictKeyBytes:], params)
	if err != nil {
		return nil, err
	}
	return priv, nil
}

func (priv *PrivStore) GetParams() *pb.Params {
	return priv.dict.GetParams()
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreStore Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"errors"
	"fmt"

	"github.com/corestoreio/csfw/storage/csdb"
	"github.com/corestoreio/csfw/storage/dbr"
	"github.com/corestoreio/csfw/utils/log"
	"github.com/spf13/cast"
	"github.com/spf13/viper"
)

const (
	LeftDelim  = "{{"
	RightDelim = "}}"

	// PathCSBaseURL main CoreStore base URL, used if no configuration on a store level can be found.
	PathCSBaseURL = "web/corestore/base_url"
	CSBaseURL     = "http://localhost:9500/"
)

const (
	URLTypeAbsent URLType = iota
	// UrlTypeWeb defines the ULR type to generate the main base URL.
	URLTypeWeb
	// UrlTypeStatic defines the url to the static assets like css, js or theme images
	URLTypeStatic
	// UrlTypeLink hmmm
	// UrlTypeLink
	// UrlTypeMedia defines the ULR type for generating URLs to product photos
	URLTypeMedia
)

type (
	// UrlType defines the type of the URL. Used in const declaration.
	// @see https://github.com/magento/magento2/blob/0.74.0-beta7/lib/internal/Magento/Framework/UrlInterface.php#L13
	URLType uint8

	Reader interface {
		// GetString returns a string from the manager. Example usage:
		// Default value: GetString(config.Path("general/locale/timezone"))
		// Website value: GetString(config.Path("general/locale/timezone"), config.ScopeWebsite(w))
		// Store   value: GetString(config.Path("general/locale/timezone"), config.ScopeStore(s))
		GetString(...ScopeOption) string

		// GetBool returns bool from the manager. Example usage see GetString.
		GetBool(...ScopeOption) bool
	}

	Writer interface {
		// Write puts a value back into the manager. Example usage:
		// Default Scope: Write(config.Path("currency", "option", "base"), config.Value("USD"))
		// Website Scope: Write(config.Path("currency", "option", "base"), config.Value("EUR"), config.ScopeWebsite(w))
		// Store   Scope: Write(config.Path("currency", "option", "base"), config.ValueReader(resp.Body), config.ScopeStore(s))
		Write(...ScopeOption) error
	}

	// Manager main configuration struct
	Manager struct {
		// why is Viper private? Because it can maybe replaced by something else ...
		v *viper.Viper
	}
)

var (
	_ Reader = (*Manager)(nil)
	_ Writer = (*Manager)(nil)

	// TableCollection handles all tables and its columns. init() in generated Go file will set the value.
	TableCollection csdb.TableStructureSlice
	ErrNoArguments  = errors.New("No arguments provided")
	DefaultManager  *Manager
)

func init() {
	DefaultManager = NewManager()
}

// NewManager creates the main new configuration for all scopes: default, website and store
func NewManager() *Manager {
	s := &Manager{
		v: viper.New(),
	}
	s.v.SetDefault(newArg(Path(PathCSBaseURL)).scopePath(), CSBaseURL)
	return s
}

// ApplyDefaults reads the map and applies the keys and values to the default configuration
func (m *Manager) ApplyDefaults(ss Sectioner) *Manager {
	for k, v := range ss.Defaults() {
		if log.IsDebug() {
			log.Debug("Scope=ApplyDefaults", k, v)
		}
		m.v.SetDefault(k, v)
	}
	return m
}

// ApplyCoreConfigData reads the table core_config_data into the Manager and overrides
// existing values.
func (m *Manager) ApplyCoreConfigData(dbrSess dbr.SessionRunner) error {
	var ccd TableCoreConfigDataSlice
	rows, err := csdb.LoadSlice(dbrSess, TableCollection, TableIndexCoreConfigData, &ccd)
	if log.IsDebug() {
		log.Debug("Manager=ApplyCoreConfigData", "rows", rows)
	}
	if err != nil {
		return log.Error("Manager=ApplyCoreConfigData", "err", err)
	}

	for _, cd := range ccd {
		if cd.Value.Valid {
			// ScopeID(cd.ScopeID) because cd.ScopeID is a struct field and cannot satisfy interface ScopeIDer
			m.Write(Path(cd.Path), Scope(GetScopeGroup(cd.Scope), ScopeID(cd.ScopeID)))
		}
		fmt.Printf("\n%#v\n", cd)
	}
	return nil
}

// Write puts a value back into the manager. Example usage:
// Default Scope: Write(config.Path("currency", "option", "base"), config.Value("USD"))
// Website Scope: Write(config.Path("currency", "option", "base"), config.Value("EUR"), config.ScopeWebsite(w))
// Store   Scope: Write(config.Path("currency", "option", "base"), config.ValueReader(resp.Body), config.ScopeStore(s))
func (m *Manager) Write(o ...ScopeOption) error {
	a := newArg(o...)
	if a == nil {
		return ErrNoArguments
	}
	if a.isDefault() || a.isBubbling() {
		if log.IsDebug() {
			log.Debug("Manager=Write", "path", a.scopePathDefault(), "bubble", a.isBubbling(), "val", a.v)
		}
		m.v.SetDefault(a.scopePathDefault(), a.v)
	}
	if false == a.isDefault() {
		if log.IsDebug() {
			log.Debug("Manager=Write", "path", a.scopePath(), "val", a.v)
		}
		m.v.Set(a.scopePath(), a.v)
	}
	return nil
}

func (m *Manager) get(o ...ScopeOption) interface{} {
	a := newArg(o...)
	vs := m.v.Get(a.scopePath()) // vs = value scope
	if vs == nil && a.isBubbling() {
		vs = m.v.Get(a.scopePathDefault())
	}
	return vs
}

// GetString returns a string from the manager. Example usage:
// Default value: GetString(config.Path("general/locale/timezone"))
// Website value: GetString(config.Path("general/locale/timezone"), config.ScopeWebsite(w))
// Store   value: GetString(config.Path("general/locale/timezone"), config.ScopeStore(s))
func (m *Manager) GetString(o ...ScopeOption) string {
	vs := m.get(o...)
	if vs == nil {
		return ""
	}
	return cast.ToString(vs)
}

// @todo use the backend model of a config value. most/all magento string slices are comma lists.
func (m *Manager) GetStringSlice(o ...ScopeOption) []string {
	return nil
	//	return m.v.GetStringSlice(newArg(o...))
}

// GetBool returns bool from the manager. Example usage see GetString.
func (m *Manager) GetBool(o ...ScopeOption) bool {
	vs := m.get(o...)
	if vs == nil {
		return false
	}
	return cast.ToBool(vs)
}

// GetFloat64 returns a float64 from the manager. Example usage see GetString.
func (m *Manager) GetFloat64(o ...ScopeOption) float64 {
	vs := m.get(o...)
	if vs == nil {
		return 0.0
	}
	return cast.ToFloat64(vs)
}

// @todo consider adding other Get* from the viper package

// AllKeys return all keys regardless where they are set
func (m *Manager) AllKeys() []string { return m.v.AllKeys() }

var _ Reader = (*mockReader)(nil)

// MockScopeReader used for testing
type mockReader struct {
	s func(path string) string
	b func(path string) bool
}

// NewMockReader used for testing
func NewMockReader(
	s func(path string) string,
	b func(path string) bool,
) *mockReader {

	return &mockReader{
		s: s,
		b: b,
	}
}

func (sr mockReader) GetString(opts ...ScopeOption) string {
	if sr.s == nil {
		return ""
	}
	return sr.s(newArg(opts...).scopePath())
}

func (sr mockReader) GetBool(opts ...ScopeOption) bool {
	if sr.b == nil {
		return false
	}
	return sr.b(newArg(opts...).scopePath())
}
<commit_msg>Package config: Manager add GetInt, GetDateTime<commit_after>// Copyright 2015 CoreStore Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"errors"
	"fmt"
	"time"

	"github.com/corestoreio/csfw/storage/csdb"
	"github.com/corestoreio/csfw/storage/dbr"
	"github.com/corestoreio/csfw/utils/log"
	"github.com/spf13/cast"
	"github.com/spf13/viper"
)

const (
	LeftDelim  = "{{"
	RightDelim = "}}"

	// PathCSBaseURL main CoreStore base URL, used if no configuration on a store level can be found.
	PathCSBaseURL = "web/corestore/base_url"
	CSBaseURL     = "http://localhost:9500/"
)

const (
	URLTypeAbsent URLType = iota
	// UrlTypeWeb defines the ULR type to generate the main base URL.
	URLTypeWeb
	// UrlTypeStatic defines the url to the static assets like css, js or theme images
	URLTypeStatic
	// UrlTypeLink hmmm
	// UrlTypeLink
	// UrlTypeMedia defines the ULR type for generating URLs to product photos
	URLTypeMedia
)

type (
	// UrlType defines the type of the URL. Used in const declaration.
	// @see https://github.com/magento/magento2/blob/0.74.0-beta7/lib/internal/Magento/Framework/UrlInterface.php#L13
	URLType uint8

	Reader interface {
		// GetString returns a string from the manager. Example usage:
		// Default value: GetString(config.Path("general/locale/timezone"))
		// Website value: GetString(config.Path("general/locale/timezone"), config.ScopeWebsite(w))
		// Store   value: GetString(config.Path("general/locale/timezone"), config.ScopeStore(s))
		GetString(...ScopeOption) string

		// GetBool returns bool from the manager. Example usage see GetString.
		GetBool(...ScopeOption) bool
	}

	Writer interface {
		// Write puts a value back into the manager. Example usage:
		// Default Scope: Write(config.Path("currency", "option", "base"), config.Value("USD"))
		// Website Scope: Write(config.Path("currency", "option", "base"), config.Value("EUR"), config.ScopeWebsite(w))
		// Store   Scope: Write(config.Path("currency", "option", "base"), config.ValueReader(resp.Body), config.ScopeStore(s))
		Write(...ScopeOption) error
	}

	// Manager main configuration struct
	Manager struct {
		// why is Viper private? Because it can maybe replaced by something else ...
		v *viper.Viper
	}
)

var (
	_ Reader = (*Manager)(nil)
	_ Writer = (*Manager)(nil)

	// TableCollection handles all tables and its columns. init() in generated Go file will set the value.
	TableCollection csdb.TableStructureSlice
	ErrNoArguments  = errors.New("No arguments provided")
	DefaultManager  *Manager
)

func init() {
	DefaultManager = NewManager()
}

// NewManager creates the main new configuration for all scopes: default, website and store
func NewManager() *Manager {
	s := &Manager{
		v: viper.New(),
	}
	s.v.SetDefault(newArg(Path(PathCSBaseURL)).scopePath(), CSBaseURL)
	return s
}

// ApplyDefaults reads the map and applies the keys and values to the default configuration
func (m *Manager) ApplyDefaults(ss Sectioner) *Manager {
	for k, v := range ss.Defaults() {
		if log.IsDebug() {
			log.Debug("Scope=ApplyDefaults", k, v)
		}
		m.v.SetDefault(k, v)
	}
	return m
}

// ApplyCoreConfigData reads the table core_config_data into the Manager and overrides
// existing values. If the column value is NULL entry will be ignored.
func (m *Manager) ApplyCoreConfigData(dbrSess dbr.SessionRunner) error {
	var ccd TableCoreConfigDataSlice
	rows, err := csdb.LoadSlice(dbrSess, TableCollection, TableIndexCoreConfigData, &ccd)
	if log.IsDebug() {
		log.Debug("Manager=ApplyCoreConfigData", "rows", rows)
	}
	if err != nil {
		return log.Error("Manager=ApplyCoreConfigData", "err", err)
	}

	for _, cd := range ccd {
		if cd.Value.Valid {
			// ScopeID(cd.ScopeID) because cd.ScopeID is a struct field and cannot satisfy interface ScopeIDer
			m.Write(Path(cd.Path), Scope(GetScopeGroup(cd.Scope), ScopeID(cd.ScopeID)))
		}
		fmt.Printf("\n%#v\n", cd)
	}
	return nil
}

// Write puts a value back into the manager. Example usage:
// Default Scope: Write(config.Path("currency", "option", "base"), config.Value("USD"))
// Website Scope: Write(config.Path("currency", "option", "base"), config.Value("EUR"), config.ScopeWebsite(w))
// Store   Scope: Write(config.Path("currency", "option", "base"), config.ValueReader(resp.Body), config.ScopeStore(s))
func (m *Manager) Write(o ...ScopeOption) error {
	a := newArg(o...)
	if a == nil {
		return ErrNoArguments
	}
	if a.isDefault() || a.isBubbling() {
		if log.IsDebug() {
			log.Debug("Manager=Write", "path", a.scopePathDefault(), "bubble", a.isBubbling(), "val", a.v)
		}
		m.v.SetDefault(a.scopePathDefault(), a.v)
	}
	if false == a.isDefault() {
		if log.IsDebug() {
			log.Debug("Manager=Write", "path", a.scopePath(), "val", a.v)
		}
		m.v.Set(a.scopePath(), a.v)
	}
	return nil
}

func (m *Manager) get(o ...ScopeOption) interface{} {
	a := newArg(o...)
	vs := m.v.Get(a.scopePath()) // vs = value scope
	if vs == nil && a.isBubbling() {
		vs = m.v.Get(a.scopePathDefault())
	}
	return vs
}

// GetString returns a string from the manager. Example usage:
// Default value: GetString(config.Path("general/locale/timezone"))
// Website value: GetString(config.Path("general/locale/timezone"), config.ScopeWebsite(w))
// Store   value: GetString(config.Path("general/locale/timezone"), config.ScopeStore(s))
func (m *Manager) GetString(o ...ScopeOption) string {
	vs := m.get(o...)
	if vs == nil {
		return ""
	}
	return cast.ToString(vs)
}

// @todo use the backend model of a config value. most/all magento string slices are comma lists.
func (m *Manager) GetStringSlice(o ...ScopeOption) []string {
	return nil
	//	return m.v.GetStringSlice(newArg(o...))
}

// GetBool returns bool from the manager. Example usage see GetString.
func (m *Manager) GetBool(o ...ScopeOption) bool {
	vs := m.get(o...)
	if vs == nil {
		return false
	}
	return cast.ToBool(vs)
}

// GetFloat64 returns a float64 from the manager. Example usage see GetString.
func (m *Manager) GetFloat64(o ...ScopeOption) float64 {
	vs := m.get(o...)
	if vs == nil {
		return 0.0
	}
	return cast.ToFloat64(vs)
}

// GetInt returns an int from the manager. Example usage see GetString.
func (m *Manager) GetInt(o ...ScopeOption) int {
	vs := m.get(o...)
	if vs == nil {
		return 0
	}
	return cast.ToInt(vs)
}

// GetDateTime returns a date and time object from the manager. Example usage see GetString.
func (m *Manager) GetDateTime(o ...ScopeOption) time.Time {
	vs := m.get(o...)
	t, err := cast.ToTimeE(vs)
	if err != nil {
		log.Error("Manager=GetDateTime", "err", err, "val", vs)
	}
	return t
}

// @todo consider adding other Get* from the viper package

// AllKeys return all keys regardless where they are set
func (m *Manager) AllKeys() []string { return m.v.AllKeys() }

var _ Reader = (*mockReader)(nil)

// MockScopeReader used for testing
type mockReader struct {
	s func(path string) string
	b func(path string) bool
}

// NewMockReader used for testing
func NewMockReader(
	s func(path string) string,
	b func(path string) bool,
) *mockReader {

	return &mockReader{
		s: s,
		b: b,
	}
}

func (sr mockReader) GetString(opts ...ScopeOption) string {
	if sr.s == nil {
		return ""
	}
	return sr.s(newArg(opts...).scopePath())
}

func (sr mockReader) GetBool(opts ...ScopeOption) bool {
	if sr.b == nil {
		return false
	}
	return sr.b(newArg(opts...).scopePath())
}
<|endoftext|>


<commit_before>// Copyright 2015, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"bytes"
	"encoding/json"
	"errors"
	"sort"

	"github.com/corestoreio/csfw/config/scope"
	"github.com/corestoreio/csfw/utils"
	"github.com/corestoreio/csfw/utils/log"
	"github.com/juju/errgo"
)

// ErrSectionNotFound error when a section cannot be found.
var ErrSectionNotFound = errors.New("Section not found")

type (
	// Sectioner at the moment only for testing
	Sectioner interface {
		// Defaults generates the default configuration from all fields. Key is the path and value the value.
		Defaults() DefaultMap
	}

	// SectionSlice contains a set of Sections. Some nifty helper functions exists.
	SectionSlice []*Section
	// Section defines the layout for the configuration section which contains groups and fields.
	Section struct {
		// ID unique ID and merged with others. 1st part of the path.
		ID    string
		Label string `json:",omitempty"`
		// Scope: bit value eg: showInDefault="1" showInWebsite="1" showInStore="1"
		Scope     scope.Perm `json:",omitempty"`
		SortOrder int        `json:",omitempty"`
		// Permission some kind of ACL if someone is allowed for no,read or write access @todo
		Permission uint `json:",omitempty"`
		Groups     GroupSlice
	}
)

var _ Sectioner = (*SectionSlice)(nil)

// NewConfiguration creates a new validated SectionSlice with a three level configuration.
// Panics if a path is redundant.
func NewConfiguration(sections ...*Section) SectionSlice {
	ss := SectionSlice(sections)
	if err := ss.Validate(); err != nil {
		if log.IsWarn() {
			log.Warn("config.NewConfiguration.Validate", "err", err)
		}
		panic(err)
	}
	return ss
}

// NewConfigurationMerge creates a new validated SectionSlice with a three level configuration.
// Before validation, slices are all merged together. Panics if a path is redundant.
// Only use this function if your package configuration really has duplicated entries.
func NewConfigurationMerge(sections ...*Section) SectionSlice {
	var ss SectionSlice
	ss.Merge(sections...)
	if err := ss.Validate(); err != nil {
		if log.IsWarn() {
			log.Warn("config.NewConfigurationMerge.Validate", "err", err)
		}
		panic(err)
	}
	return ss
}

// Defaults iterates over all slices, creates a path and uses the default value
// to return a map.
func (ss SectionSlice) Defaults() DefaultMap {
	var dm = make(DefaultMap)
	for _, s := range ss {
		for _, g := range s.Groups {
			for _, f := range g.Fields {
				arg, err := newArg(Path(s.ID, g.ID, f.ID))
				if err != nil {
					log.Error("config.SectionSlice.Defaults.newArg", "err", err, "s", s, "g", g, "f", f)
				}
				dm[arg.scopePath()] = f.Default
			}
		}
	}
	return dm
}

// TotalFields calculates the total amount of all fields
func (ss SectionSlice) TotalFields() int {
	fs := 0
	for _, s := range ss {
		for _, g := range s.Groups {
			for range g.Fields {
				fs++
			}
		}
	}
	return -^+^-fs
}

// MergeMultiple merges n SectionSlices into the current slice. Behaviour for duplicates: Last item wins.
func (ss *SectionSlice) MergeMultiple(sSlices ...SectionSlice) error {
	for _, sl := range sSlices {
		if err := (*ss).Merge(sl...); err != nil {
			return err
		}
	}
	return nil
}

// Merge merges n Sections into the current slice. Behaviour for duplicates: Last item wins.
func (ss *SectionSlice) Merge(sections ...*Section) error {
	for _, s := range sections {
		if s != nil {
			if err := (*ss).merge(s); err != nil {
				return errgo.Mask(err)
			}
		}
	}
	return nil
}

// Merge copies the data from a Section into this slice. Appends if ID is not found
// in this slice otherwise overrides struct fields if not empty.
func (ss *SectionSlice) merge(s *Section) error {
	if s == nil {
		return nil
	}
	cs, err := (*ss).FindByID(s.ID) // cs current section
	if cs == nil || err != nil {
		cs = &Section{ID: s.ID}
		(*ss).Append(cs)
	}

	if s.Label != "" {
		cs.Label = s.Label
	}
	if s.Scope > 0 {
		cs.Scope = s.Scope
	}
	if s.SortOrder != 0 {
		cs.SortOrder = s.SortOrder
	}
	if s.Permission > 0 {
		cs.Permission = s.Permission
	}
	return cs.Groups.Merge(s.Groups...)
}

// FindByID returns a Section pointer or nil if not found. Please check for nil and do not a
func (ss SectionSlice) FindByID(id string) (*Section, error) {
	for _, s := range ss {
		if s != nil && s.ID == id {
			return s, nil
		}
	}
	return nil, ErrSectionNotFound
}

// FindGroupByPath searches for a group using the first two path segments.
// If one argument is given then considered as the full path e.g. a/b/c
// If two or more arguments are given then each argument will be treated as a path part.
func (ss SectionSlice) FindGroupByPath(paths ...string) (*Group, error) {
	if len(paths) == 1 {
		paths = scope.PathSplit(paths[0])
	}
	if len(paths) < 2 {
		return nil, errgo.Mask(ErrGroupNotFound)
	}
	cs, err := ss.FindByID(paths[0])
	if err != nil {
		return nil, errgo.Mask(err)
	}
	return cs.Groups.FindByID(paths[1])
}

// FindFieldByPath searches for a field using all three path segments.
// If one argument is given then considered as the full path e.g. a/b/c
// If three arguments are given then each argument will be treated as a path part.
func (ss SectionSlice) FindFieldByPath(paths ...string) (*Field, error) {
	if len(paths) == 1 {
		paths = scope.PathSplit(paths[0])
	}
	if len(paths) < 3 {
		return nil, errgo.Mask(ErrFieldNotFound)
	}
	cg, err := ss.FindGroupByPath(paths...)
	if err != nil {
		return nil, errgo.Mask(err)
	}
	return cg.Fields.FindByID(paths[2])
}

// Append adds 0..n *Section
func (ss *SectionSlice) Append(s ...*Section) *SectionSlice {
	*ss = append(*ss, s...)
	return ss
}

// ToJSON transforms the whole slice into JSON
func (ss SectionSlice) ToJSON() string {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(ss); err != nil {
		log.Error("config.SectionSlice.ToJSON.Encode", "err", err)
		return ""
	}
	return buf.String()
}

// Validate checks for duplicated configuration paths in all three hierarchy levels.
func (ss SectionSlice) Validate() error {
	if len(ss) == 0 {
		return errgo.New("SectionSlice is empty")
	}
	// @todo try to pick the right strategy between maps and slice depending on the overall size of a full SectionSlice
	var pc = make(utils.StringSlice, ss.TotalFields()) // pc path checker
	i := 0
	for _, s := range ss {
		for _, g := range s.Groups {
			for _, f := range g.Fields {
				arg, err := newArg(Path(s.ID, g.ID, f.ID))
				if err != nil {
					log.Error("config.SectionSlice.Validate.newArg", "err", err, "s", s, "g", g, "f", f)
				}
				p := arg.scopePath()
				if pc.Include(p) {
					return errgo.Newf("Duplicate entry for path %s :: %s", p, ss.ToJSON())
				}
				pc[i] = p
				i++
			}
		}
	}
	return nil
}

// SortAll recursively sorts all slices
func (ss *SectionSlice) SortAll() *SectionSlice {
	for _, s := range *ss {
		for _, g := range s.Groups {
			g.Fields.Sort()
		}
		s.Groups.Sort()
	}
	return ss.Sort()
}

// Sort convenience helper
func (ss *SectionSlice) Sort() *SectionSlice {
	sort.Sort(ss)
	return ss
}

func (ss *SectionSlice) Len() int {
	return len(*ss)
}

func (ss *SectionSlice) Swap(i, j int) {
	(*ss)[i], (*ss)[j] = (*ss)[j], (*ss)[i]
}

func (ss *SectionSlice) Less(i, j int) bool {
	return (*ss)[i].SortOrder < (*ss)[j].SortOrder
}
<commit_msg>config: Remove errgo.Mask from SecionSlice<commit_after>// Copyright 2015, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"bytes"
	"encoding/json"
	"errors"
	"sort"

	"github.com/corestoreio/csfw/config/scope"
	"github.com/corestoreio/csfw/utils"
	"github.com/corestoreio/csfw/utils/log"
	"github.com/juju/errgo"
)

// ErrSectionNotFound error when a section cannot be found.
var ErrSectionNotFound = errors.New("Section not found")

type (
	// Sectioner at the moment only for testing
	Sectioner interface {
		// Defaults generates the default configuration from all fields. Key is the path and value the value.
		Defaults() DefaultMap
	}

	// SectionSlice contains a set of Sections. Some nifty helper functions exists.
	SectionSlice []*Section
	// Section defines the layout for the configuration section which contains groups and fields.
	Section struct {
		// ID unique ID and merged with others. 1st part of the path.
		ID    string
		Label string `json:",omitempty"`
		// Scope: bit value eg: showInDefault="1" showInWebsite="1" showInStore="1"
		Scope     scope.Perm `json:",omitempty"`
		SortOrder int        `json:",omitempty"`
		// Permission some kind of ACL if someone is allowed for no,read or write access @todo
		Permission uint `json:",omitempty"`
		Groups     GroupSlice
	}
)

var _ Sectioner = (*SectionSlice)(nil)

// NewConfiguration creates a new validated SectionSlice with a three level configuration.
// Panics if a path is redundant.
func NewConfiguration(sections ...*Section) SectionSlice {
	ss := SectionSlice(sections)
	if err := ss.Validate(); err != nil {
		if log.IsWarn() {
			log.Warn("config.NewConfiguration.Validate", "err", err)
		}
		panic(err)
	}
	return ss
}

// NewConfigurationMerge creates a new validated SectionSlice with a three level configuration.
// Before validation, slices are all merged together. Panics if a path is redundant.
// Only use this function if your package configuration really has duplicated entries.
func NewConfigurationMerge(sections ...*Section) SectionSlice {
	var ss SectionSlice
	ss.Merge(sections...)
	if err := ss.Validate(); err != nil {
		if log.IsWarn() {
			log.Warn("config.NewConfigurationMerge.Validate", "err", err)
		}
		panic(err)
	}
	return ss
}

// Defaults iterates over all slices, creates a path and uses the default value
// to return a map.
func (ss SectionSlice) Defaults() DefaultMap {
	var dm = make(DefaultMap)
	for _, s := range ss {
		for _, g := range s.Groups {
			for _, f := range g.Fields {
				arg, err := newArg(Path(s.ID, g.ID, f.ID))
				if err != nil {
					log.Error("config.SectionSlice.Defaults.newArg", "err", err, "s", s, "g", g, "f", f)
				}
				dm[arg.scopePath()] = f.Default
			}
		}
	}
	return dm
}

// TotalFields calculates the total amount of all fields
func (ss SectionSlice) TotalFields() int {
	fs := 0
	for _, s := range ss {
		for _, g := range s.Groups {
			for range g.Fields {
				fs++
			}
		}
	}
	return -^+^-fs
}

// MergeMultiple merges n SectionSlices into the current slice. Behaviour for duplicates: Last item wins.
func (ss *SectionSlice) MergeMultiple(sSlices ...SectionSlice) error {
	for _, sl := range sSlices {
		if err := (*ss).Merge(sl...); err != nil {
			return err
		}
	}
	return nil
}

// Merge merges n Sections into the current slice. Behaviour for duplicates: Last item wins.
func (ss *SectionSlice) Merge(sections ...*Section) error {
	for _, s := range sections {
		if s != nil {
			if err := (*ss).merge(s); err != nil {
				return errgo.Mask(err)
			}
		}
	}
	return nil
}

// Merge copies the data from a Section into this slice. Appends if ID is not found
// in this slice otherwise overrides struct fields if not empty.
func (ss *SectionSlice) merge(s *Section) error {
	if s == nil {
		return nil
	}
	cs, err := (*ss).FindByID(s.ID) // cs current section
	if cs == nil || err != nil {
		cs = &Section{ID: s.ID}
		(*ss).Append(cs)
	}

	if s.Label != "" {
		cs.Label = s.Label
	}
	if s.Scope > 0 {
		cs.Scope = s.Scope
	}
	if s.SortOrder != 0 {
		cs.SortOrder = s.SortOrder
	}
	if s.Permission > 0 {
		cs.Permission = s.Permission
	}
	return cs.Groups.Merge(s.Groups...)
}

// FindByID returns a Section pointer or nil if not found. Please check for nil and do not a
func (ss SectionSlice) FindByID(id string) (*Section, error) {
	for _, s := range ss {
		if s != nil && s.ID == id {
			return s, nil
		}
	}
	return nil, ErrSectionNotFound
}

// FindGroupByPath searches for a group using the first two path segments.
// If one argument is given then considered as the full path e.g. a/b/c
// If two or more arguments are given then each argument will be treated as a path part.
func (ss SectionSlice) FindGroupByPath(paths ...string) (*Group, error) {
	if len(paths) == 1 {
		paths = scope.PathSplit(paths[0])
	}
	if len(paths) < 2 {
		return nil, ErrGroupNotFound
	}
	cs, err := ss.FindByID(paths[0])
	if err != nil {
		return nil, errgo.Mask(err)
	}
	return cs.Groups.FindByID(paths[1])
}

// FindFieldByPath searches for a field using all three path segments.
// If one argument is given then considered as the full path e.g. a/b/c
// If three arguments are given then each argument will be treated as a path part.
func (ss SectionSlice) FindFieldByPath(paths ...string) (*Field, error) {
	if len(paths) == 1 {
		paths = scope.PathSplit(paths[0])
	}
	if len(paths) < 3 {
		return nil, ErrFieldNotFound
	}
	cg, err := ss.FindGroupByPath(paths...)
	if err != nil {
		return nil, errgo.Mask(err)
	}
	return cg.Fields.FindByID(paths[2])
}

// Append adds 0..n *Section
func (ss *SectionSlice) Append(s ...*Section) *SectionSlice {
	*ss = append(*ss, s...)
	return ss
}

// ToJSON transforms the whole slice into JSON
func (ss SectionSlice) ToJSON() string {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(ss); err != nil {
		log.Error("config.SectionSlice.ToJSON.Encode", "err", err)
		return ""
	}
	return buf.String()
}

// Validate checks for duplicated configuration paths in all three hierarchy levels.
func (ss SectionSlice) Validate() error {
	if len(ss) == 0 {
		return errgo.New("SectionSlice is empty")
	}
	// @todo try to pick the right strategy between maps and slice depending on the overall size of a full SectionSlice
	var pc = make(utils.StringSlice, ss.TotalFields()) // pc path checker
	i := 0
	for _, s := range ss {
		for _, g := range s.Groups {
			for _, f := range g.Fields {
				arg, err := newArg(Path(s.ID, g.ID, f.ID))
				if err != nil {
					log.Error("config.SectionSlice.Validate.newArg", "err", err, "s", s, "g", g, "f", f)
				}
				p := arg.scopePath()
				if pc.Include(p) {
					return errgo.Newf("Duplicate entry for path %s :: %s", p, ss.ToJSON())
				}
				pc[i] = p
				i++
			}
		}
	}
	return nil
}

// SortAll recursively sorts all slices
func (ss *SectionSlice) SortAll() *SectionSlice {
	for _, s := range *ss {
		for _, g := range s.Groups {
			g.Fields.Sort()
		}
		s.Groups.Sort()
	}
	return ss.Sort()
}

// Sort convenience helper
func (ss *SectionSlice) Sort() *SectionSlice {
	sort.Sort(ss)
	return ss
}

func (ss *SectionSlice) Len() int {
	return len(*ss)
}

func (ss *SectionSlice) Swap(i, j int) {
	(*ss)[i], (*ss)[j] = (*ss)[j], (*ss)[i]
}

func (ss *SectionSlice) Less(i, j int) bool {
	return (*ss)[i].SortOrder < (*ss)[j].SortOrder
}
<|endoftext|>