<commit_before>package gads

import "encoding/xml"

// CommonAd define the parent type Ad type as defined
// https://developers.google.com/adwords/api/docs/reference/v201506/AdGroupAdService.Ad
type CommonAd struct {
	Type                string            `xml:"xsi:type,attr,omitempty"`
	ID                  int64             `xml:"id,omitempty"`
	URL                 *string            `xml:"url"`
	DisplayURL          string            `xml:"displayUrl,omitempty"`
	FinalURLs           []string          `xml:"finalUrls,omitempty"`
	FinalMobileURLs     []string          `xml:"finalMobileUrls,omitempty"`
	FinalAppURLs        []AppUrl          `xml:"finalAppUrls,omitempty"`
	TrackingURLTemplate *string           `xml:"trackingUrlTemplate"`
	URLCustomParameters *CustomParameters `xml:"urlCustomParameters,omitempty"`
	DevicePreference    int64             `xml:"devicePreference,omitempty"`
}

// Ad is the common interface for accessing properties shared by all
// child types of ads
type Ad interface {
	GetID() int64
	GetURL() string
	GetTrackingURLTemplate() *string
	GetFinalURLs() []string

	CloneForTemplate([]string, *string) Ad
}

// GetID returns the Ad's id
func (c CommonAd) GetID() int64 {
	return c.ID
}

// GetURL returns the Ad's url (a.k.a destination url)
func (c CommonAd) GetURL() string {
	return c.URL
}

// GetTrackingURLTemplate returns the  tracking url template
func (c CommonAd) GetTrackingURLTemplate() *string {
	return c.TrackingURLTemplate
}

// GetFinalURLs returns the list of Final urls (can be empty array)
func (c CommonAd) GetFinalURLs() []string {
	return c.FinalURLs
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template (as Ad are immutable)
func (c CommonAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0   // value used by go for omitempty
	c.URL = "" // template needs an empty destination url (as it deprecates this field)
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template
func (c TextAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0   // value used by go for omitempty
	c.URL = "" // template needs an empty destination url (as it deprecates this field)
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate

	return c
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template
func (c ImageAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0   // value used by go for omitempty
	c.URL = "" // template needs an empty destination url (as it deprecates this field)
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template
func (c TemplateAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0   // value used by go for omitempty
	c.URL = "" // template needs an empty destination url (as it deprecates this field)
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template
func (c MobileAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0   // value used by go for omitempty
	c.URL = "" // template needs an empty destination url (as it deprecates this field)
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template
func (c DynamicSearchAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0   // value used by go for omitempty
	c.URL = "" // template needs an empty destination url (as it deprecates this field)
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

// TextAd represents the TextAd object as documented here
// https://developers.google.com/adwords/api/docs/reference/v201506/AdGroupAdService.TextAd
type TextAd struct {
	CommonAd
	Headline     string `xml:"headline"`
	Description1 string `xml:"description1"`
	Description2 string `xml:"description2"`
}

// DynamicSearchAd represents the equivalent object documented here
// https://developers.google.com/adwords/api/docs/reference/v201502/AdGroupAdService.DynamicSearchAd
type DynamicSearchAd struct {
	CommonAd
	Description1 string `xml:"description1"`
	Description2 string `xml:"description2"`
}

// ImageAd represents the equivalent object documented here
// https://developers.google.com/adwords/api/docs/reference/v201502/AdGroupAdService.ImageAd
type ImageAd struct {
	CommonAd
	Image             int64  `xml:"imageId"` //TODO should actually be Image object, not just an int
	Name              string `xml:"name"`
	AdToCopyImageFrom int64  `xml:"adToCopyImageFrom"`
}

// MobileAd represents the equivalent object documented here
// https://developers.google.com/adwords/api/docs/reference/v201502/AdGroupAdService.MobileAd
type MobileAd struct {
	CommonAd
	Headline        string   `xml:"headline"`
	Description     string   `xml:"description"`
	MarkupLanguages []string `xml:"markupLanguages"`
	MobileCarriers  []string `xml:"mobileCarriers"`
	BusinessName    string   `xml:"businessName"`
	CountryCode     string   `xml:"countryCode"`
	PhoneNumber     string   `xml:"phoneNumber"`
}

// TemplateAd represents the equivalent object documented here
// https://developers.google.com/adwords/api/docs/reference/v201502/AdGroupAdService.TemplateAd
type TemplateAd struct {
	CommonAd
	TemplateID       int64             `xml:"templateId"`
	AdUnionID        int64             `xml:"adUnionId"`
	TemplateElements []TemplateElement `xml:"templateElements"`
	Dimensions       []Dimensions      `xml:"dimensions"`
	Name             string            `xml:"name"`
	Duration         int64             `xml:"duration"`
	originAdID       *int64            `xml:"originAdId"`
}

// MarshalXML returns unimplemented error as the structure does not
// match yet 100% of the field required by google api
func (c ImageAd) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	return ERROR_NOT_YET_IMPLEMENTED
}

// MarshalXML returns unimplemented error as the structure does not
// match yet 100% of the field required by google api
func (c TemplateAd) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	return ERROR_NOT_YET_IMPLEMENTED
}
<commit_msg>we dont need Ad's URL as pointer<commit_after>package gads

import "encoding/xml"

// CommonAd define the parent type Ad type as defined
// https://developers.google.com/adwords/api/docs/reference/v201506/AdGroupAdService.Ad
type CommonAd struct {
	Type                string            `xml:"xsi:type,attr,omitempty"`
	ID                  int64             `xml:"id,omitempty"`
	URL                 string            `xml:"url,omitempty"`
	DisplayURL          string            `xml:"displayUrl,omitempty"`
	FinalURLs           []string          `xml:"finalUrls,omitempty"`
	FinalMobileURLs     []string          `xml:"finalMobileUrls,omitempty"`
	FinalAppURLs        []AppUrl          `xml:"finalAppUrls,omitempty"`
	TrackingURLTemplate *string           `xml:"trackingUrlTemplate"`
	URLCustomParameters *CustomParameters `xml:"urlCustomParameters,omitempty"`
	DevicePreference    int64             `xml:"devicePreference,omitempty"`
}

// Ad is the common interface for accessing properties shared by all
// child types of ads
type Ad interface {
	GetID() int64
	GetURL() string
	GetTrackingURLTemplate() *string
	GetFinalURLs() []string

	CloneForTemplate([]string, *string) Ad
}

// GetID returns the Ad's id
func (c CommonAd) GetID() int64 {
	return c.ID
}

// GetURL returns the Ad's url (a.k.a destination url)
func (c CommonAd) GetURL() string {
	return c.URL
}

// GetTrackingURLTemplate returns the  tracking url template
func (c CommonAd) GetTrackingURLTemplate() *string {
	return c.TrackingURLTemplate
}

// GetFinalURLs returns the list of Final urls (can be empty array)
func (c CommonAd) GetFinalURLs() []string {
	return c.FinalURLs
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template (as Ad are immutable)
func (c CommonAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0   // value used by go for omitempty
	c.URL = "" // template needs an empty destination url (as it deprecates this field)
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template
func (c TextAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0   // value used by go for omitempty
	c.URL = "" // template needs an empty destination url (as it deprecates this field)
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate

	return c
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template
func (c ImageAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0   // value used by go for omitempty
	c.URL = "" // template needs an empty destination url (as it deprecates this field)
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template
func (c TemplateAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0   // value used by go for omitempty
	c.URL = "" // template needs an empty destination url (as it deprecates this field)
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template
func (c MobileAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0   // value used by go for omitempty
	c.URL = "" // template needs an empty destination url (as it deprecates this field)
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template
func (c DynamicSearchAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0   // value used by go for omitempty
	c.URL = "" // template needs an empty destination url (as it deprecates this field)
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

// TextAd represents the TextAd object as documented here
// https://developers.google.com/adwords/api/docs/reference/v201506/AdGroupAdService.TextAd
type TextAd struct {
	CommonAd
	Headline     string `xml:"headline"`
	Description1 string `xml:"description1"`
	Description2 string `xml:"description2"`
}

// DynamicSearchAd represents the equivalent object documented here
// https://developers.google.com/adwords/api/docs/reference/v201502/AdGroupAdService.DynamicSearchAd
type DynamicSearchAd struct {
	CommonAd
	Description1 string `xml:"description1"`
	Description2 string `xml:"description2"`
}

// ImageAd represents the equivalent object documented here
// https://developers.google.com/adwords/api/docs/reference/v201502/AdGroupAdService.ImageAd
type ImageAd struct {
	CommonAd
	Image             int64  `xml:"imageId"` //TODO should actually be Image object, not just an int
	Name              string `xml:"name"`
	AdToCopyImageFrom int64  `xml:"adToCopyImageFrom"`
}

// MobileAd represents the equivalent object documented here
// https://developers.google.com/adwords/api/docs/reference/v201502/AdGroupAdService.MobileAd
type MobileAd struct {
	CommonAd
	Headline        string   `xml:"headline"`
	Description     string   `xml:"description"`
	MarkupLanguages []string `xml:"markupLanguages"`
	MobileCarriers  []string `xml:"mobileCarriers"`
	BusinessName    string   `xml:"businessName"`
	CountryCode     string   `xml:"countryCode"`
	PhoneNumber     string   `xml:"phoneNumber"`
}

// TemplateAd represents the equivalent object documented here
// https://developers.google.com/adwords/api/docs/reference/v201502/AdGroupAdService.TemplateAd
type TemplateAd struct {
	CommonAd
	TemplateID       int64             `xml:"templateId"`
	AdUnionID        int64             `xml:"adUnionId"`
	TemplateElements []TemplateElement `xml:"templateElements"`
	Dimensions       []Dimensions      `xml:"dimensions"`
	Name             string            `xml:"name"`
	Duration         int64             `xml:"duration"`
	originAdID       *int64            `xml:"originAdId"`
}

// MarshalXML returns unimplemented error as the structure does not
// match yet 100% of the field required by google api
func (c ImageAd) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	return ERROR_NOT_YET_IMPLEMENTED
}

// MarshalXML returns unimplemented error as the structure does not
// match yet 100% of the field required by google api
func (c TemplateAd) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	return ERROR_NOT_YET_IMPLEMENTED
}
<|endoftext|>


<commit_before>package ak

import (
	"os"
	"path"
)

//var DefaultServer = NewServer()

func NewDefaultServer() *Server {
	wd, _ := os.Getwd()
	cfg := &serverConfig{}
	cfg.basePath = wd
	cfg.profiler = true
	cfg.leftDelim = "{{"
	cfg.rightDelim = "}}"
	cfg.defaultStaticDirs = append(cfg.defaultStaticDirs, path.Join(wd, "web"))
	cfg.tplPath = path.Join(wd, "web")
	return &Server{config: cfg, spool: newspool(), filterChain: make([]Filter, 0)}
}

////启动服务
//func Run(s *Server,addr string) {
//	wd, err := os.Getwd()
//	if err != nil {
//		log.Fatal(err)
//	}
//	log.Println(wd)
//	mux := http.NewServeMux()
//	mux.Handle("/", s)
//	err = http.ListenAndServe(addr, mux)
//	if err != nil {
//		log.Fatal(err)
//	}
//}
<commit_msg>添加默认服务<commit_after>package ak

import (
	"os"
	"path"
)

func NewDefaultServer() *Server {
	wd, _ := os.Getwd()
	cfg := &serverConfig{}
	cfg.basePath = wd
	cfg.profiler = true
	cfg.leftDelim = "{{"
	cfg.rightDelim = "}}"
	cfg.defaultStaticDirs = append(cfg.defaultStaticDirs, path.Join(wd, "web"))
	cfg.tplPath = path.Join(wd, "web")
	return &Server{config: cfg, spool: newspool(), filterChain: make([]Filter, 0)}
}

//启动服务
func RunDefaultServer(addr string) {
	wd, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}
	log.Println(wd)
	mux := http.NewServeMux()
	mux.Handle("/", NewDefaultServer())
	err = http.ListenAndServe(addr, mux)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"fmt"
	"log"
	"net/http"
	"strings"
	"time"

	"golang.org/x/crypto/acme/autocert"
)

func getTlsConfig(useAcme bool, cacheDir string, domains []string) *tls.Config {
	config := &tls.Config{
		// Causes servers to use Go's default ciphersuite preferences,
		// which are tuned to avoid attacks. Does nothing on clients.
		PreferServerCipherSuites: true,
		// Only use curves which have assembly implementations
		CurvePreferences: []tls.CurveID{
			tls.CurveP256,
			// tls.X25519, // Go 1.8 only
		},
		MinVersion: tls.VersionTLS12,
		CipherSuites: []uint16{
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			// tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, // Go 1.8 only
			// tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,   // Go 1.8 only
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,

			// Best disabled, as they don't provide Forward Secrecy,
			// but might be necessary for some clients
			// tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
			// tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
		},
	}

	// If desired used ACME to automatically get a certificate.
	if useAcme {
		cache := autocert.DirCache(cacheDir)
		hostPolicy := autocert.HostWhitelist(domains...)

		m := autocert.Manager{
			Prompt:     autocert.AcceptTOS,
			Cache:      cache,
			HostPolicy: hostPolicy,
		}

		config.GetCertificate = m.GetCertificate
	}

	return config
}

func startHttpRedirectServer(fromPort, toPort int) {
	// Redirect http to https.
	redirectSrv := &http.Server{
		Addr:         fmt.Sprintf(":%v", fromPort),
		ReadTimeout:  5 * time.Second,
		WriteTimeout: 5 * time.Second,
		Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			w.Header().Set("Connection", "close")
			host := req.Host
			httpsHost := strings.Split(host, ":")[0] + fmt.Sprintf(":%v", toPort)
			url := "https://" + httpsHost + req.URL.String()
			http.Redirect(w, req, url, http.StatusMovedPermanently)
		}),
	}
	go func() { log.Fatal(redirectSrv.ListenAndServe()) }()
}
<commit_msg>Enable go 1.8 cipher suites<commit_after>package main

import (
	"crypto/tls"
	"fmt"
	"log"
	"net/http"
	"strings"
	"time"

	"golang.org/x/crypto/acme/autocert"
)

func getTlsConfig(useAcme bool, cacheDir string, domains []string) *tls.Config {
	config := &tls.Config{
		// Causes servers to use Go's default ciphersuite preferences,
		// which are tuned to avoid attacks. Does nothing on clients.
		PreferServerCipherSuites: true,
		// Only use curves which have assembly implementations
		CurvePreferences: []tls.CurveID{
			tls.CurveP256,
			tls.X25519,
		},
		MinVersion: tls.VersionTLS12,
		CipherSuites: []uint16{
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,

			// Best disabled, as they don't provide Forward Secrecy,
			// but might be necessary for some clients
			// tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
			// tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
		},
	}

	// If desired used ACME to automatically get a certificate.
	if useAcme {
		cache := autocert.DirCache(cacheDir)
		hostPolicy := autocert.HostWhitelist(domains...)

		m := autocert.Manager{
			Prompt:     autocert.AcceptTOS,
			Cache:      cache,
			HostPolicy: hostPolicy,
		}

		config.GetCertificate = m.GetCertificate
	}

	return config
}

func startHttpRedirectServer(fromPort, toPort int) {
	// Redirect http to https.
	redirectSrv := &http.Server{
		Addr:         fmt.Sprintf(":%v", fromPort),
		ReadTimeout:  5 * time.Second,
		WriteTimeout: 5 * time.Second,
		Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			w.Header().Set("Connection", "close")
			host := req.Host
			httpsHost := strings.Split(host, ":")[0] + fmt.Sprintf(":%v", toPort)
			url := "https://" + httpsHost + req.URL.String()
			http.Redirect(w, req, url, http.StatusMovedPermanently)
		}),
	}
	go func() { log.Fatal(redirectSrv.ListenAndServe()) }()
}
<|endoftext|>


<commit_before>package stemcell

import (
	bosherr "github.com/cloudfoundry/bosh-agent/errors"

	bmconfig "github.com/cloudfoundry/bosh-micro-cli/config"
)

// Repo persists stemcells metadata
type Repo interface {
	Save(stemcell Stemcell, cid CID) error
	Find(stemcell Stemcell) (CID, bool, error)
}

type repo struct {
	configService bmconfig.DeploymentConfigService
}

func NewRepo(configService bmconfig.DeploymentConfigService) repo {
	return repo{
		configService: configService,
	}
}

// Save extracts the stemcell archive,
// parses the stemcell manifest,
// and stores the stemcell archive in the repo.
// The repo stemcell record is indexed by name & sha1 (as specified by the manifest).
func (s repo) Save(stemcell Stemcell, cid CID) error {
	config, err := s.configService.Load()
	if err != nil {
		return bosherr.WrapError(err, "Loading existing config")
	}

	records := config.Stemcells
	if records == nil {
		records = []bmconfig.StemcellRecord{}
	}

	newRecord := bmconfig.StemcellRecord{
		Name:    stemcell.Name,
		Version: stemcell.Version,
		SHA1:    stemcell.SHA1,
	}

	oldRecord, found := s.find(records, newRecord)
	if found {
		return bosherr.New("Failed to save stemcell record `%s', existing record found `%s'", newRecord, oldRecord)
	}

	newRecord.CID = cid.String()
	records = append(records, newRecord)
	config.Stemcells = records

	err = s.configService.Save(config)
	if err != nil {
		//		s.logger.Error("Failed saving updated config: %s", config)
		return bosherr.WrapError(err, "Saving new config")
	}
	return nil
}

func (s repo) Find(stemcell Stemcell) (cid CID, found bool, err error) {
	config, err := s.configService.Load()
	if err != nil {
		return cid, false, bosherr.WrapError(err, "Loading existing config")
	}

	records := config.Stemcells
	if records == nil {
		records = []bmconfig.StemcellRecord{}
	}

	newRecord := bmconfig.StemcellRecord{
		Name:    stemcell.Name,
		Version: stemcell.Version,
		SHA1:    stemcell.SHA1,
	}

	oldRecord, found := s.find(records, newRecord)
	return CID(oldRecord.CID), found, nil
}

func (s repo) find(records []bmconfig.StemcellRecord, record bmconfig.StemcellRecord) (bmconfig.StemcellRecord, bool) {
	for _, existingRecord := range records {
		// "key" excludes CID
		if record.Name == existingRecord.Name &&
			record.Version == existingRecord.Version &&
			record.SHA1 == existingRecord.SHA1 {
			return existingRecord, true
		}
	}
	return bmconfig.StemcellRecord{}, false
}
<commit_msg>s repo -> r repo<commit_after>package stemcell

import (
	bosherr "github.com/cloudfoundry/bosh-agent/errors"

	bmconfig "github.com/cloudfoundry/bosh-micro-cli/config"
)

// Repo persists stemcells metadata
type Repo interface {
	Save(stemcell Stemcell, cid CID) error
	Find(stemcell Stemcell) (CID, bool, error)
}

type repo struct {
	configService bmconfig.DeploymentConfigService
}

func NewRepo(configService bmconfig.DeploymentConfigService) repo {
	return repo{
		configService: configService,
	}
}

// Save extracts the stemcell archive,
// parses the stemcell manifest,
// and stores the stemcell archive in the repo.
// The repo stemcell record is indexed by name & sha1 (as specified by the manifest).
func (r repo) Save(stemcell Stemcell, cid CID) error {
	config, err := r.configService.Load()
	if err != nil {
		return bosherr.WrapError(err, "Loading existing config")
	}

	records := config.Stemcells
	if records == nil {
		records = []bmconfig.StemcellRecord{}
	}

	newRecord := bmconfig.StemcellRecord{
		Name:    stemcell.Name,
		Version: stemcell.Version,
		SHA1:    stemcell.SHA1,
	}

	oldRecord, found := r.find(records, newRecord)
	if found {
		return bosherr.New("Failed to save stemcell record `%s', existing record found `%s'", newRecord, oldRecord)
	}

	newRecord.CID = cid.String()
	records = append(records, newRecord)
	config.Stemcells = records

	err = r.configService.Save(config)
	if err != nil {
		//		r.logger.Error("Failed saving updated config: %s", config)
		return bosherr.WrapError(err, "Saving new config")
	}
	return nil
}

func (r repo) Find(stemcell Stemcell) (cid CID, found bool, err error) {
	config, err := r.configService.Load()
	if err != nil {
		return cid, false, bosherr.WrapError(err, "Loading existing config")
	}

	records := config.Stemcells
	if records == nil {
		records = []bmconfig.StemcellRecord{}
	}

	newRecord := bmconfig.StemcellRecord{
		Name:    stemcell.Name,
		Version: stemcell.Version,
		SHA1:    stemcell.SHA1,
	}

	oldRecord, found := r.find(records, newRecord)
	return CID(oldRecord.CID), found, nil
}

func (r repo) find(records []bmconfig.StemcellRecord, record bmconfig.StemcellRecord) (bmconfig.StemcellRecord, bool) {
	for _, existingRecord := range records {
		// "key" excludes CID
		if record.Name == existingRecord.Name &&
			record.Version == existingRecord.Version &&
			record.SHA1 == existingRecord.SHA1 {
			return existingRecord, true
		}
	}
	return bmconfig.StemcellRecord{}, false
}
<|endoftext|>


<commit_before>package stream

import (
	"bytes"
	"fmt"
	"github.com/andygrunwald/gotrap/config"
	"github.com/andygrunwald/gotrap/gerrit"
	"github.com/andygrunwald/gotrap/github"
	"log"
	"regexp"
	"text/template"
)

type Gotrap struct {
	githubClient github.GithubClient
	gerritClient gerrit.GerritInstance
	config       *config.Configuration
	Message      gerrit.Message
}

func (trap *Gotrap) TakeAction() {
	// Stream events are documented
	// See https://git.eclipse.org/r/Documentation/cmd-stream-events.html
	switch trap.Message.Type {
	case "patchset-created":
		log.Printf("> New patchset-created message incoming for ref \"%s\" in \"%s\" (%s)", trap.Message.Patchset.Ref, trap.Message.Change.Project, trap.Message.Change.URL)

		// Check if Project is configured
		if _, err := trap.IsProjectConfigured(trap.Message.Change.Project); err != nil {
			log.Printf("> %s", err)
			return
		}

		// Check if branch is configured
		if _, err := trap.IsBranchConfigured(trap.Message.Change.Project, trap.Message.Change.Branch); err != nil {
			log.Printf("> %s", err)
			return
		}

		log.Printf("> Getting details of change %s", trap.Message.Change.ID)
		gerritChangeSet, err := trap.gerritClient.GetChangeInformation(trap.Message.Change.ID)
		if err != nil {
			log.Printf("> Error getting details of change %s creating new pull request: %s", trap.Message.Change.ID, err)
			return
		}

		// Check if the status of the changeset is NEW and not
		// SUBMITTED, MERGED, ABANDONED or DRAFT
		// We only accept NEW changesets
		if gerritChangeSet.Status != "NEW" {
			log.Printf("> Changeset skipped, because status is \"%s\" and not \"NEW\"", gerritChangeSet.Status)
			return
		}

		// TODO: Get current revision number
		// If this revision / patchset number is not the current number
		// we will skip this patchset-created request, because
		// why should we create a pull request for an old patchset?
		// The current patchset will be delivered later as message.
		// So we won`t skip this changeset
		// https://review.typo3.org/a/changes/I640486e9f32da6ac1eba05e3c38d15a0aba41055/?o=CURRENT_REVISION
		if currentPatchset, _ := trap.gerritClient.IsPatchsetTheCurrentPatchset(gerritChangeSet, trap.Message.Patchset.Number); currentPatchset == false {
			log.Printf("> Patchset skipped, because it is not the current one (Ref: %s of %s)", trap.Message.Patchset.Ref, trap.Message.Change.URL)
			return
		}

		// Check if change subject is excluded
		if res, matchedPattern := trap.IsSubjectExcludedByPattern(trap.Message.Change.Subject); res == true {
			log.Printf("> Subject \"%s\" excluded by pattern \"%s\"", trap.Message.Change.Subject, matchedPattern)
			return
		}

		// Create the pull request
		pullRequest, err := trap.githubClient.CreatePullRequestForPatchset(&trap.Message)
		if err != nil {
			// TODO: I don`t have an idea what to do if i fail to create a PR
			log.Println("> Error during creating new pull request", err)
			return
		}

		log.Printf("> New pull request created: %s", *pullRequest.HTMLURL)

		// Poll travis ci and wait until the PR got a status
		s, _ := trap.githubClient.WaitUntilCommitStatusIsAvailable(*pullRequest)

		// Build a combined data structure for templating
		gotrapResult := github.PullRequest{
			PullRequest:    pullRequest,
			CombinedStatus: s,
		}

		var vote int
		// TODO Are this all errors? Is there no "error" state?
		switch *s.State {
		// Success if the latest status for all contexts is success
		case "success":
			vote = 0

		// Failure if any of the contexts report as error or failure
		case "failure":
			vote = -1

		}

		// Build message to post results back to Gerrit
		statusDetailsBuffer := new(bytes.Buffer)
		var statusDetailsTemplate = template.Must(template.New("status-details").Parse(trap.gerritClient.Template))
		err = statusDetailsTemplate.Execute(statusDetailsBuffer, gotrapResult)
		if err != nil {
			log.Println("> Error during prepare the status detail message", err)
			return
		}

		// Post Command + Vote on Changeset
		trap.gerritClient.PostCommentOnChangeset(&trap.Message, vote, statusDetailsBuffer.String())

		// TODO: Make text configurable
		msg := "This PR will be closed, because the tests results were reported back to Gerrit. See [{{.message.Change.Subject}}]({{.message.Change.URL}}) for details."

		// Build message to close the Pull Request
		closeMsgBuffer := new(bytes.Buffer)
		var closeMsgTemplate = template.Must(template.New("pull-request-close-message").Parse(msg))
		err = closeMsgTemplate.Execute(closeMsgBuffer, *trap)
		if err != nil {
			log.Println("> Error during prepare the pull request close message", err)
			return
		}

		_, err = trap.githubClient.AddCommentToPullRequest(pullRequest, closeMsgBuffer.String())
		if err != nil {
			log.Printf("> Error during adding a comment to a pull request %s: %s", *pullRequest.HTMLURL, err)
		} else {
			log.Printf("> Comment added to pull request: %s", *pullRequest.HTMLURL)
		}

		_, err = trap.githubClient.ClosePullRequest(pullRequest)
		if err != nil {
			log.Printf("> Error during closing a pull request %s: %s", *pullRequest.HTMLURL, err)
		} else {
			log.Printf("> Pull request closed: %s", *pullRequest.HTMLURL)
		}

	case "change-abandoned":
		// We have to close all PR`s
		// change-restored
		// change-merged
		// ref-updated
		// ref-replicated
		// ref-replication-done
		// comment-added
		// topic-changed
		// ....
	default:
		log.Printf("> Skipped AMQP message (uncovered message type: %s)\n", trap.Message.Type)
	}

	return
}

func (trap *Gotrap) IsProjectConfigured(project string) (bool, error) {
	if _, ok := trap.config.Gerrit.Projects[project]; !ok {
		return false, fmt.Errorf("Project \"%s\" is not configured", project)
	}

	return true, nil
}

func (trap *Gotrap) IsBranchConfigured(project, branch string) (bool, error) {
	// If no branch is configured, we assume that all branches should be covered
	if len(trap.config.Gerrit.Projects[project]) == 0 {
		return true, nil
	}

	// If the branch exists and is configured as true, then it is valid configured
	if val, ok := trap.config.Gerrit.Projects[project][branch]; ok && val == true {
		return true, nil
	}

	return false, fmt.Errorf("Branch \"%s\" not configured for project \"%s\"", branch, project)
}

func (trap *Gotrap) IsSubjectExcludedByPattern(subject string) (bool, string) {
	for _, pattern := range trap.config.Gerrit.ExcludePattern {
		if matched, err := regexp.MatchString(pattern, subject); err == nil && matched == true {
			return matched, pattern
		}
	}

	return false, ""
}
<commit_msg>Fixed "Error during prepare the pull request close message template: pull-request-close-message:1:95: executing "pull-request-close-message" at <.message.Change.Subj...>: message is not a field of struct type stream.Gotrap"<commit_after>package stream

import (
	"bytes"
	"fmt"
	"github.com/andygrunwald/gotrap/config"
	"github.com/andygrunwald/gotrap/gerrit"
	"github.com/andygrunwald/gotrap/github"
	"log"
	"regexp"
	"text/template"
)

type Gotrap struct {
	githubClient github.GithubClient
	gerritClient gerrit.GerritInstance
	config       *config.Configuration
	Message      gerrit.Message
}

func (trap *Gotrap) TakeAction() {
	// Stream events are documented
	// See https://git.eclipse.org/r/Documentation/cmd-stream-events.html
	switch trap.Message.Type {
	case "patchset-created":
		log.Printf("> New patchset-created message incoming for ref \"%s\" in \"%s\" (%s)", trap.Message.Patchset.Ref, trap.Message.Change.Project, trap.Message.Change.URL)

		// Check if Project is configured
		if _, err := trap.IsProjectConfigured(trap.Message.Change.Project); err != nil {
			log.Printf("> %s", err)
			return
		}

		// Check if branch is configured
		if _, err := trap.IsBranchConfigured(trap.Message.Change.Project, trap.Message.Change.Branch); err != nil {
			log.Printf("> %s", err)
			return
		}

		log.Printf("> Getting details of change %s", trap.Message.Change.ID)
		gerritChangeSet, err := trap.gerritClient.GetChangeInformation(trap.Message.Change.ID)
		if err != nil {
			log.Printf("> Error getting details of change %s creating new pull request: %s", trap.Message.Change.ID, err)
			return
		}

		// Check if the status of the changeset is NEW and not
		// SUBMITTED, MERGED, ABANDONED or DRAFT
		// We only accept NEW changesets
		if gerritChangeSet.Status != "NEW" {
			log.Printf("> Changeset skipped, because status is \"%s\" and not \"NEW\"", gerritChangeSet.Status)
			return
		}

		// TODO: Get current revision number
		// If this revision / patchset number is not the current number
		// we will skip this patchset-created request, because
		// why should we create a pull request for an old patchset?
		// The current patchset will be delivered later as message.
		// So we won`t skip this changeset
		// https://review.typo3.org/a/changes/I640486e9f32da6ac1eba05e3c38d15a0aba41055/?o=CURRENT_REVISION
		if currentPatchset, _ := trap.gerritClient.IsPatchsetTheCurrentPatchset(gerritChangeSet, trap.Message.Patchset.Number); currentPatchset == false {
			log.Printf("> Patchset skipped, because it is not the current one (Ref: %s of %s)", trap.Message.Patchset.Ref, trap.Message.Change.URL)
			return
		}

		// Check if change subject is excluded
		if res, matchedPattern := trap.IsSubjectExcludedByPattern(trap.Message.Change.Subject); res == true {
			log.Printf("> Subject \"%s\" excluded by pattern \"%s\"", trap.Message.Change.Subject, matchedPattern)
			return
		}

		// Create the pull request
		pullRequest, err := trap.githubClient.CreatePullRequestForPatchset(&trap.Message)
		if err != nil {
			// TODO: I don`t have an idea what to do if i fail to create a PR
			log.Println("> Error during creating new pull request", err)
			return
		}

		log.Printf("> New pull request created: %s", *pullRequest.HTMLURL)

		// Poll travis ci and wait until the PR got a status
		s, _ := trap.githubClient.WaitUntilCommitStatusIsAvailable(*pullRequest)

		// Build a combined data structure for templating
		gotrapResult := github.PullRequest{
			PullRequest:    pullRequest,
			CombinedStatus: s,
		}

		var vote int
		// TODO Are this all errors? Is there no "error" state?
		switch *s.State {
		// Success if the latest status for all contexts is success
		case "success":
			vote = 0

		// Failure if any of the contexts report as error or failure
		case "failure":
			vote = -1

		}

		// Build message to post results back to Gerrit
		statusDetailsBuffer := new(bytes.Buffer)
		var statusDetailsTemplate = template.Must(template.New("status-details").Parse(trap.gerritClient.Template))
		err = statusDetailsTemplate.Execute(statusDetailsBuffer, gotrapResult)
		if err != nil {
			log.Println("> Error during prepare the status detail message", err)
			return
		}

		// Post Command + Vote on Changeset
		trap.gerritClient.PostCommentOnChangeset(&trap.Message, vote, statusDetailsBuffer.String())

		// TODO: Make text configurable
		msg := "This PR will be closed, because the tests results were reported back to Gerrit. See [{{.Message.Change.Subject}}]({{.Message.Change.URL}}) for details."

		// Build message to close the Pull Request
		closeMsgBuffer := new(bytes.Buffer)
		var closeMsgTemplate = template.Must(template.New("pull-request-close-message").Parse(msg))
		err = closeMsgTemplate.Execute(closeMsgBuffer, *trap)
		if err != nil {
			log.Println("> Error during prepare the pull request close message", err)
			return
		}

		_, err = trap.githubClient.AddCommentToPullRequest(pullRequest, closeMsgBuffer.String())
		if err != nil {
			log.Printf("> Error during adding a comment to a pull request %s: %s", *pullRequest.HTMLURL, err)
		} else {
			log.Printf("> Comment added to pull request: %s", *pullRequest.HTMLURL)
		}

		_, err = trap.githubClient.ClosePullRequest(pullRequest)
		if err != nil {
			log.Printf("> Error during closing a pull request %s: %s", *pullRequest.HTMLURL, err)
		} else {
			log.Printf("> Pull request closed: %s", *pullRequest.HTMLURL)
		}

	case "change-abandoned":
		// We have to close all PR`s
		// change-restored
		// change-merged
		// ref-updated
		// ref-replicated
		// ref-replication-done
		// comment-added
		// topic-changed
		// ....
	default:
		log.Printf("> Skipped AMQP message (uncovered message type: %s)\n", trap.Message.Type)
	}

	return
}

func (trap *Gotrap) IsProjectConfigured(project string) (bool, error) {
	if _, ok := trap.config.Gerrit.Projects[project]; !ok {
		return false, fmt.Errorf("Project \"%s\" is not configured", project)
	}

	return true, nil
}

func (trap *Gotrap) IsBranchConfigured(project, branch string) (bool, error) {
	// If no branch is configured, we assume that all branches should be covered
	if len(trap.config.Gerrit.Projects[project]) == 0 {
		return true, nil
	}

	// If the branch exists and is configured as true, then it is valid configured
	if val, ok := trap.config.Gerrit.Projects[project][branch]; ok && val == true {
		return true, nil
	}

	return false, fmt.Errorf("Branch \"%s\" not configured for project \"%s\"", branch, project)
}

func (trap *Gotrap) IsSubjectExcludedByPattern(subject string) (bool, string) {
	for _, pattern := range trap.config.Gerrit.ExcludePattern {
		if matched, err := regexp.MatchString(pattern, subject); err == nil && matched == true {
			return matched, pattern
		}
	}

	return false, ""
}
<|endoftext|>


<commit_before>package client

import (
	"archive/zip"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"time"

	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/armor"
	"golang.org/x/crypto/openpgp/packet"
)

// LoadKey loads the key associated with this username,
// first by loooking at ~/.sendto/users/recipient/key.pub
// or if that fails by fetching it from the internet and saving at that location
// it returns the path of the downloaded key file
func LoadKey(recipient string, url string) (string, error) {
	fmt.Printf("Loading key for %s...\n", recipient)

	// Define a local keypath of users/name/key.pub
	keyPath := filepath.Join(configPath(), "users", recipient, "key.pub")

	// Check if the key file exists at ~/.sendto/users/recipient/key.pub
	if !fileExists(keyPath) {
		// Make the user directory
		createFolder(filepath.Join("users", recipient))

		// Fetch the key from our server
		err := DownloadData(url, keyPath)
		if err != nil {
			return "", err
		}

		// Tell user we fetched key
		fmt.Printf("Fetched key for user:%s from:%s\n", recipient, url)

		// Print the key for the user as we have fetched it for the first time?
		/*
		   key, err := ioutil.ReadFile(keyPath)
		   if err != nil {
		     return "", err
		   }
		*/
	}

	return keyPath, nil
}

// EncryptFiles zips then encrypts our arguments (files or folders) using a public key
func EncryptFiles(args []string, recipient string, keyPath string) (string, error) {

	// First open and parse recipient key
	publicKey, err := ParsePublicKey(keyPath)
	if err != nil {
		return "", err
	}

	fmt.Printf("Using key: %x\n", publicKey.PrimaryKey.Fingerprint)

	// Make the user files directory
	createFolder(filepath.Join("files", recipient))

	// Should caller set the filename required for the zip?
	// hash of username + time or something? Doesn't really matter but should be unique I guess
	// it doesn't have to be unique though... just use caller name for now
	name := path.Clean(path.Base(args[0]))
	outPath := filepath.Join(configPath(), "files", recipient, fmt.Sprintf("%s.zip.gpg", name))
	out, err := os.Create(outPath)
	if err != nil {
		return "", err
	}
	defer out.Close()

	// Create encryption writer
	hints := &openpgp.FileHints{IsBinary: true, FileName: fmt.Sprintf("%s.zip", name), ModTime: time.Now()}
	pgpWriter, err := openpgp.Encrypt(out, []*openpgp.Entity{publicKey}, nil, hints, nil)
	if err != nil {
		return "", err
	}

	// Now create a zipwriter, which writes to this pgpWriter
	zipWriter := zip.NewWriter(pgpWriter)

	// Add the files/folders from our args
	for _, arg := range args {

		// For each argument, walk the file path adding files to our zip
		err := filepath.Walk(arg, func(p string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if info.IsDir() {
				return nil
			}

			f, err := os.Open(p)
			if err != nil {
				return err
			}
			defer f.Close()

			// Support unicode filenames by default
			h := &zip.FileHeader{Name: p, Method: zip.Deflate, Flags: 0x800}
			z, err := zipWriter.CreateHeader(h)
			// Doesn't support unicode file names?
			// z, err := zipWriter.Create(p)
			if err != nil {
				return err
			}
			io.Copy(z, f)
			zipWriter.Flush()
			return nil
		})
		if err != nil {
			return "", err
		}

	}
	err = zipWriter.Flush()
	if err != nil {
		return "", err
	}
	err = zipWriter.Close()
	if err != nil {
		return "", err
	}

	// close the encPipe to finish the process
	err = pgpWriter.Close()

	return outPath, err
}

// ParsePublicKey parses the given public key file
func ParsePublicKey(keyPath string) (*openpgp.Entity, error) {
	f, err := os.Open(keyPath)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	// Parse our key
	key, err := armor.Decode(f)
	if err != nil {
		return nil, err
	}
	if key.Type != openpgp.PublicKeyType {
		return nil, fmt.Errorf("Key of wrong type:%s", key.Type)
	}
	r := packet.NewReader(key.Body)
	to, err := openpgp.ReadEntity(r)
	if err != nil {
		return nil, err
	}

	return to, nil
}

// DecryptFiles decrypts then unzips a file using a private key
// and returns the path of the resulting file/folder on success
// zip step should be optional TODO
func DecryptFiles(p string, key string) (string, error) {

	return "", nil
}
<commit_msg>Added fix from mattn for windows paths (untested as yet)<commit_after>package client

import (
	"archive/zip"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"time"

	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/armor"
	"golang.org/x/crypto/openpgp/packet"
)

// LoadKey loads the key associated with this username,
// first by loooking at ~/.sendto/users/recipient/key.pub
// or if that fails by fetching it from the internet and saving at that location
// it returns the path of the downloaded key file
func LoadKey(recipient string, url string) (string, error) {
	fmt.Printf("Loading key for %s...\n", recipient)

	// Define a local keypath of users/name/key.pub
	keyPath := filepath.Join(configPath(), "users", recipient, "key.pub")

	// Check if the key file exists at ~/.sendto/users/recipient/key.pub
	if !fileExists(keyPath) {
		// Make the user directory
		createFolder(filepath.Join("users", recipient))

		// Fetch the key from our server
		err := DownloadData(url, keyPath)
		if err != nil {
			return "", err
		}

		// Tell user we fetched key
		fmt.Printf("Fetched key for user:%s from:%s\n", recipient, url)

		// Print the key for the user as we have fetched it for the first time?
		/*
		   key, err := ioutil.ReadFile(keyPath)
		   if err != nil {
		     return "", err
		   }
		*/
	}

	return keyPath, nil
}

// EncryptFiles zips then encrypts our arguments (files or folders) using a public key
func EncryptFiles(args []string, recipient string, keyPath string) (string, error) {

	// First open and parse recipient key
	publicKey, err := ParsePublicKey(keyPath)
	if err != nil {
		return "", err
	}

	fmt.Printf("Using key: %x\n", publicKey.PrimaryKey.Fingerprint)

	// Make the user files directory
	createFolder(filepath.Join("files", recipient))

	// At present the caller sets the filename - perhaps reconsider as this leaks info - FIXME
	name := path.Clean(path.Base(args[0]))

	// Create a file at config/files/recipient/name.zip.gpg
	outPath := filepath.Join(configPath(), "files", recipient, fmt.Sprintf("%s.zip.gpg", name))

	// Create the file
	out, err := os.Create(outPath)
	if err != nil {
		return "", err
	}
	defer out.Close()

	// Create encryption writer
	hints := &openpgp.FileHints{IsBinary: true, FileName: fmt.Sprintf("%s.zip", name), ModTime: time.Now()}
	pgpWriter, err := openpgp.Encrypt(out, []*openpgp.Entity{publicKey}, nil, hints, nil)
	if err != nil {
		return "", err
	}

	// Now create a zipwriter, which writes to this pgpWriter
	zipWriter := zip.NewWriter(pgpWriter)

	// Add the files/folders from our args
	for _, arg := range args {

		// For each argument, walk the file path adding files to our zip
		err := filepath.Walk(arg, func(p string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if info.IsDir() {
				return nil
			}

			f, err := os.Open(p)
			if err != nil {
				return err
			}
			defer f.Close()

			// We need to add the filename *removing* the current path so that we don't store
			// an entire path hierarchy to the folder/file FIXME

			// Support unicode filenames by default
			h := &zip.FileHeader{Name: p, Method: zip.Deflate, Flags: 0x800}
			z, err := zipWriter.CreateHeader(h)
			// Doesn't support unicode file names?
			// z, err := zipWriter.Create(p)
			if err != nil {
				return err
			}
			io.Copy(z, f)
			zipWriter.Flush()
			return nil
		})
		if err != nil {
			return "", err
		}

	}
	err = zipWriter.Flush()
	if err != nil {
		return "", err
	}
	err = zipWriter.Close()
	if err != nil {
		return "", err
	}

	// close the encPipe to finish the process
	err = pgpWriter.Close()

	// Make sure the file path returned uses forward slashes on windows
	// - change from mattn moved here
	outPath = filepath.ToSlash(outPath)

	return outPath, err
}

// ParsePublicKey parses the given public key file
func ParsePublicKey(keyPath string) (*openpgp.Entity, error) {
	f, err := os.Open(keyPath)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	// Parse our key
	key, err := armor.Decode(f)
	if err != nil {
		return nil, err
	}
	if key.Type != openpgp.PublicKeyType {
		return nil, fmt.Errorf("Key of wrong type:%s", key.Type)
	}
	r := packet.NewReader(key.Body)
	to, err := openpgp.ReadEntity(r)
	if err != nil {
		return nil, err
	}

	return to, nil
}

// DecryptFiles decrypts then unzips a file using a private key
// and returns the path of the resulting file/folder on success
// zip step should be optional TODO
func DecryptFiles(p string, key string) (string, error) {

	return "", nil
}
<|endoftext|>


<commit_before><commit_msg>replace deprecated ioutil package with io package<commit_after><|endoftext|>


<commit_before>package structure

// Set is a map with string key and bool value
// It is not thread safe and modeled after https://github.com/deckarep/golang-set/blob/master/threadunsafe.go
type Set map[string]bool

// NewSet return a pointer of a set using arguments passed to the function
func NewSet(args ...string) *Set {
	// TODO: would this length for the map?
	m := make(Set, len(args))
	for _, v := range args {
		m[v] = true
	}
	return &m
}

// Contains check if a key is presented in the map, it does NOT check the bool value
func (set *Set) Contains(key string) bool {
	_, ok := (*set)[key]
	return ok
}
<commit_msg>[common][structure] Add `Add` method to Set<commit_after>package structure

// Set is a map with string key and bool value
// It is not thread safe and modeled after https://github.com/deckarep/golang-set/blob/master/threadunsafe.go
type Set map[string]bool

// NewSet return a pointer of a set using arguments passed to the function
func NewSet(args ...string) *Set {
	// TODO: would this length for the map?
	m := make(Set, len(args))
	for _, v := range args {
		m[v] = true
	}
	return &m
}

// Contains check if a key is presented in the map, it does NOT check the bool value
func (set *Set) Contains(key string) bool {
	_, ok := (*set)[key]
	return ok
}

func (set *Set) Add(key string) {
	(*set)[key] = true
}
<|endoftext|>


<commit_before>package clock

import (
	"github.com/pachyderm/pachyderm/src/server/pfs/persist"
)

// NewBranchClocks creates a new BranchClocks given a branch name
// "master" -> [[(master, 0)]]
func NewBranchClocks(branch string) persist.BranchClocks {
	return nil
}

// NewChild returns the child of a BranchClock
// [(master, 0), (foo, 0)] -> [(master, 0), (foo, 1)]
func NewChild(parent *persist.BranchClock) *persist.BranchClock {
	return nil
}

// NewBranch takes a BranchClock and a branch name, and returns a new BranchClock
// that contains the new Branch
// Returns an error if the branch already exists on the BranchClock
// [(master, 0), (foo, 1)], "bar" -> [(master, 0), (foo, 1), (bar, 0)]
func NewBranch(parent *persist.BranchClock, branch string) (*persist.BranchClock, error) {
	return nil, nil
}

// NewChildOfBranchClocks takes a BranchClocks and a branch name, and returns a
// BranchClocks that contains one BranchClock which is the child of the BranchClock
// specified by the given branch name.
//
// Returns an error if the branch is not found
//
// Args: [[(foo, 1), (bar, 1)], [(master, 1)]], "master"
// Return: [[(master, 2)]], true
func NewChildOfBranchClocks(parent persist.BranchClocks, branch string) (persist.BranchClocks, error) {
	return nil, nil
}

// AddClock adds a BranchClock to a BranchClocks.
// Returns an error if the BranchClock already exists in the BranchClocks
func AddClock(b persist.BranchClocks, c *persist.BranchClock) (persist.BranchClocks, error) {
	return nil, nil
}
<commit_msg>Fix merge conflict<commit_after>package clock

import (
	"fmt"

	"github.com/pachyderm/pachyderm/src/server/pfs/persist"
)

type ErrBranchExists struct {
	error
}

type ErrBranchNotFound struct {
	error
}

// NewBranchClocks creates a new BranchClocks given a branch name
// "master" -> [[(master, 0)]]
func NewBranchClocks(branch string) persist.BranchClocks {
	return persist.BranchClocks{{
		Clocks: []*persist.Clock{{
			Branch: branch,
			Clock:  0,
		}},
	}}
}

// NewChild returns the child of a BranchClock
// [(master, 0), (foo, 0)] -> [(master, 0), (foo, 1)]
func NewChild(parent *persist.BranchClock) *persist.BranchClock {
	if len(parent.Clocks) == 0 {
		return parent
	} else {
		parent.Clocks[len(parent.Clocks)-1].Clock += 1
		return parent
	}
}

// NewBranch takes a BranchClock and a branch name, and returns a new BranchClock
// that contains the new Branch
// Returns an error if the branch already exists on the BranchClock
// [(master, 0), (foo, 1)], "bar" -> [(master, 0), (foo, 1), (bar, 0)]
func NewBranch(parent *persist.BranchClock, branch string) (*persist.BranchClock, error) {
	for _, clock := range parent.Clocks {
		if clock.Branch == branch {
			return nil, ErrBranchExists{
				error: fmt.Errorf("branch %s already exists in the branch clock", branch),
			}
		}
	}
	parent.Clocks = append(parent.Clocks, &persist.Clock{
		Branch: branch,
		Clock:  0,
	})
	return parent, nil
}

// NewChildOfBranchClocks takes a BranchClocks and a branch name, and returns a
// BranchClocks that contains one BranchClock which is the child of the BranchClock
// specified by the given branch name.
//
// Returns an error if the branch is not found
//
// Args: [[(foo, 1), (bar, 1)], [(master, 1)]], "master"
// Return: [[(master, 2)]], true
func NewChildOfBranchClocks(parent persist.BranchClocks, branch string) (persist.BranchClocks, error) {
	for _, branchClock := range parent {
		if len(branchClock.Clocks) > 0 && branchClock.Clocks[len(branchClock.Clocks)-1].Branch == branch {
			return persist.BranchClocks{NewChild(branchClock)}, nil
		}
	}
	return nil, ErrBranchNotFound{fmt.Errorf("branch %s not found in branch clocks")}
}

// AddClock adds a BranchClock to a BranchClocks.
// Returns an error if the branch already exists in the BranchClocks
//
// Args: [[(foo, 1), (bar, 1)], [(master, 1)]], [(buzz, 2)]
// Return: [[(foo, 1), (bar, 1)], [(master, 1)], [(buzz, 2)]]
func AddClock(b persist.BranchClocks, c *persist.BranchClock) (persist.BranchClocks, error) {
	if c == nil || len(c.Clocks) == 0 {
		return b, nil
	}

	branch := c.Clocks[len(c.Clocks)-1].Branch

	for _, bc := range b {
		if len(bc.Clocks) > 0 && bc.Clocks[len(bc.Clocks)-1].Branch == branch {
			return nil, ErrBranchExists{fmt.Errorf("branch %s already exists in the branch clock", branch)}
		}
	}

	return append(b, c), nil
}
<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"database/sql"
	"database/sql/driver"
	"fmt"
	"hash/crc32"
	"strings"
	"time"

	pbinlog "github.com/cwen0/cdb-syncer/protocol"
	"github.com/go-sql-driver/mysql"
	"github.com/juju/errors"
	"github.com/ngaut/log"
	tddl "github.com/pingcap/tidb/ddl"
	"github.com/pingcap/tidb/infoschema"
	tmysql "github.com/pingcap/tidb/mysql"
	"github.com/pingcap/tidb/terror"
)

type job struct {
	tp    pbinlog.BinlogType
	sql   string
	args  []interface{}
	key   string
	retry bool
	pos   Position
}

func newJob(tp pbinlog.BinlogType, sql string, args []interface{}, key string, retry bool, pos Position) *job {
	return &job{tp: tp, sql: sql, args: args, key: key, retry: retry, pos: pos}
}

func genHashKey(key string) uint32 {
	return crc32.ChecksumIEEE([]byte(key))
}

func genPKey(rows []*pbinlog.Row) string {
	var values []string
	for _, row := range rows {
		values = append(values, row.GetColumnValue())
	}
	return strings.Join(values, ",")
}

func genInsertSQL(binlog pbinlog.Binlog) (string, string, []interface{}, error) {
	var sql string
	var values []string
	sql += "replace into " + binlog.GetDbName() + "." + binlog.GetTableName() + "("
	rows := binlog.GetRows()
	for _, row := range rows {
		sql += row.GetColumnName() + ","
		values = append(values, row.GetColumnValue())
	}
	sql = sql[0:len(sql)-1] + ") values ("
	for _, _ = range rows {
		sql += "?,"
	}
	sql = sql[0:len(sql)-1] + ")"

	return sql, binlog.GetTableName() + genPKey(binlog.GetPrimaryKey()), stringToInterface(values), nil
}

func genUpdateSQL(binlog pbinlog.Binlog) (string, string, []interface{}, error) {
	var sql string
	var values []string
	sql += "update " + binlog.GetDbName() + "." + binlog.GetTableName() + " set "
	rows := binlog.GetRows()
	for _, row := range rows {
		sql += row.GetColumnName() + "=?,"
		values = append(values, row.GetColumnValue())
	}
	sql = sql[0:len(sql)-1] + " where 1=1 "
	for _, row := range binlog.GetPrimaryKey() {
		sql += " and " + row.GetColumnName() + " = ? "
		values = append(values, row.GetColumnValue())
	}
	return sql, binlog.GetTableName() + genPKey(binlog.GetPrimaryKey()), stringToInterface(values), nil
}

func genDeleteSQL(binlog pbinlog.Binlog) (string, string, []interface{}, error) {
	var sql string
	var values []string
	sql += "delete from " + binlog.GetDbName() + "." + binlog.GetTableName() + " where 1=1 "
	for _, row := range binlog.GetPrimaryKey() {
		sql += " and " + row.GetColumnName() + " = ? "
		values = append(values, row.GetColumnValue())
	}
	return sql, binlog.GetTableName() + genPKey(binlog.GetPrimaryKey()), stringToInterface(values), nil
}

func genDdlSQL(binlog pbinlog.Binlog) (string, string, []interface{}, error) {
	var sql string
	sql += "use " + binlog.GetDbName() + ";"
	rows := binlog.GetRows()
	for _, row := range rows {
		sql += row.GetSql() + ";"
	}
	empty := make([]interface{}, 0)
	return sql, "", empty, nil
}

func ignoreDDLError(err error) bool {
	mysqlErr, ok := errors.Cause(err).(*mysql.MySQLError)
	if !ok {
		return false
	}

	errCode := terror.ErrCode(mysqlErr.Number)
	switch errCode {
	case infoschema.ErrDatabaseExists.Code(), infoschema.ErrDatabaseNotExists.Code(), infoschema.ErrDatabaseDropExists.Code(),
		infoschema.ErrTableExists.Code(), infoschema.ErrTableNotExists.Code(), infoschema.ErrTableDropExists.Code(),
		infoschema.ErrColumnExists.Code(), infoschema.ErrColumnNotExists.Code(),
		infoschema.ErrIndexExists.Code(), tddl.ErrCantDropFieldOrKey.Code():
		return true
	default:
		return false
	}
}

func isRetryableError(err error) bool {
	if err == driver.ErrBadConn {
		return true
	}
	var e error
	for {
		e = errors.Cause(err)
		if err == e {
			break
		}
		err = e
	}
	mysqlErr, ok := err.(*mysql.MySQLError)
	if ok {
		if mysqlErr.Number == tmysql.ErrUnknown {
			return true
		}
		return false
	}

	return true
}

func querySQL(db *sql.DB, query string) (*sql.Rows, error) {
	var (
		err  error
		rows *sql.Rows
	)

	for i := 0; i < maxRetryCount; i++ {
		if i > 0 {
			log.Warnf("query sql retry %d - %s", i, query)
			time.Sleep(retryTimeout)
		}

		log.Debugf("[query][sql]%s", query)

		rows, err = db.Query(query)
		if err != nil {
			if !isRetryableError(err) {
				return rows, errors.Trace(err)
			}
			log.Warnf("[query][sql]%s[error]%v", query, err)
			continue
		}

		return rows, nil
	}

	if err != nil {
		log.Errorf("query sql[%s] failed %v", query, errors.ErrorStack(err))
		return nil, errors.Trace(err)
	}

	return nil, errors.Errorf("query sql[%s] failed", query)
}

func executeSQL(db *sql.DB, sqls []string, args [][]interface{}, retry bool) error {
	if len(sqls) == 0 {
		return nil
	}

	var (
		err error
		txn *sql.Tx
	)

	retryCount := 1
	if retry {
		retryCount = maxRetryCount
	}

LOOP:
	for i := 0; i < retryCount; i++ {
		if i > 0 {
			log.Warnf("exec sql retry %d - %v - %v", i, sqls, args)
			time.Sleep(retryTimeout)
		}

		txn, err = db.Begin()
		if err != nil {
			log.Errorf("exec sqls[%v] begin failed %v", sqls, errors.ErrorStack(err))
			continue
		}

		for i := range sqls {
			log.Debugf("[exec][sql]%s[args]%v", sqls[i], args[i])

			_, err = txn.Exec(sqls[i], args[i]...)
			if err != nil {
				if !isRetryableError(err) {
					rerr := txn.Rollback()
					if rerr != nil {
						log.Errorf("[exec][sql]%s[args]%v[error]%v", sqls[i], args[i], rerr)
					}
					break LOOP
				}

				log.Warnf("[exec][sql]%s[args]%v[error]%v", sqls[i], args[i], err)
				rerr := txn.Rollback()
				if rerr != nil {
					log.Errorf("[exec][sql]%s[args]%v[error]%v", sqls[i], args[i], rerr)
				}
				continue LOOP
			}
		}
		err = txn.Commit()
		if err != nil {
			log.Errorf("exec sqls[%v] commit failed %v", sqls, errors.ErrorStack(err))
			continue
		}

		return nil
	}

	if err != nil {
		log.Errorf("exec sqls[%v] failed %v", sqls, errors.ErrorStack(err))
		return errors.Trace(err)
	}

	return errors.Errorf("exec sqls[%v] failed", sqls)
}

func createDB(cfg DBConfig) (*sql.DB, error) {
	dbDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/?charset=utf8&interpolateParams=true", cfg.User, cfg.Password, cfg.Host, cfg.Port)
	db, err := sql.Open("mysql", dbDSN)
	if err != nil {
		return nil, errors.Trace(err)
	}

	return db, nil
}

func closeDB(db *sql.DB) error {
	if db == nil {
		return nil
	}

	return errors.Trace(db.Close())
}

func createDBs(cfg DBConfig, count int) ([]*sql.DB, error) {
	dbs := make([]*sql.DB, 0, count)
	for i := 0; i < count; i++ {
		db, err := createDB(cfg)
		if err != nil {
			return nil, errors.Trace(err)
		}

		dbs = append(dbs, db)
	}

	return dbs, nil
}

func closeDBs(dbs ...*sql.DB) {
	for _, db := range dbs {
		err := closeDB(db)
		if err != nil {
			log.Errorf("close db failed - %v", err)
		}
	}
}
<commit_msg>fix gen ddl<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"database/sql"
	"database/sql/driver"
	"fmt"
	"hash/crc32"
	"strings"
	"time"

	pbinlog "github.com/cwen0/cdb-syncer/protocol"
	"github.com/go-sql-driver/mysql"
	"github.com/juju/errors"
	"github.com/ngaut/log"
	tddl "github.com/pingcap/tidb/ddl"
	"github.com/pingcap/tidb/infoschema"
	tmysql "github.com/pingcap/tidb/mysql"
	"github.com/pingcap/tidb/terror"
)

type job struct {
	tp    pbinlog.BinlogType
	sql   string
	args  []interface{}
	key   string
	retry bool
	pos   Position
}

func newJob(tp pbinlog.BinlogType, sql string, args []interface{}, key string, retry bool, pos Position) *job {
	return &job{tp: tp, sql: sql, args: args, key: key, retry: retry, pos: pos}
}

func genHashKey(key string) uint32 {
	return crc32.ChecksumIEEE([]byte(key))
}

func genPKey(rows []*pbinlog.Row) string {
	var values []string
	for _, row := range rows {
		values = append(values, row.GetColumnValue())
	}
	return strings.Join(values, ",")
}

func genInsertSQL(binlog pbinlog.Binlog) (string, string, []interface{}, error) {
	var sql string
	var values []string
	sql += "replace into " + binlog.GetDbName() + "." + binlog.GetTableName() + "("
	rows := binlog.GetRows()
	for _, row := range rows {
		sql += row.GetColumnName() + ","
		values = append(values, row.GetColumnValue())
	}
	sql = sql[0:len(sql)-1] + ") values ("
	for _, _ = range rows {
		sql += "?,"
	}
	sql = sql[0:len(sql)-1] + ")"

	return sql, binlog.GetTableName() + genPKey(binlog.GetPrimaryKey()), stringToInterface(values), nil
}

func genUpdateSQL(binlog pbinlog.Binlog) (string, string, []interface{}, error) {
	var sql string
	var values []string
	sql += "update " + binlog.GetDbName() + "." + binlog.GetTableName() + " set "
	rows := binlog.GetRows()
	for _, row := range rows {
		sql += row.GetColumnName() + "=?,"
		values = append(values, row.GetColumnValue())
	}
	sql = sql[0:len(sql)-1] + " where 1=1 "
	for _, row := range binlog.GetPrimaryKey() {
		sql += " and " + row.GetColumnName() + " = ? "
		values = append(values, row.GetColumnValue())
	}
	return sql, binlog.GetTableName() + genPKey(binlog.GetPrimaryKey()), stringToInterface(values), nil
}

func genDeleteSQL(binlog pbinlog.Binlog) (string, string, []interface{}, error) {
	var sql string
	var values []string
	sql += "delete from " + binlog.GetDbName() + "." + binlog.GetTableName() + " where 1=1 "
	for _, row := range binlog.GetPrimaryKey() {
		sql += " and " + row.GetColumnName() + " = ? "
		values = append(values, row.GetColumnValue())
	}
	return sql, binlog.GetTableName() + genPKey(binlog.GetPrimaryKey()), stringToInterface(values), nil
}

func genDdlSQL(binlog pbinlog.Binlog) (string, string, []interface{}, error) {
	var sql string
	if binlog.GetDbName() != "" {
		sql += "use " + binlog.GetDbName() + ";"
	}
	rows := binlog.GetRows()
	for _, row := range rows {
		sql += row.GetSql() + ";"
	}
	empty := make([]interface{}, 0)
	return sql, "", empty, nil
}

func ignoreDDLError(err error) bool {
	mysqlErr, ok := errors.Cause(err).(*mysql.MySQLError)
	if !ok {
		return false
	}

	errCode := terror.ErrCode(mysqlErr.Number)
	switch errCode {
	case infoschema.ErrDatabaseExists.Code(), infoschema.ErrDatabaseNotExists.Code(), infoschema.ErrDatabaseDropExists.Code(),
		infoschema.ErrTableExists.Code(), infoschema.ErrTableNotExists.Code(), infoschema.ErrTableDropExists.Code(),
		infoschema.ErrColumnExists.Code(), infoschema.ErrColumnNotExists.Code(),
		infoschema.ErrIndexExists.Code(), tddl.ErrCantDropFieldOrKey.Code():
		return true
	default:
		return false
	}
}

func isRetryableError(err error) bool {
	if err == driver.ErrBadConn {
		return true
	}
	var e error
	for {
		e = errors.Cause(err)
		if err == e {
			break
		}
		err = e
	}
	mysqlErr, ok := err.(*mysql.MySQLError)
	if ok {
		if mysqlErr.Number == tmysql.ErrUnknown {
			return true
		}
		return false
	}

	return true
}

func querySQL(db *sql.DB, query string) (*sql.Rows, error) {
	var (
		err  error
		rows *sql.Rows
	)

	for i := 0; i < maxRetryCount; i++ {
		if i > 0 {
			log.Warnf("query sql retry %d - %s", i, query)
			time.Sleep(retryTimeout)
		}

		log.Debugf("[query][sql]%s", query)

		rows, err = db.Query(query)
		if err != nil {
			if !isRetryableError(err) {
				return rows, errors.Trace(err)
			}
			log.Warnf("[query][sql]%s[error]%v", query, err)
			continue
		}

		return rows, nil
	}

	if err != nil {
		log.Errorf("query sql[%s] failed %v", query, errors.ErrorStack(err))
		return nil, errors.Trace(err)
	}

	return nil, errors.Errorf("query sql[%s] failed", query)
}

func executeSQL(db *sql.DB, sqls []string, args [][]interface{}, retry bool) error {
	if len(sqls) == 0 {
		return nil
	}

	var (
		err error
		txn *sql.Tx
	)

	retryCount := 1
	if retry {
		retryCount = maxRetryCount
	}

LOOP:
	for i := 0; i < retryCount; i++ {
		if i > 0 {
			log.Warnf("exec sql retry %d - %v - %v", i, sqls, args)
			time.Sleep(retryTimeout)
		}

		txn, err = db.Begin()
		if err != nil {
			log.Errorf("exec sqls[%v] begin failed %v", sqls, errors.ErrorStack(err))
			continue
		}

		for i := range sqls {
			log.Debugf("[exec][sql]%s[args]%v", sqls[i], args[i])

			_, err = txn.Exec(sqls[i], args[i]...)
			if err != nil {
				if !isRetryableError(err) {
					rerr := txn.Rollback()
					if rerr != nil {
						log.Errorf("[exec][sql]%s[args]%v[error]%v", sqls[i], args[i], rerr)
					}
					break LOOP
				}

				log.Warnf("[exec][sql]%s[args]%v[error]%v", sqls[i], args[i], err)
				rerr := txn.Rollback()
				if rerr != nil {
					log.Errorf("[exec][sql]%s[args]%v[error]%v", sqls[i], args[i], rerr)
				}
				continue LOOP
			}
		}
		err = txn.Commit()
		if err != nil {
			log.Errorf("exec sqls[%v] commit failed %v", sqls, errors.ErrorStack(err))
			continue
		}

		return nil
	}

	if err != nil {
		log.Errorf("exec sqls[%v] failed %v", sqls, errors.ErrorStack(err))
		return errors.Trace(err)
	}

	return errors.Errorf("exec sqls[%v] failed", sqls)
}

func createDB(cfg DBConfig) (*sql.DB, error) {
	dbDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/?charset=utf8&interpolateParams=true", cfg.User, cfg.Password, cfg.Host, cfg.Port)
	db, err := sql.Open("mysql", dbDSN)
	if err != nil {
		return nil, errors.Trace(err)
	}

	return db, nil
}

func closeDB(db *sql.DB) error {
	if db == nil {
		return nil
	}

	return errors.Trace(db.Close())
}

func createDBs(cfg DBConfig, count int) ([]*sql.DB, error) {
	dbs := make([]*sql.DB, 0, count)
	for i := 0; i < count; i++ {
		db, err := createDB(cfg)
		if err != nil {
			return nil, errors.Trace(err)
		}

		dbs = append(dbs, db)
	}

	return dbs, nil
}

func closeDBs(dbs ...*sql.DB) {
	for _, db := range dbs {
		err := closeDB(db)
		if err != nil {
			log.Errorf("close db failed - %v", err)
		}
	}
}
<|endoftext|>


<commit_before>package titan

// DB wraps all database related functions.
type DB interface {
	UserDB
}

// todo: db.GetByMail is not good, either use udb.GetByMail -or- db.Users.GetByMail

// UserDB presists user information in database.
type UserDB interface {
	GetByID(id string) (*User, bool)
	GetByMail(mail string) (*User, bool)
	SaveUser(u *User) error
}

// InMemDB is an in-memory database.
type InMemDB struct {
	InMemMsgQ
	InMemUserDB
}

// InMemMsgQ is a DB impelementation for development and testing.
type InMemMsgQ struct{}

// NewInMemDB creates a new in-memory database.
func NewInMemDB() InMemDB {
	return InMemDB{
		InMemUserDB: InMemUserDB{
			ids:    make(map[string]*User),
			emails: make(map[string]*User),
		},
	}
}

// InMemUserDB is in-memory user database.
type InMemUserDB struct {
	ids    map[string]*User
	emails map[string]*User
}

// GetByID retrieves a user by ID.
func (db InMemUserDB) GetByID(id string) (u *User, ok bool) {
	u, ok = db.ids[id]
	return
}

// GetByMail retrieves a user by e-mail address.
func (db InMemUserDB) GetByMail(email string) (u *User, ok bool) {
	u, ok = db.emails[email]
	return
}

// SaveUser save or updates a user object in the database.
func (db InMemUserDB) SaveUser(u *User) error {
	if u.ID == "" {
		u.ID = string(len(db.ids) + 1)
	}

	db.ids[u.ID] = u
	db.emails[u.Email] = u
	return nil
}
<commit_msg>db: fix user id conversion<commit_after>package titan

import "strconv"

// DB wraps all database related functions.
type DB interface {
	UserDB
}

// todo: db.GetByMail is not good, either use udb.GetByMail -or- db.Users.GetByMail

// UserDB presists user information in database.
type UserDB interface {
	GetByID(id string) (*User, bool)
	GetByMail(mail string) (*User, bool)
	SaveUser(u *User) error
}

// InMemDB is an in-memory database.
type InMemDB struct {
	InMemMsgQ
	InMemUserDB
}

// InMemMsgQ is a DB impelementation for development and testing.
type InMemMsgQ struct{}

// NewInMemDB creates a new in-memory database.
func NewInMemDB() InMemDB {
	return InMemDB{
		InMemUserDB: InMemUserDB{
			ids:    make(map[string]*User),
			emails: make(map[string]*User),
		},
	}
}

// InMemUserDB is in-memory user database.
type InMemUserDB struct {
	ids    map[string]*User
	emails map[string]*User
}

// GetByID retrieves a user by ID.
func (db InMemUserDB) GetByID(id string) (u *User, ok bool) {
	u, ok = db.ids[id]
	return
}

// GetByMail retrieves a user by e-mail address.
func (db InMemUserDB) GetByMail(email string) (u *User, ok bool) {
	u, ok = db.emails[email]
	return
}

// SaveUser save or updates a user object in the database.
func (db InMemUserDB) SaveUser(u *User) error {
	if u.ID == "" {
		u.ID = strconv.Itoa(len(db.ids) + 1)
	}

	db.ids[u.ID] = u
	db.emails[u.Email] = u
	return nil
}
<|endoftext|>


<commit_before>package server

import (
	"../hashtree"
	"../network"
	"io"
	"time"
	"sync"
)

// Implements WaitFor for Database, a database expect WaitFor
// to happen on different go routines than all other functions
type ListeningDatabase struct {
	Database
	listeners map[string]map[chan FileState] bool
	lock sync.RWMutex //locks listener R & W
}

func NewListeningDatabase(d Database) *ListeningDatabase {
	return &ListeningDatabase{d, make(map[string]map[chan FileState] bool), sync.RWMutex{}}
}

func (d *ListeningDatabase) AddListener(id network.StaticId, listener chan FileState) {
	sid := id.CompactId()
	d.lock.Lock()
	defer d.lock.Unlock()
	ls, ok := d.listeners[sid]
	if !ok {
		ls = make(map[chan FileState] bool)
		d.listeners[sid] = ls
	}
	ls[listener] = true
}

func (d *ListeningDatabase) RemoveListener(id network.StaticId, listener chan FileState) {
	sid := id.CompactId()
	d.lock.Lock()
	defer d.lock.Unlock()
	ls, ok := d.listeners[sid]
	if ok {
		delete(ls, listener)
		if len(ls) == 0 {
			delete(d.listeners,sid)
		}
	}
}

func (d *ListeningDatabase) writeHappend(id network.StaticId) {
	sid := id.CompactId()
	d.lock.RLock()
	defer d.lock.RUnlock()
	ls, ok := d.listeners[sid]
	if ok {
		state := d.GetState(id)
		for listener := range ls {
			listener <- state
		}
	}
}

func (d *ListeningDatabase) Close() {
	d.Database.Close()
	// terminate all listners
}

func (d *ListeningDatabase) ImportFromReader(r io.Reader) network.StaticId {
	id := d.Database.ImportFromReader(r)
	d.writeHappend(id)
	return id
}

func (d *ListeningDatabase) WaitFor(id network.StaticId, toState FileState, timeOut time.Duration) (ok bool, curState FileState) {
	listener := make(chan FileState)
	defer close(listener)
	d.AddListener(id, listener)
	defer d.RemoveListener(id, listener)

	startState := d.GetState(id)
	if startState == toState {
		return true, startState
	}
	for true {
		select {
		case state := <-listener:
			if state == toState {
				return true, state
			}
		case <-time.After(timeOut):
			state := d.GetState(id)
			return state == toState, state
		}
	}
	panic("code should not reach here")
}

func (d *ListeningDatabase) StartPart(id network.StaticId) error {
	err := d.Database.StartPart(id)
	d.writeHappend(id)
	return err
}

func (d *ListeningDatabase) PutAt(b []byte, id network.StaticId, off hashtree.Bytes) (has hashtree.Nodes, complete bool, err error) {
	has, complete, err = d.Database.PutAt(b, id, off)
	d.writeHappend(id)
	return
}

func (d *ListeningDatabase) PutInnerHashes(id network.StaticId, set network.InnerHashes) (has hashtree.Nodes, complete bool, err error) {
	has, complete, err = d.Database.PutInnerHashes(id, set)
	d.writeHappend(id)
	return
}
<commit_msg>optimize locking to move file access out of critical section<commit_after>package server

import (
	"../hashtree"
	"../network"
	"io"
	"sync"
	"time"
)

// Implements WaitFor for Database, a database expect WaitFor
// to happen on different go routines than all other functions
type ListeningDatabase struct {
	Database
	listeners map[string]map[chan FileState]bool
	lock      sync.RWMutex //locks listeners R & W
}

func NewListeningDatabase(d Database) *ListeningDatabase {
	return &ListeningDatabase{d, make(map[string]map[chan FileState]bool), sync.RWMutex{}}
}

func (d *ListeningDatabase) AddListener(id network.StaticId, listener chan FileState) {
	sid := id.CompactId()
	d.lock.Lock()
	defer d.lock.Unlock()
	ls, ok := d.listeners[sid]
	if !ok {
		ls = make(map[chan FileState]bool)
		d.listeners[sid] = ls
	}
	ls[listener] = true
}

func (d *ListeningDatabase) RemoveListener(id network.StaticId, listener chan FileState) {
	sid := id.CompactId()
	d.lock.Lock()
	defer d.lock.Unlock()
	ls, ok := d.listeners[sid]
	if ok {
		delete(ls, listener)
		if len(ls) == 0 {
			delete(d.listeners, sid)
		}
	}
}

func (d *ListeningDatabase) writeHappend(id network.StaticId) {
	sid := id.CompactId()
	_, ok := d.listeners[sid]
	if ok {
		state := d.GetState(id)
		// only get state (which may use disk) if there are listeners,
		// but before lock so it does not block, then we have to get
		// listeners again after lock (which is just one more map lookup)
		d.lock.RLock()
		defer d.lock.RUnlock()
		ls, ok := d.listeners[sid]
		if ok {
			for listener := range ls {
				listener <- state
			}
		}
	}
}

func (d *ListeningDatabase) Close() {
	d.Database.Close()
	// terminate all listners
}

func (d *ListeningDatabase) ImportFromReader(r io.Reader) network.StaticId {
	id := d.Database.ImportFromReader(r)
	d.writeHappend(id)
	return id
}

func (d *ListeningDatabase) WaitFor(id network.StaticId, toState FileState, timeOut time.Duration) (ok bool, curState FileState) {
	listener := make(chan FileState)
	defer close(listener)
	d.AddListener(id, listener)
	defer d.RemoveListener(id, listener)

	startState := d.GetState(id)
	if startState == toState {
		return true, startState
	}
	for true {
		select {
		case state := <-listener:
			if state == toState {
				return true, state
			}
		case <-time.After(timeOut):
			state := d.GetState(id)
			return state == toState, state
		}
	}
	panic("code should not reach here")
}

func (d *ListeningDatabase) StartPart(id network.StaticId) error {
	err := d.Database.StartPart(id)
	d.writeHappend(id)
	return err
}

func (d *ListeningDatabase) PutAt(b []byte, id network.StaticId, off hashtree.Bytes) (has hashtree.Nodes, complete bool, err error) {
	has, complete, err = d.Database.PutAt(b, id, off)
	d.writeHappend(id)
	return
}

func (d *ListeningDatabase) PutInnerHashes(id network.StaticId, set network.InnerHashes) (has hashtree.Nodes, complete bool, err error) {
	has, complete, err = d.Database.PutInnerHashes(id, set)
	d.writeHappend(id)
	return
}
<|endoftext|>


<commit_before>package y

import (
	"database/sql"
	"log"

	sq "github.com/lann/squirrel"
)

// Execer decribes exec operation
type Execer interface {
	Exec(string, ...interface{}) (sql.Result, error)
}

// Queryer decribes query operation
type Queryer interface {
	Query(string, ...interface{}) (*sql.Rows, error)
	QueryRow(string, ...interface{}) *sql.Row
}

// Versionable mixins a version to a model
type Versionable struct {
	Version sql.NullInt64 `json:"-" y:"_version"`
}

// MakeVersionable inits a new version
func MakeVersionable(n int64) Versionable {
	return Versionable{
		sql.NullInt64{Int64: 1, Valid: true},
	}
}

// DB describes db operations
type DB interface {
	Execer
	Queryer
}

// Qualifier updates a select builder if you need
type Qualifier func(sq.SelectBuilder) sq.SelectBuilder

// ByEq returns the filter by squirrel.Eq
var ByEq = func(eq sq.Eq) Qualifier {
	return func(b sq.SelectBuilder) sq.SelectBuilder {
		return b.Where(eq)
	}
}

// ByID returns the filter by ID
var ByID = func(id interface{}) Qualifier {
	return ByEq(sq.Eq{"id": id})
}

// TxPipe run some db statement
type TxPipe func(db DB, v interface{}) error

// Tx executes statements in a transaction
func Tx(db *sql.DB, v interface{}, pipes ...TxPipe) (err error) {
	tx, err := db.Begin()
	if err != nil {
		return
	}
	for _, pipe := range pipes {
		err = pipe(tx, v)
		if err != nil {
			tx.Rollback()
			return
		}
	}
	tx.Commit()
	return
}

func sqlize(q sq.Sqlizer) (sql string, args []interface{}) {
	sql, args, _ = q.ToSql()
	if Debug {
		log.Printf("y/db: SQL: %s, args: %#v", sql, args)
	}
	return
}

func exec(q sq.Sqlizer, db DB) (sql.Result, error) {
	sql, args := sqlize(q)
	return db.Exec(sql, args...)
}

func query(q sq.Sqlizer, db DB) (*sql.Rows, error) {
	sql, args := sqlize(q)
	return db.Query(sql, args...)
}

func queryRow(q sq.Sqlizer, db DB) *sql.Row {
	sql, args := sqlize(q)
	return db.QueryRow(sql, args...)
}
<commit_msg>fixed stupid bug in MakeVersionable<commit_after>package y

import (
	"database/sql"
	"log"

	sq "github.com/lann/squirrel"
)

// Execer decribes exec operation
type Execer interface {
	Exec(string, ...interface{}) (sql.Result, error)
}

// Queryer decribes query operation
type Queryer interface {
	Query(string, ...interface{}) (*sql.Rows, error)
	QueryRow(string, ...interface{}) *sql.Row
}

// Versionable mixins a version to a model
type Versionable struct {
	Version sql.NullInt64 `json:"-" y:"_version"`
}

// MakeVersionable inits a new version
func MakeVersionable(n int64) Versionable {
	return Versionable{
		sql.NullInt64{Int64: n, Valid: true},
	}
}

// DB describes db operations
type DB interface {
	Execer
	Queryer
}

// Qualifier updates a select builder if you need
type Qualifier func(sq.SelectBuilder) sq.SelectBuilder

// ByEq returns the filter by squirrel.Eq
var ByEq = func(eq sq.Eq) Qualifier {
	return func(b sq.SelectBuilder) sq.SelectBuilder {
		return b.Where(eq)
	}
}

// ByID returns the filter by ID
var ByID = func(id interface{}) Qualifier {
	return ByEq(sq.Eq{"id": id})
}

// TxPipe run some db statement
type TxPipe func(db DB, v interface{}) error

// Tx executes statements in a transaction
func Tx(db *sql.DB, v interface{}, pipes ...TxPipe) (err error) {
	tx, err := db.Begin()
	if err != nil {
		return
	}
	for _, pipe := range pipes {
		err = pipe(tx, v)
		if err != nil {
			tx.Rollback()
			return
		}
	}
	tx.Commit()
	return
}

func sqlize(q sq.Sqlizer) (sql string, args []interface{}) {
	sql, args, _ = q.ToSql()
	if Debug {
		log.Printf("y/db: SQL: %s, args: %#v", sql, args)
	}
	return
}

func exec(q sq.Sqlizer, db DB) (sql.Result, error) {
	sql, args := sqlize(q)
	return db.Exec(sql, args...)
}

func query(q sq.Sqlizer, db DB) (*sql.Rows, error) {
	sql, args := sqlize(q)
	return db.Query(sql, args...)
}

func queryRow(q sq.Sqlizer, db DB) *sql.Row {
	sql, args := sqlize(q)
	return db.QueryRow(sql, args...)
}
<|endoftext|>


<commit_before>package simpledb

import (
	//"encoding/json"
	"bytes"
	"errors"
	"fmt"
	"strconv"

	"github.com/garyburd/redigo/redis"
)

type DB struct {
	Name string
}

func Open(name string) *DB {
	return &DB{Name: name}
}

func (db *DB) Close() {
}

func (db *DB) GenMaxIdKey() (maxIdKey string) {
	return fmt.Sprintf("%v/maxid", db.Name)
}

func (db *DB) GetMaxId(c redis.Conn) (maxId uint64, err error) {
	k := db.GenMaxIdKey()
	exists, err := redis.Bool(c.Do("EXISTS", k))
	if err != nil {
		goto end
	}

	if !exists {
		maxId = 0
		goto end
	}

	maxId, err = redis.Uint64(c.Do("GET", k))
	if err != nil {
		goto end
	}

end:
	if err != nil {
		debugPrintf("GenMaxId() error: %v\n", err)
		return 0, err
	}

	return maxId, nil
}

func (db *DB) GenMaxBucketIdKey() (maxBucketIdKey string) {
	return fmt.Sprintf("%v/maxbucketid", db.Name)
}

func (db *DB) GetMaxBucketId(c redis.Conn) (maxBucketId uint64, err error) {
	k := db.GenMaxBucketIdKey()
	exists, err := redis.Bool(c.Do("EXISTS", k))
	if err != nil {
		debugPrintf("XX error: %v\n", err)
		goto end
	}

	if !exists {
		_, err := c.Do("SET", k, 1)
		if err != nil {
			goto end
		}
	}

	maxBucketId, err = redis.Uint64(c.Do("GET", k))
	if err != nil {
		goto end
	}

end:
	if err != nil {
		debugPrintf("GetMaxBucketId() error: %v\n", err)
		return 1, err
	}

	return maxBucketId, nil
}

func (db *DB) GenHashKey(bucketId uint64) (recordHashKey, indexHashKey string) {
	bucketIdStr := strconv.FormatUint(bucketId, 10)
	recordHashKey = fmt.Sprintf("%v/bucket/%v", db.Name, bucketIdStr)
	indexHashKey = fmt.Sprintf("%v/idx/bucket/%v", db.Name, bucketIdStr)
	return recordHashKey, indexHashKey
}

func (db *DB) Exists(c redis.Conn, data string) (exists bool, err error) {
	exists = false
	maxBucketId, err := db.GetMaxBucketId(c)
	indexHashKey := ""
	indexHashField := data

	if err != nil {
		goto end
	}

	for i := maxBucketId; i >= 1; i-- {
		_, indexHashKey = db.GenHashKey(i)
		exists, err = redis.Bool(c.Do("HEXISTS", indexHashKey, indexHashField))
		if err != nil {
			goto end
		}

		if exists {
			exists = true
			goto end
		}
	}
end:
	if err != nil {
		debugPrintf("Exists() error: %v\n", err)
		return false, err
	}

	return exists, nil
}

func (db *DB) Create(c redis.Conn, data string) (id string, err error) {
	ids := []string{}
	ids, err = db.BatchCreate(c, []string{data})
	if err != nil {
		goto end
	}

	if len(ids) != 1 {
		err = errors.New("Count of created record != 1.")
		goto end
	}

end:
	if err != nil {
		debugPrintf("Create() error: %v\n", err)
		return "", err
	}

	return ids[0], nil
}

func (db *DB) BatchCreate(c redis.Conn, dataArr []string) (ids []string, err error) {
	var checkedData map[string]int = make(map[string]int) // key: data, value: order in dataArr.
	var nId, maxId, bucketId, maxBucketId, recordHashField uint64
	var maxBucketIdKey, recordHashKey, indexHashKey, indexHashField, maxIdKey string
	var ret interface{}
	err = nil
	ids = []string{}
	id := ""
	ok := false
	exists := false
	alreadySendMULTI := false

	// Check data.
	for i, data := range dataArr {
		// Check empty data.
		if len(data) == 0 {
			err = errors.New("Empty data.")
			goto end
		}

		// Check redundant data in dataArr.
		if _, ok = checkedData[data]; ok {
			err = errors.New(fmt.Sprintf("Redundant data found in dataArr: %v", data))
			goto end
		}
		checkedData[data] = i

		// Check if data already exist in db.
		exists, err = db.Exists(c, data)
		if err != nil {
			goto end
		}

		if exists {
			err = errors.New(fmt.Sprintf("Data already exists in db: %v.", data))
			goto end
		}
	}

	// Get max id.
	maxId, err = db.GetMaxId(c)
	if err != nil {
		goto end
	}

	// Get max id key.
	maxIdKey = db.GenMaxIdKey()

	// Get max bucket id.
	maxBucketId, err = db.GetMaxBucketId(c)
	if err != nil {
		goto end
	}

	// Prepare piplined transaction.
	c.Send("MULTI")
	alreadySendMULTI = true

	for i, data := range dataArr {
		// Increase Id
		nId = maxId + uint64(i+1)
		id = strconv.FormatUint(nId, 10)
		// Insert to result id array
		ids = append(ids, id)

		// Compute bucket id.
		bucketId = ComputeBucketId(nId)

		// Increase max bucket id if need.
		if bucketId > maxBucketId {
			c.Send("SET", maxBucketIdKey, bucketId)
		}

		// Generate hash key for record and index.
		recordHashKey, indexHashKey = db.GenHashKey(bucketId)

		// Create record and index.
		recordHashField = nId
		indexHashField = data

		c.Send("HSET", recordHashKey, recordHashField, data)
		c.Send("HSET", indexHashKey, indexHashField, nId)
		c.Send("INCR", maxIdKey)
	}

	// Do piplined transaction.
	ret, err = c.Do("EXEC")
	if err != nil {
		goto end
	}

	debugPrintf("BatchCreate() ok. ret: %v, ids: %v\n", ret, ids)

end:
	if err != nil {
		if alreadySendMULTI {
			c.Do("DISCARD")
		}
		debugPrintf("BatchCreate() error: %v\n", err)
		return []string{}, err
	}

	return ids, nil
}

func (db *DB) IdExists(c redis.Conn, id string) (exists bool, recordHashKey, indexHashKey string, recordHashField uint64, err error) {
	var nId, bucketId uint64

	nId, err = strconv.ParseUint(id, 10, 64)
	if err != nil {
		goto end
	}

	bucketId = ComputeBucketId(nId)
	recordHashKey, indexHashKey = db.GenHashKey(bucketId)
	recordHashField = nId

	exists, err = redis.Bool(c.Do("HEXISTS", recordHashKey, recordHashField))
	if err != nil {
		goto end
	}

end:
	if err != nil {
		debugPrintf("IdExists()  error: %v\n", err)
		return false, "", "", 0, err
	}

	return exists, recordHashKey, indexHashKey, recordHashField, nil
}

func (db *DB) Get(c redis.Conn, id string) (data string, err error) {
	exists, recordHashKey, _, recordHashField, err := db.IdExists(c, id)
	if err != nil {
		goto end
	}

	if !exists {
		err = errors.New("Record filed does not exists in hash key.")
		goto end
	}

	data, err = redis.String(c.Do("HGET", recordHashKey, recordHashField))
	if err != nil {
		goto end
	}

end:
	if err != nil {
		debugPrintf("Get() error: %v\n", err)
		return "", err
	}

	return data, nil
}

func (db *DB) BatchGet(c redis.Conn, ids []string) (dataMap map[string]string, err error) {
	dataMap = make(map[string]string)
	data := ""

	for _, id := range ids {
		data, err = db.Get(c, id)
		if err != nil {
			debugPrintf("BatchGet(): db.Get() error: %v\n", err)
			return dataMap, err
		}
		dataMap[id] = data
	}
	return dataMap, nil
}

func (db *DB) Update(c redis.Conn, id, data string) error {
	var oldData, oldIndexHashField, newIndexHashField string
	var nId uint64 = 0
	var ret interface{}

	exists, recordHashKey, indexHashKey, recordHashField, err := db.IdExists(c, id)
	if err != nil {
		goto end
	}

	if !exists {
		err = errors.New("Record does not exist.")
		goto end
	}

	oldData, err = db.Get(c, id)
	if err != nil {
		goto end
	}

	oldIndexHashField = oldData
	newIndexHashField = data
	nId = recordHashField

	// Check if data already exists.
	exists, err = db.Exists(c, data)
	if err != nil {
		goto end
	}

	// Index already exists, there's already a record has the same value / index.
	if exists {
		err = errors.New("Same data / index already exists.")
		goto end
	}

	c.Send("MULTI")
	c.Send("HSET", recordHashKey, recordHashField, data)
	c.Send("HSET", indexHashKey, newIndexHashField, nId)
	c.Send("HDEL", indexHashKey, oldIndexHashField)
	ret, err = c.Do("EXEC")
	if err != nil {
		goto end
	}

	debugPrintf("Update() ok. ret: %v\n", ret)

end:
	if err != nil {
		debugPrintf("Update() error: %v\n", err)
		return err
	}

	return nil
}

func (db *DB) BatchUpdate(c redis.Conn, dataMap map[string]string) (err error) {
	type updateInfo struct {
		data              string
		recordHashKey     string
		recordHashField   uint64
		indexHashKey      string
		oldIndexHashField string
		newIndexHashField string
	}

	var recordHashKey, indexHashKey, oldData string
	var recordHashField uint64
	var ret interface{}
	exists := false
	updateInfoMap := make(map[uint64]updateInfo)
	alreadySendMULTI := false

	// Check and input dataArr
	for id, data := range dataMap {
		exists, recordHashKey, indexHashKey, recordHashField, err = db.IdExists(c, id)
		if err != nil {
			goto end
		}

		if !exists {
			err = errors.New(fmt.Sprintf("Id: %v does not exist.", id))
			goto end
		}

		oldData, err = db.Get(c, id)
		if err != nil {
			goto end
		}

		// Check if new data equals to old data: no need to update.
		if !bytes.Equal([]byte(data), []byte(oldData)) {
			updateInfoMap[recordHashField] = updateInfo{
				data:              data,
				recordHashKey:     recordHashKey,
				recordHashField:   recordHashField,
				indexHashKey:      indexHashKey,
				oldIndexHashField: oldData,
				newIndexHashField: data}
		}

		// Check if data already exists in db(may be with another id).
		exists, err = db.Exists(c, data)
		if err != nil {
			goto end
		}

		if exists {
			err = errors.New(fmt.Sprintf("Data already exists in db: %v.", data))
			goto end
		}
	}

	// Prepare pipelined transaction.
	c.Send("MULTI")
	alreadySendMULTI = true

	for nId, info := range updateInfoMap {
		c.Send("HSET", info.recordHashKey, info.recordHashField, info.data)
		c.Send("HSET", info.indexHashKey, info.newIndexHashField, nId)
		c.Send("HDEL", info.indexHashKey, info.oldIndexHashField)
	}

	ret, err = c.Do("EXEC")
	if err != nil {
		goto end
	}

	debugPrintf("BatchUpdate() ok. ret: %v\n", ret)

end:
	if err != nil {
		if alreadySendMULTI {
			c.Do("DISCARD")
		}
		debugPrintf("BatchUpdate() error: %v\n", err)
		return err
	}

	return nil
}

func (db *DB) Search(c redis.Conn, pattern string) (ids []string, err error) {
	var maxBucketId, cursor uint64
	var l int = 0
	var v []interface{}
	indexHashKey := ""
	items := []string{}
	ids = []string{}

	if len(pattern) == 0 {
		err = errors.New("Empty pattern.")
		goto end
	}

	maxBucketId, err = db.GetMaxBucketId(c)
	if err != nil {
		goto end
	}

	for i := maxBucketId; i >= 1; i-- {
		_, indexHashKey = db.GenHashKey(i)
		cursor = 0
		for {
			v, err = redis.Values(c.Do("HSCAN", indexHashKey, cursor, "match", pattern))
			if err != nil {
				goto end
			}

			v, err = redis.Scan(v, &cursor, &items)
			if err != nil {
				goto end
			}

			l = len(items)
			if l > 0 && l%2 == 0 {
				for m := 1; m < l; m += 2 {
					ids = append(ids, items[m])
				}
			}

			if cursor == 0 {
				break
			}
		}
	}
end:
	if err != nil {
		debugPrintf("Search() error: %v\n", err)
		return []string{}, err
	}

	return ids, nil
}
<commit_msg>Implement DB.Create() with DB.BatchCreate().<commit_after>package simpledb

import (
	//"encoding/json"
	"bytes"
	"errors"
	"fmt"
	"strconv"

	"github.com/garyburd/redigo/redis"
)

type DB struct {
	Name string
}

func Open(name string) *DB {
	return &DB{Name: name}
}

func (db *DB) Close() {
}

func (db *DB) GenMaxIdKey() (maxIdKey string) {
	return fmt.Sprintf("%v/maxid", db.Name)
}

func (db *DB) GetMaxId(c redis.Conn) (maxId uint64, err error) {
	k := db.GenMaxIdKey()
	exists, err := redis.Bool(c.Do("EXISTS", k))
	if err != nil {
		goto end
	}

	if !exists {
		maxId = 0
		goto end
	}

	maxId, err = redis.Uint64(c.Do("GET", k))
	if err != nil {
		goto end
	}

end:
	if err != nil {
		debugPrintf("GenMaxId() error: %v\n", err)
		return 0, err
	}

	return maxId, nil
}

func (db *DB) GenMaxBucketIdKey() (maxBucketIdKey string) {
	return fmt.Sprintf("%v/maxbucketid", db.Name)
}

func (db *DB) GetMaxBucketId(c redis.Conn) (maxBucketId uint64, err error) {
	k := db.GenMaxBucketIdKey()
	exists, err := redis.Bool(c.Do("EXISTS", k))
	if err != nil {
		debugPrintf("XX error: %v\n", err)
		goto end
	}

	if !exists {
		_, err := c.Do("SET", k, 1)
		if err != nil {
			goto end
		}
	}

	maxBucketId, err = redis.Uint64(c.Do("GET", k))
	if err != nil {
		goto end
	}

end:
	if err != nil {
		debugPrintf("GetMaxBucketId() error: %v\n", err)
		return 1, err
	}

	return maxBucketId, nil
}

func (db *DB) GenHashKey(bucketId uint64) (recordHashKey, indexHashKey string) {
	bucketIdStr := strconv.FormatUint(bucketId, 10)
	recordHashKey = fmt.Sprintf("%v/bucket/%v", db.Name, bucketIdStr)
	indexHashKey = fmt.Sprintf("%v/idx/bucket/%v", db.Name, bucketIdStr)
	return recordHashKey, indexHashKey
}

func (db *DB) Exists(c redis.Conn, data string) (exists bool, err error) {
	exists = false
	maxBucketId, err := db.GetMaxBucketId(c)
	indexHashKey := ""
	indexHashField := data

	if err != nil {
		goto end
	}

	for i := maxBucketId; i >= 1; i-- {
		_, indexHashKey = db.GenHashKey(i)
		exists, err = redis.Bool(c.Do("HEXISTS", indexHashKey, indexHashField))
		if err != nil {
			goto end
		}

		if exists {
			exists = true
			goto end
		}
	}
end:
	if err != nil {
		debugPrintf("Exists() error: %v\n", err)
		return false, err
	}

	return exists, nil
}

func (db *DB) Create(c redis.Conn, data string) (id string, err error) {
	ids := []string{}
	ids, err = db.BatchCreate(c, []string{data})
	if err != nil {
		goto end
	}

	if len(ids) != 1 {
		err = errors.New("Count of created record != 1.")
		goto end
	}

end:
	if err != nil {
		debugPrintf("Create() error: %v\n", err)
		return "", err
	}

	return ids[0], nil
}

func (db *DB) BatchCreate(c redis.Conn, dataArr []string) (ids []string, err error) {
	var checkedData map[string]int = make(map[string]int) // key: data, value: order in dataArr.
	var nId, maxId, bucketId, maxBucketId, recordHashField uint64
	var maxBucketIdKey, recordHashKey, indexHashKey, indexHashField, maxIdKey string
	var ret interface{}
	err = nil
	ids = []string{}
	id := ""
	ok := false
	exists := false
	alreadySendMULTI := false

	// Check data.
	for i, data := range dataArr {
		// Check empty data.
		if len(data) == 0 {
			err = errors.New("Empty data.")
			goto end
		}

		// Check redundant data in dataArr.
		if _, ok = checkedData[data]; ok {
			err = errors.New(fmt.Sprintf("Redundant data found in dataArr: %v", data))
			goto end
		}
		checkedData[data] = i

		// Check if data already exist in db.
		exists, err = db.Exists(c, data)
		if err != nil {
			goto end
		}

		if exists {
			err = errors.New(fmt.Sprintf("Data already exists in db: %v.", data))
			goto end
		}
	}

	// Get max id.
	maxId, err = db.GetMaxId(c)
	if err != nil {
		goto end
	}

	// Get max id key.
	maxIdKey = db.GenMaxIdKey()

	// Get max bucket id.
	maxBucketId, err = db.GetMaxBucketId(c)
	if err != nil {
		goto end
	}

	// Prepare piplined transaction.
	c.Send("MULTI")
	alreadySendMULTI = true

	for i, data := range dataArr {
		// Increase Id
		nId = maxId + uint64(i+1)
		id = strconv.FormatUint(nId, 10)
		// Insert to result id array
		ids = append(ids, id)

		// Compute bucket id.
		bucketId = ComputeBucketId(nId)

		// Increase max bucket id if need.
		if bucketId > maxBucketId {
			c.Send("SET", maxBucketIdKey, bucketId)
		}

		// Generate hash key for record and index.
		recordHashKey, indexHashKey = db.GenHashKey(bucketId)

		// Create record and index.
		recordHashField = nId
		indexHashField = data

		c.Send("HSET", recordHashKey, recordHashField, data)
		c.Send("HSET", indexHashKey, indexHashField, nId)
		c.Send("INCR", maxIdKey)
	}

	// Do piplined transaction.
	ret, err = c.Do("EXEC")
	if err != nil {
		goto end
	}

	debugPrintf("BatchCreate() ok. ret: %v, ids: %v\n", ret, ids)

end:
	if err != nil {
		if alreadySendMULTI {
			c.Do("DISCARD")
		}
		debugPrintf("BatchCreate() error: %v\n", err)
		return []string{}, err
	}

	return ids, nil
}

func (db *DB) IdExists(c redis.Conn, id string) (exists bool, recordHashKey, indexHashKey string, recordHashField uint64, err error) {
	var nId, bucketId uint64

	nId, err = strconv.ParseUint(id, 10, 64)
	if err != nil {
		goto end
	}

	bucketId = ComputeBucketId(nId)
	recordHashKey, indexHashKey = db.GenHashKey(bucketId)
	recordHashField = nId

	exists, err = redis.Bool(c.Do("HEXISTS", recordHashKey, recordHashField))
	if err != nil {
		goto end
	}

end:
	if err != nil {
		debugPrintf("IdExists()  error: %v\n", err)
		return false, "", "", 0, err
	}

	return exists, recordHashKey, indexHashKey, recordHashField, nil
}

func (db *DB) Get(c redis.Conn, id string) (data string, err error) {
	exists, recordHashKey, _, recordHashField, err := db.IdExists(c, id)
	if err != nil {
		goto end
	}

	if !exists {
		err = errors.New("Record filed does not exists in hash key.")
		goto end
	}

	data, err = redis.String(c.Do("HGET", recordHashKey, recordHashField))
	if err != nil {
		goto end
	}

end:
	if err != nil {
		debugPrintf("Get() error: %v\n", err)
		return "", err
	}

	return data, nil
}

func (db *DB) BatchGet(c redis.Conn, ids []string) (dataMap map[string]string, err error) {
	dataMap = make(map[string]string)
	data := ""

	for _, id := range ids {
		data, err = db.Get(c, id)
		if err != nil {
			debugPrintf("BatchGet(): db.Get() error: %v\n", err)
			return dataMap, err
		}
		dataMap[id] = data
	}
	return dataMap, nil
}

func (db *DB) Update(c redis.Conn, id, data string) error {
	dataMap := make(map[string]string)
	dataMap[id] = data
	return db.BatchUpdate(c, dataMap)
}

func (db *DB) BatchUpdate(c redis.Conn, dataMap map[string]string) (err error) {
	type updateInfo struct {
		data              string
		recordHashKey     string
		recordHashField   uint64
		indexHashKey      string
		oldIndexHashField string
		newIndexHashField string
	}

	var recordHashKey, indexHashKey, oldData string
	var recordHashField uint64
	var ret interface{}
	exists := false
	updateInfoMap := make(map[uint64]updateInfo)
	alreadySendMULTI := false

	// Check and input dataArr
	for id, data := range dataMap {
		exists, recordHashKey, indexHashKey, recordHashField, err = db.IdExists(c, id)
		if err != nil {
			goto end
		}

		if !exists {
			err = errors.New(fmt.Sprintf("Id: %v does not exist.", id))
			goto end
		}

		oldData, err = db.Get(c, id)
		if err != nil {
			goto end
		}

		// Check if new data equals to old data: no need to update.
		if !bytes.Equal([]byte(data), []byte(oldData)) {
			updateInfoMap[recordHashField] = updateInfo{
				data:              data,
				recordHashKey:     recordHashKey,
				recordHashField:   recordHashField,
				indexHashKey:      indexHashKey,
				oldIndexHashField: oldData,
				newIndexHashField: data}
		}

		// Check if data already exists in db(may be with another id).
		exists, err = db.Exists(c, data)
		if err != nil {
			goto end
		}

		if exists {
			err = errors.New(fmt.Sprintf("Data already exists in db: %v.", data))
			goto end
		}
	}

	// Prepare pipelined transaction.
	c.Send("MULTI")
	alreadySendMULTI = true

	for nId, info := range updateInfoMap {
		c.Send("HSET", info.recordHashKey, info.recordHashField, info.data)
		c.Send("HSET", info.indexHashKey, info.newIndexHashField, nId)
		c.Send("HDEL", info.indexHashKey, info.oldIndexHashField)
	}

	ret, err = c.Do("EXEC")
	if err != nil {
		goto end
	}

	debugPrintf("BatchUpdate() ok. ret: %v\n", ret)

end:
	if err != nil {
		if alreadySendMULTI {
			c.Do("DISCARD")
		}
		debugPrintf("BatchUpdate() error: %v\n", err)
		return err
	}

	return nil
}

func (db *DB) Search(c redis.Conn, pattern string) (ids []string, err error) {
	var maxBucketId, cursor uint64
	var l int = 0
	var v []interface{}
	indexHashKey := ""
	items := []string{}
	ids = []string{}

	if len(pattern) == 0 {
		err = errors.New("Empty pattern.")
		goto end
	}

	maxBucketId, err = db.GetMaxBucketId(c)
	if err != nil {
		goto end
	}

	for i := maxBucketId; i >= 1; i-- {
		_, indexHashKey = db.GenHashKey(i)
		cursor = 0
		for {
			v, err = redis.Values(c.Do("HSCAN", indexHashKey, cursor, "match", pattern))
			if err != nil {
				goto end
			}

			v, err = redis.Scan(v, &cursor, &items)
			if err != nil {
				goto end
			}

			l = len(items)
			if l > 0 && l%2 == 0 {
				for m := 1; m < l; m += 2 {
					ids = append(ids, items[m])
				}
			}

			if cursor == 0 {
				break
			}
		}
	}
end:
	if err != nil {
		debugPrintf("Search() error: %v\n", err)
		return []string{}, err
	}

	return ids, nil
}
<|endoftext|>