<commit_before>package database_test

import (
	. "github.com/pivotal-cf-experimental/cf-mysql-quota-enforcer/database"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"errors"

	"database/sql"

	sqlfakes "github.com/pivotal-cf-experimental/cf-mysql-quota-enforcer/sql/fakes"
	"github.com/pivotal-golang/lager/lagertest"
)

var _ = Describe("Database", func() {

	const dbName = "fake-db-name"
	var (
		logger   *lagertest.TestLogger
		database Database
		fakeDB   *sqlfakes.FakeDB
	)

	BeforeEach(func() {
		logger = lagertest.NewTestLogger("ProxyRunner test")
		fakeDB = &sqlfakes.FakeDB{}
		database = New(dbName, fakeDB, logger)
	})

	Describe("RevokePrivileges", func() {
		var fakeResult *sqlfakes.FakeResult

		BeforeEach(func() {
			fakeResult = &sqlfakes.FakeResult{}
			fakeDB.ExecReturns(fakeResult, nil)

			fakeResult.RowsAffectedReturns(1, nil)
		})

		It("makes a sql query to revoke priveledges on a database and then flushes privileges", func() {
			err := database.RevokePrivileges()
			Expect(err).ToNot(HaveOccurred())

			Expect(fakeDB.ExecCallCount()).To(Equal(2))

			query, args := fakeDB.ExecArgsForCall(0)
			Expect(query).To(Equal(`UPDATE mysql.db
SET Insert_priv = 'N', Update_priv = 'N', Create_priv = 'N'
WHERE Db = ?`))
			Expect(args).To(Equal([]interface{}{dbName}))

			query, args = fakeDB.ExecArgsForCall(1)
			Expect(query).To(Equal("FLUSH PRIVILEGES"))
			Expect(args).To(BeEmpty())
		})

		Context("when the query fails", func() {
			BeforeEach(func() {
				fakeDB.ExecReturns(nil, errors.New("fake-query-error"))
			})

			It("returns an error", func() {
				err := database.RevokePrivileges()
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-query-error"))
				Expect(err.Error()).To(ContainSubstring(dbName))

				Expect(fakeDB.ExecCallCount()).To(Equal(1))
			})
		})

		Context("when getting the number of affected rows fails", func() {
			BeforeEach(func() {
				fakeResult.RowsAffectedReturns(0, errors.New("fake-rows-affected-error"))
			})

			It("returns an error", func() {
				err := database.RevokePrivileges()
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-rows-affected-error"))
				Expect(err.Error()).To(ContainSubstring("Getting rows affected"))
				Expect(err.Error()).To(ContainSubstring(dbName))

				Expect(fakeDB.ExecCallCount()).To(Equal(1))
				Expect(fakeResult.RowsAffectedCallCount()).To(Equal(1))
			})
		})

		Context("when flushing privileges fails", func() {
			BeforeEach(func() {
				fakeDB.ExecStub = func(query string, args ...interface{}) (sql.Result, error) {
					if query == "FLUSH PRIVILEGES" {
						return nil, errors.New("fake-flush-error")
					}
					return fakeResult, nil
				}
			})

			It("returns an error", func() {
				err := database.RevokePrivileges()
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-flush-error"))

				Expect(fakeDB.ExecCallCount()).To(Equal(2))
			})
		})
	})

	Describe("GrantPrivileges", func() {
		var fakeResult *sqlfakes.FakeResult

		BeforeEach(func() {
			fakeResult = &sqlfakes.FakeResult{}
			fakeDB.ExecReturns(fakeResult, nil)

			fakeResult.RowsAffectedReturns(1, nil)
		})

		It("Expects priviledges to be granted to the database", func() {
			err := database.GrantPrivileges()
			Expect(err).ToNot(HaveOccurred())

			Expect(fakeDB.ExecCallCount()).To(Equal(2))

			query, args := fakeDB.ExecArgsForCall(0)
			Expect(query).To(Equal(`UPDATE mysql.db
SET Insert_priv = 'Y', Update_priv = 'Y', Create_priv = 'Y'
WHERE Db = ?`))
			Expect(args).To(Equal([]interface{}{dbName}))

			query, args = fakeDB.ExecArgsForCall(1)
			Expect(query).To(Equal("FLUSH PRIVILEGES"))
			Expect(args).To(BeEmpty())
		})

		Context("when the query fails", func() {
			BeforeEach(func() {
				fakeDB.ExecReturns(nil, errors.New("fake-query-error"))
			})

			It("returns an error", func() {
				err := database.GrantPrivileges()
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-query-error"))
				Expect(err.Error()).To(ContainSubstring(dbName))

				Expect(fakeDB.ExecCallCount()).To(Equal(1))
			})
		})

		Context("when getting the number of affected rows fails", func() {
			BeforeEach(func() {
				fakeResult.RowsAffectedReturns(0, errors.New("fake-rows-affected-error"))
			})

			It("returns an error", func() {
				err := database.GrantPrivileges()
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-rows-affected-error"))
				Expect(err.Error()).To(ContainSubstring("Getting rows affected"))
				Expect(err.Error()).To(ContainSubstring(dbName))

				Expect(fakeDB.ExecCallCount()).To(Equal(1))
				Expect(fakeResult.RowsAffectedCallCount()).To(Equal(1))
			})
		})

		Context("when flushing privileges fails", func() {
			BeforeEach(func() {
				fakeDB.ExecStub = func(query string, args ...interface{}) (sql.Result, error) {
					if query == "FLUSH PRIVILEGES" {
						return nil, errors.New("fake-flush-error")
					}
					return fakeResult, nil
				}
			})

			It("returns an error", func() {
				err := database.GrantPrivileges()
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("fake-flush-error"))

				Expect(fakeDB.ExecCallCount()).To(Equal(2))
			})
		})

	})

	Describe("ResetActivePrivileges", func() {

	})
})
<commit_msg>Switch from counterfeiter mocks to go-sqlmock<commit_after>package database_test

import (
	. "github.com/pivotal-cf-experimental/cf-mysql-quota-enforcer/database"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

    "errors"

	"database/sql"

	"github.com/DATA-DOG/go-sqlmock"
	sqlfakes "github.com/pivotal-cf-experimental/cf-mysql-quota-enforcer/sql/fakes"
	"github.com/pivotal-golang/lager/lagertest"
)

var _ = Describe("Database", func() {

	const dbName = "fake-db-name"
	var (
		logger   *lagertest.TestLogger
		database Database
		fakeDB   *sql.DB
	)

	BeforeEach(func() {
        var err error
		fakeDB, err = sqlmock.New()
		Expect(err).ToNot(HaveOccurred())

		logger = lagertest.NewTestLogger("ProxyRunner test")
		database = New(dbName, fakeDB, logger)
	})

	AfterEach(func() {
		err := fakeDB.Close()
		Expect(err).ToNot(HaveOccurred())
	})

	Describe("RevokePrivileges", func() {
		var fakeResult *sqlfakes.FakeResult

		BeforeEach(func() {
			fakeResult = &sqlfakes.FakeResult{}
			fakeResult.RowsAffectedReturns(1, nil)
		})

		It("makes a sql query to revoke priveledges on a database and then flushes privileges", func() {
			sqlmock.ExpectExec("UPDATE mysql.db SET Insert_priv = 'N', Update_priv = 'N', Create_priv = 'N' WHERE Db = ?").
                WithArgs(dbName).
                WillReturnResult(fakeResult)

			sqlmock.ExpectExec("FLUSH PRIVILEGES").
                WithArgs().
                WillReturnResult(&sqlfakes.FakeResult{})

			err := database.RevokePrivileges()
			Expect(err).ToNot(HaveOccurred())
		})

        Context("when the query fails", func() {
            BeforeEach(func() {
                sqlmock.ExpectExec("UPDATE mysql.db SET Insert_priv = 'N', Update_priv = 'N', Create_priv = 'N' WHERE Db = ?").
                    WithArgs(dbName).
                    WillReturnError(errors.New("fake-query-error"))
            })

            It("returns an error", func() {
                err := database.RevokePrivileges()
                Expect(err).To(HaveOccurred())
                Expect(err.Error()).To(ContainSubstring("fake-query-error"))
                Expect(err.Error()).To(ContainSubstring(dbName))
            })
        })

        Context("when getting the number of affected rows fails", func() {
            BeforeEach(func() {
                sqlmock.ExpectExec("UPDATE mysql.db SET Insert_priv = 'N', Update_priv = 'N', Create_priv = 'N' WHERE Db = ?").
                    WithArgs(dbName).
                    WillReturnResult(fakeResult)

                fakeResult.RowsAffectedReturns(0, errors.New("fake-rows-affected-error"))
            })

            It("returns an error", func() {
                err := database.RevokePrivileges()
                Expect(err).To(HaveOccurred())
                Expect(err.Error()).To(ContainSubstring("fake-rows-affected-error"))
                Expect(err.Error()).To(ContainSubstring("Getting rows affected"))
                Expect(err.Error()).To(ContainSubstring(dbName))

                Expect(fakeResult.RowsAffectedCallCount()).To(Equal(1))
            })
        })

        Context("when flushing privileges fails", func() {
            BeforeEach(func() {
                sqlmock.ExpectExec("UPDATE mysql.db SET Insert_priv = 'N', Update_priv = 'N', Create_priv = 'N' WHERE Db = ?").
                    WithArgs(dbName).
                    WillReturnResult(fakeResult)

                sqlmock.ExpectExec("FLUSH PRIVILEGES").
                    WithArgs().
                    WillReturnError(errors.New("fake-flush-error"))
            })

            It("returns an error", func() {
                err := database.RevokePrivileges()
                Expect(err).To(HaveOccurred())
                Expect(err.Error()).To(ContainSubstring("fake-flush-error"))
            })
        })
    })

    Describe("GrantPrivileges", func() {
        var fakeResult *sqlfakes.FakeResult

        BeforeEach(func() {
            fakeResult = &sqlfakes.FakeResult{}
            fakeResult.RowsAffectedReturns(1, nil)
        })

        It("grants priviledges to the database", func() {
            sqlmock.ExpectExec("UPDATE mysql.db SET Insert_priv = 'Y', Update_priv = 'Y', Create_priv = 'Y' WHERE Db = ?").
                WithArgs(dbName).
                WillReturnResult(fakeResult)

            sqlmock.ExpectExec("FLUSH PRIVILEGES").
                WithArgs().
                WillReturnResult(&sqlfakes.FakeResult{})

            err := database.GrantPrivileges()
            Expect(err).ToNot(HaveOccurred())
        })

        Context("when the query fails", func() {
            BeforeEach(func() {
                sqlmock.ExpectExec("UPDATE mysql.db SET Insert_priv = 'Y', Update_priv = 'Y', Create_priv = 'Y' WHERE Db = ?").
                    WithArgs(dbName).
                    WillReturnError(errors.New("fake-query-error"))
            })

            It("returns an error", func() {
                err := database.GrantPrivileges()
                Expect(err).To(HaveOccurred())
                Expect(err.Error()).To(ContainSubstring("fake-query-error"))
                Expect(err.Error()).To(ContainSubstring(dbName))
            })
        })

        Context("when getting the number of affected rows fails", func() {
            BeforeEach(func() {
                sqlmock.ExpectExec("UPDATE mysql.db SET Insert_priv = 'Y', Update_priv = 'Y', Create_priv = 'Y' WHERE Db = ?").
                    WithArgs(dbName).
                    WillReturnResult(fakeResult)

                fakeResult.RowsAffectedReturns(0, errors.New("fake-rows-affected-error"))
            })

            It("returns an error", func() {
                err := database.GrantPrivileges()
                Expect(err).To(HaveOccurred())
                Expect(err.Error()).To(ContainSubstring("fake-rows-affected-error"))
                Expect(err.Error()).To(ContainSubstring("Getting rows affected"))
                Expect(err.Error()).To(ContainSubstring(dbName))

                Expect(fakeResult.RowsAffectedCallCount()).To(Equal(1))
            })
        })

        Context("when flushing privileges fails", func() {
            BeforeEach(func() {
                sqlmock.ExpectExec("UPDATE mysql.db SET Insert_priv = 'Y', Update_priv = 'Y', Create_priv = 'Y' WHERE Db = ?").
                    WithArgs(dbName).
                    WillReturnResult(fakeResult)

                sqlmock.ExpectExec("FLUSH PRIVILEGES").
                    WithArgs().
                    WillReturnError(errors.New("fake-flush-error"))
            })

            It("returns an error", func() {
                err := database.GrantPrivileges()
                Expect(err).To(HaveOccurred())
                Expect(err.Error()).To(ContainSubstring("fake-flush-error"))
            })
        })

	})

	Describe("ResetActivePrivileges", func() {

	})
})
<|endoftext|>


<commit_before>// Copyright 2015 The go-fritzbox AUTHORS. All rights reserved.
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package fritzbox

import (
	"bytes"
	"crypto/md5"
	"encoding/binary"
	"encoding/xml"
	"errors"
	"fmt"
	"net/url"
	"time"
	"unicode/utf16"
)

// defaultSid is always invalid and
const (
	DefaultSid     = "0000000000000000"
	DefaultExpires = 10 * time.Minute
)

// Various errors the Session might return.
var (
	ErrInvalidCred = errors.New("fritzbox: invalid credentials")
	ErrExpiredSess = errors.New("fritzbox: session expired")
)

// Session represents a FRITZ!Box session
type Session struct {
	client *Client

	XMLName   xml.Name      `xml:"SessionInfo"`
	Sid       string        `xml:"SID"`
	Challenge string        `xml:"Challenge"`
	BlockTime time.Duration `xml:"BlockTime"`

	// Rights' representation is a little bit tricky
	// TODO: Write UnmarshalXML to merge them
	RightsName   []string `xml:"Rights>Name"`
	RightsAccess []int8   `xml:"Rights>Access"`

	// Session expires after 10 minutes
	Expires time.Time `xml:"-"`
}

// NewSession allocates a session
func newSession(c *Client) *Session {
	return &Session{
		client: c,
	}
}

// Open retrieves challenge from fritzbox
func (s *Session) Open() error {
	req, err := s.client.NewRequest("GET", "login_sid.lua", nil)
	if err != nil {
		return err
	}

	_, err = s.client.Do(req, s)
	if err != nil {
		return err
	}

	return nil
}

// Auth authenticates with a username and challenge-response
// It returns an error, if any.
func (s *Session) Auth(username, password string) error {
	cr, err := computeResponse(s.Challenge, password)
	if err != nil {
		return err
	}

	req, err := s.client.NewRequest("POST", "login_sid.lua", url.Values{
		"username": {username},
		"response": {cr},
	})
	if err != nil {
		return err
	}

	_, err = s.client.Do(req, s)
	if err != nil {
		return err
	}

	// Checks whether a login attempt was successfull or not
	if s.Sid == DefaultSid {
		return ErrInvalidCred
	}

	s.Refresh()
	return nil
}

// Close closes a session
func (s *Session) Close() {
	s.Sid = DefaultSid
}

// IsExpired returns true if s is expired
func (s *Session) IsExpired() bool {
	return s.Expires.Before(time.Now())
}

// Refresh updates expires
func (s *Session) Refresh() error {
	if s.IsExpired() && (s.Expires != time.Time{}) {
		s.Close()
		return ErrExpiredSess
	}
	s.Expires = time.Now().Add(DefaultExpires)
	return nil
}

// ComputeResponse generates a response for challenge-response auth
// with the given challenge and secret. It returns the reponse and
// and an error, if any.
func computeResponse(challenge, secret string) (string, error) {
	buf := new(bytes.Buffer)

	chars := utf16.Encode([]rune(fmt.Sprintf("%s-%s", challenge, secret)))

	for _, char := range chars {
		// According to AVM's technical notes: unicode codepoints
		// above 255 needs to be converted to "." (0x2e 0x00 in UTF-16LE)
		if char > 255 {
			char = 0x2e
		}

		err := binary.Write(buf, binary.LittleEndian, char)
		if err != nil {
			return "", err
		}
	}

	hash := md5.Sum(buf.Bytes())
	r := fmt.Sprintf("%s-%x", challenge, hash)

	return r, nil
}
<commit_msg>Supports now go version 1.1 or greater<commit_after>// Copyright 2015 The go-fritzbox AUTHORS. All rights reserved.
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package fritzbox

import (
	"bytes"
	"crypto/md5"
	"encoding/binary"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"net/url"
	"time"
	"unicode/utf16"
)

// defaultSid is always invalid and
const (
	DefaultSid     = "0000000000000000"
	DefaultExpires = 10 * time.Minute
)

// Various errors the Session might return.
var (
	ErrInvalidCred = errors.New("fritzbox: invalid credentials")
	ErrExpiredSess = errors.New("fritzbox: session expired")
)

// Session represents a FRITZ!Box session
type Session struct {
	client *Client

	XMLName   xml.Name      `xml:"SessionInfo"`
	Sid       string        `xml:"SID"`
	Challenge string        `xml:"Challenge"`
	BlockTime time.Duration `xml:"BlockTime"`

	// Rights' representation is a little bit tricky
	// TODO: Write UnmarshalXML to merge them
	RightsName   []string `xml:"Rights>Name"`
	RightsAccess []int8   `xml:"Rights>Access"`

	// Session expires after 10 minutes
	Expires time.Time `xml:"-"`
}

// NewSession allocates a session
func newSession(c *Client) *Session {
	return &Session{
		client: c,
	}
}

// Open retrieves challenge from fritzbox
func (s *Session) Open() error {
	req, err := s.client.NewRequest("GET", "login_sid.lua", nil)
	if err != nil {
		return err
	}

	_, err = s.client.Do(req, s)
	if err != nil {
		return err
	}

	return nil
}

// Auth authenticates with a username and challenge-response
// It returns an error, if any.
func (s *Session) Auth(username, password string) error {
	cr, err := computeResponse(s.Challenge, password)
	if err != nil {
		return err
	}

	req, err := s.client.NewRequest("POST", "login_sid.lua", url.Values{
		"username": {username},
		"response": {cr},
	})
	if err != nil {
		return err
	}

	_, err = s.client.Do(req, s)
	if err != nil {
		return err
	}

	// Checks whether a login attempt was successfull or not
	if s.Sid == DefaultSid {
		return ErrInvalidCred
	}

	s.Refresh()
	return nil
}

// Close closes a session
func (s *Session) Close() {
	s.Sid = DefaultSid
}

// IsExpired returns true if s is expired
func (s *Session) IsExpired() bool {
	return s.Expires.Before(time.Now())
}

// Refresh updates expires
func (s *Session) Refresh() error {
	if s.IsExpired() && (s.Expires != time.Time{}) {
		s.Close()
		return ErrExpiredSess
	}
	s.Expires = time.Now().Add(DefaultExpires)
	return nil
}

// ComputeResponse generates a response for challenge-response auth
// with the given challenge and secret. It returns the reponse and
// and an error, if any.
func computeResponse(challenge, secret string) (string, error) {
	buf := new(bytes.Buffer)
	h := md5.New()

	chars := utf16.Encode([]rune(fmt.Sprintf("%s-%s", challenge, secret)))

	for _, char := range chars {
		// According to AVM's technical notes: unicode codepoints
		// above 255 needs to be converted to "." (0x2e 0x00 in UTF-16LE)
		if char > 255 {
			char = 0x2e
		}

		err := binary.Write(buf, binary.LittleEndian, char)
		if err != nil {
			return "", err
		}
	}

	io.Copy(h, buf)
	r := fmt.Sprintf("%s-%x", challenge, h.Sum(nil))

	return r, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"time"

	log "github.com/Sirupsen/logrus"
	docker "github.com/smashwilson/go-dockerclient"
)

// OutputCollector is an io.Writer that accumulates output from a specified stream in an attached
// Docker container and appends it to the appropriate field within a SubmittedJob.
type OutputCollector struct {
	context  *Context
	job      *SubmittedJob
	isStdout bool
}

// Write appends bytes to the selected stream and updates the SubmittedJob.
func (c OutputCollector) Write(p []byte) (int, error) {
	var stream string
	if c.isStdout {
		stream = "stdout"
	} else {
		stream = "stderr"
	}
	log.WithFields(log.Fields{
		"length": len(p),
		"bytes":  string(p),
		"stream": stream,
	}).Debug("Received output from a job")

	if c.isStdout {
		c.job.Stdout += string(p)
	} else {
		c.job.Stderr += string(p)
	}

	if err := c.context.UpdateJob(c.job); err != nil {
		return 0, err
	}

	return len(p), nil
}

// Runner is the main entry point for the job runner goroutine.
func Runner(c *Context) {
	var client *docker.Client
	var err error

	if c.DockerTLS {
		client, err = docker.NewTLSClient(c.DockerHost, c.DockerCert, c.DockerKey, c.DockerCACert)
		if err != nil {
			log.WithFields(log.Fields{
				"docker host":    c.DockerHost,
				"docker cert":    c.DockerCert,
				"docker key":     c.DockerKey,
				"docker CA cert": c.DockerCACert,
			}).Fatal("Unable to connect to Docker with TLS.")
			return
		}
	} else {
		client, err = docker.NewClient(c.DockerHost)
		if err != nil {
			log.WithFields(log.Fields{
				"docker host": c.DockerHost,
				"error":       err,
			}).Fatal("Unable to connect to Docker.")
			return
		}
	}

	for {
		select {
		case <-time.After(time.Duration(c.Poll) * time.Millisecond):
			Claim(c, client)
		}
	}
}

// Claim acquires the oldest single pending job and launches a goroutine to execute its command in
// a new container.
func Claim(c *Context, client *docker.Client) {
	job, err := c.ClaimJob()
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Error("Unable to claim a job.")
		return
	}
	if job == nil {
		// Nothing to claim.
		return
	}

	log.WithFields(log.Fields{
		"jid": job.JID,
	}).Info("Launching a new job.")

	go Execute(c, client, job)
}

// Execute launches a container to process the submitted job. It passes any provided stdin data
// to the container and consumes stdout and stderr, updating Mongo as it runs. Once completed, it
// acquires the job's result from its configured source and marks the job as finished.
func Execute(c *Context, client *docker.Client, job *SubmittedJob) {
	job.StartedAt = StoreTime(time.Now())
	if err := c.UpdateJob(job); err != nil {
		log.WithFields(log.Fields{
			"jid":     job.JID,
			"account": job.Account,
			"error":   err,
		}).Error("Unable to update the job's start timestamp.")
		return
	}

	container, err := client.CreateContainer(docker.CreateContainerOptions{
		Name: job.ContainerName(),
		Config: &docker.Config{
			Image:     c.Image,
			Cmd:       []string{"/bin/bash", "-c", job.Command},
			OpenStdin: true,
			StdinOnce: true,
		},
	})
	if err != nil {
		log.WithFields(log.Fields{
			"jid":     job.JID,
			"account": job.Account,
			"error":   err,
		}).Error("Unable to create the job's container.")
		return
	}
	log.WithFields(log.Fields{
		"jid":            job.JID,
		"account":        job.Account,
		"container id":   container.ID,
		"container name": container.Name,
	}).Debug("Container created successfully.")

	// Prepare the input and output streams.
	stdin := bytes.NewReader(job.Stdin)
	stdout := OutputCollector{
		context:  c,
		job:      job,
		isStdout: true,
	}
	stderr := OutputCollector{
		context:  c,
		job:      job,
		isStdout: false,
	}

	go func() {
		log.Debug("About to attach.")
		err = client.AttachToContainer(docker.AttachToContainerOptions{
			Container:    container.ID,
			Stream:       true,
			InputStream:  stdin,
			OutputStream: stdout,
			ErrorStream:  stderr,
			Stdin:        true,
			Stdout:       true,
			Stderr:       true,
		})
		if err != nil {
			log.WithFields(log.Fields{
				"jid":            job.JID,
				"account":        job.Account,
				"container id":   container.ID,
				"container name": container.Name,
				"error":          err,
			}).Error("Unable to attach to the job's container.")
			return
		}
		log.WithFields(log.Fields{
			"jid":            job.JID,
			"account":        job.Account,
			"container id":   container.ID,
			"container name": container.Name,
		}).Debug("Attached to the job's container.")
	}()

	// Start the created container.
	if err := client.StartContainer(container.ID, &docker.HostConfig{}); err != nil {
		log.WithFields(log.Fields{
			"jid":            job.JID,
			"account":        job.Account,
			"container id":   container.ID,
			"container name": container.Name,
			"error":          err,
		}).Error("Unable to start the job's container.")
		return
	}
	log.WithFields(log.Fields{
		"jid":            job.JID,
		"account":        job.Account,
		"container id":   container.ID,
		"container name": container.Name,
	}).Debug("Container started successfully.")

	log.Debug("Waiting for job completion.")
	status, err := client.WaitContainer(container.ID)
	if err != nil {
		log.WithFields(log.Fields{
			"jid":            job.JID,
			"account":        job.Account,
			"container id":   container.ID,
			"container name": container.Name,
			"error":          err,
		}).Error("Unable to wait for the container to terminate.")
		return
	}

	job.FinishedAt = StoreTime(time.Now())
	if status == 0 {
		// Successful termination.
		job.Status = StatusDone
	} else {
		// Something went wrong.
		job.Status = StatusError
	}

	if err := c.UpdateJob(job); err != nil {
		log.WithFields(log.Fields{
			"jid":   job.JID,
			"error": err,
		}).Error("Unable to update job status.")
		return
	}

	log.WithFields(log.Fields{
		"jid": job.JID,
	}).Info("Job complete.")
}
<commit_msg>Remove containers when you're done with them.<commit_after>package main

import (
	"bytes"
	"time"

	log "github.com/Sirupsen/logrus"
	docker "github.com/smashwilson/go-dockerclient"
)

// OutputCollector is an io.Writer that accumulates output from a specified stream in an attached
// Docker container and appends it to the appropriate field within a SubmittedJob.
type OutputCollector struct {
	context  *Context
	job      *SubmittedJob
	isStdout bool
}

// Write appends bytes to the selected stream and updates the SubmittedJob.
func (c OutputCollector) Write(p []byte) (int, error) {
	var stream string
	if c.isStdout {
		stream = "stdout"
	} else {
		stream = "stderr"
	}
	log.WithFields(log.Fields{
		"length": len(p),
		"bytes":  string(p),
		"stream": stream,
	}).Debug("Received output from a job")

	if c.isStdout {
		c.job.Stdout += string(p)
	} else {
		c.job.Stderr += string(p)
	}

	if err := c.context.UpdateJob(c.job); err != nil {
		return 0, err
	}

	return len(p), nil
}

// Runner is the main entry point for the job runner goroutine.
func Runner(c *Context) {
	var client *docker.Client
	var err error

	if c.DockerTLS {
		client, err = docker.NewTLSClient(c.DockerHost, c.DockerCert, c.DockerKey, c.DockerCACert)
		if err != nil {
			log.WithFields(log.Fields{
				"docker host":    c.DockerHost,
				"docker cert":    c.DockerCert,
				"docker key":     c.DockerKey,
				"docker CA cert": c.DockerCACert,
			}).Fatal("Unable to connect to Docker with TLS.")
			return
		}
	} else {
		client, err = docker.NewClient(c.DockerHost)
		if err != nil {
			log.WithFields(log.Fields{
				"docker host": c.DockerHost,
				"error":       err,
			}).Fatal("Unable to connect to Docker.")
			return
		}
	}

	for {
		select {
		case <-time.After(time.Duration(c.Poll) * time.Millisecond):
			Claim(c, client)
		}
	}
}

// Claim acquires the oldest single pending job and launches a goroutine to execute its command in
// a new container.
func Claim(c *Context, client *docker.Client) {
	job, err := c.ClaimJob()
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Error("Unable to claim a job.")
		return
	}
	if job == nil {
		// Nothing to claim.
		return
	}

	log.WithFields(log.Fields{
		"jid": job.JID,
	}).Info("Launching a new job.")

	go Execute(c, client, job)
}

// Execute launches a container to process the submitted job. It passes any provided stdin data
// to the container and consumes stdout and stderr, updating Mongo as it runs. Once completed, it
// acquires the job's result from its configured source and marks the job as finished.
func Execute(c *Context, client *docker.Client, job *SubmittedJob) {
	job.StartedAt = StoreTime(time.Now())
	if err := c.UpdateJob(job); err != nil {
		log.WithFields(log.Fields{
			"jid":     job.JID,
			"account": job.Account,
			"error":   err,
		}).Error("Unable to update the job's start timestamp.")
		return
	}

	container, err := client.CreateContainer(docker.CreateContainerOptions{
		Name: job.ContainerName(),
		Config: &docker.Config{
			Image:     c.Image,
			Cmd:       []string{"/bin/bash", "-c", job.Command},
			OpenStdin: true,
			StdinOnce: true,
		},
	})
	if err != nil {
		log.WithFields(log.Fields{
			"jid":     job.JID,
			"account": job.Account,
			"error":   err,
		}).Error("Unable to create the job's container.")
		return
	}
	log.WithFields(log.Fields{
		"jid":            job.JID,
		"account":        job.Account,
		"container id":   container.ID,
		"container name": container.Name,
	}).Debug("Container created successfully.")

	// Prepare the input and output streams.
	stdin := bytes.NewReader(job.Stdin)
	stdout := OutputCollector{
		context:  c,
		job:      job,
		isStdout: true,
	}
	stderr := OutputCollector{
		context:  c,
		job:      job,
		isStdout: false,
	}

	go func() {
		log.Debug("About to attach.")
		err = client.AttachToContainer(docker.AttachToContainerOptions{
			Container:    container.ID,
			Stream:       true,
			InputStream:  stdin,
			OutputStream: stdout,
			ErrorStream:  stderr,
			Stdin:        true,
			Stdout:       true,
			Stderr:       true,
		})
		if err != nil {
			log.WithFields(log.Fields{
				"jid":            job.JID,
				"account":        job.Account,
				"container id":   container.ID,
				"container name": container.Name,
				"error":          err,
			}).Error("Unable to attach to the job's container.")
			return
		}
		log.WithFields(log.Fields{
			"jid":            job.JID,
			"account":        job.Account,
			"container id":   container.ID,
			"container name": container.Name,
		}).Debug("Attached to the job's container.")
	}()

	// Start the created container.
	if err := client.StartContainer(container.ID, &docker.HostConfig{}); err != nil {
		log.WithFields(log.Fields{
			"jid":            job.JID,
			"account":        job.Account,
			"container id":   container.ID,
			"container name": container.Name,
			"error":          err,
		}).Error("Unable to start the job's container.")
		return
	}
	log.WithFields(log.Fields{
		"jid":            job.JID,
		"account":        job.Account,
		"container id":   container.ID,
		"container name": container.Name,
	}).Debug("Container started successfully.")

	log.Debug("Waiting for job completion.")
	status, err := client.WaitContainer(container.ID)
	if err != nil {
		log.WithFields(log.Fields{
			"jid":            job.JID,
			"account":        job.Account,
			"container id":   container.ID,
			"container name": container.Name,
			"error":          err,
		}).Error("Unable to wait for the container to terminate.")
		return
	}

	log.Debug("Removing container.")
	if err := client.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID}); err != nil {
		log.WithFields(log.Fields{
			"jid":            job.JID,
			"account":        job.Account,
			"container id":   container.ID,
			"container name": container.Name,
			"error":          err,
		}).Error("Unable to remove the container.")
	}

	job.FinishedAt = StoreTime(time.Now())
	if status == 0 {
		// Successful termination.
		job.Status = StatusDone
	} else {
		// Something went wrong.
		job.Status = StatusError
	}

	if err := c.UpdateJob(job); err != nil {
		log.WithFields(log.Fields{
			"jid":   job.JID,
			"error": err,
		}).Error("Unable to update job status.")
		return
	}

	log.WithFields(log.Fields{
		"jid": job.JID,
	}).Info("Job complete.")
}
<|endoftext|>


<commit_before>package lib

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"strings"
)

type Action interface {
	getName() string
	getHint() string
	firstUp() Action
	getQuestions() Questions
	nextQuestion() *Question
	askQuestion()
}

const (
	start_action, help_action = "/start", "/help"

	say_action      = "/say"
	say_action_hint = "/say <message>"

	chat_action = "/chat"

	review_action      = "/review"
	review_action_hint = "/review <days>"

	remind_action      = "/remind"
	remind_action_hint = "/remind in <days> to <message>"
)

func NewAction(in *Incoming, s *session) Action {

	if in.getCmd() == say_action {
		return SayAction{in: in, s: s}
	} else if in.getCmd() == remind_action {
		return &RemindAction{in: in, s: s}
	} else if in.getCmd() == review_action {
		return &ReviewAction{in: in, s: s}
	} else if in.getCmd() == chat_action {
		return &ChatAction{in: in, s: s}
	} else if in.isSticker() {
		return &StickerChatAction{in: in, s: s}
	}
	log.Println("asked ", in.getCmd())
	return &HelpAction{in: in, s: s}
}

func load(name string, q interface{}) {

	if strings.Contains(name, "/") {
		name = strings.Split(name, "/")[1]
	}

	file, err := os.Open(fmt.Sprintf("./lib/config/%s.json", name))
	if err != nil {
		log.Panic("could not load QandA language file ", err.Error())
	}

	err = json.NewDecoder(file).Decode(&q)
	if err != nil {
		log.Panic("could not decode QandA ", err.Error())
	}

	return
}

type SayAction struct {
	s  *session
	in *Incoming
}

func (a SayAction) getName() string {
	return say_action
}
func (a SayAction) getHint() string {
	return say_action_hint
}
func (a SayAction) firstUp() Action {
	a.s.save(a.in.getNote())
	return a
}

func (a SayAction) nextQuestion() *Question {

	q := a.getQuestions()

	if a.in.isCmd() {
		return q.First()
	}

	next, save := q.next(a.in.msg.Text)
	if save {
		a.s.save(a.in.getNote(a.getName(), next.RelatesTo.SaveTag))
	}
	return next
}

func (a SayAction) askQuestion() {
	q := a.nextQuestion()
	a.s.send(q.Context...)
	a.s.sendWithKeyboard(q.QuestionText, q.makeKeyboard())
}

func (a SayAction) getQuestions() Questions {

	var q struct {
		Questions `json:"QandA"`
	}

	if a.in.hasSubmisson() {
		load(default_script, &q)
		return q.Questions
	}
	load(a.getName(), &q)
	return q.Questions
}

type HelpAction struct {
	s  *session
	in *Incoming
	q  struct {
		Questions `json:"QandA"`
	}
}

func (a HelpAction) getName() string {
	return say_action
}
func (a HelpAction) getHint() string {
	return say_action_hint
}
func (a HelpAction) firstUp() Action {
	log.Println("help first up")
	helpInfo := fmt.Sprintf("%s %s %s %s ",
		fmt.Sprintf("Hey %s! We can't doFirst it all but we can:\n\n", a.in.sender().Username),
		chat_action+" - to have a *quick chat* about what your up-to \n\n",
		say_action_hint+" - to say *anything* thats on your mind \n\n",
		review_action_hint+" - to review what has been happening \n\n",
	)
	a.s.send(helpInfo)
	return a
}

func (a HelpAction) nextQuestion() *Question {
	return nil
}

func (a HelpAction) getQuestions() Questions {
	return nil
}

func (a HelpAction) askQuestion() {
	return
}

type ChatAction struct {
	s  *session
	in *Incoming
}

func (a ChatAction) getName() string {
	return chat_action
}
func (a ChatAction) getHint() string {
	return ""
}
func (a ChatAction) firstUp() Action {
	//nothing to doFirst
	return a
}

func (a ChatAction) getQuestions() Questions {

	var q struct {
		Questions `json:"QandA"`
	}

	load(a.getName(), &q)
	return q.Questions
}

func (a ChatAction) nextQuestion() *Question {

	q := a.getQuestions()

	if a.in.isCmd() {
		return q.First()
	}
	next, save := q.next(a.in.msg.Text)
	if save {
		a.s.save(a.in.getNote(a.getName(), next.RelatesTo.SaveTag))
	}
	return next
}

func (a ChatAction) askQuestion() {
	q := a.nextQuestion()
	a.s.send(q.Context...)
	a.s.sendWithKeyboard(q.QuestionText, q.makeKeyboard())
	return
}

type ReviewAction struct {
	s  *session
	in *Incoming
	q  struct {
		Questions `json:"QandA"`
	}
}

func (a ReviewAction) getName() string {
	return review_action
}
func (a ReviewAction) getHint() string {
	return review_action_hint
}
func (a ReviewAction) firstUp() Action {
	log.Println("doFirsting review ...")
	n := a.s.getNotes(a.in.msg)

	saidTxt := fmt.Sprintf("%s you said: \n\n %s", a.in.sender().Username, n.FilterBy(said_tag).ToString())
	a.s.send(saidTxt)
	talkedTxt := fmt.Sprintf("%s we talked about: \n\n %s", a.in.sender().Username, n.FilterBy(chat_tag).ToString())
	a.s.send(talkedTxt)
	return a
}

func (a ReviewAction) getQuestions() Questions {

	var q struct {
		Questions `json:"QandA"`
	}

	load(a.getName(), &q)

	return q.Questions
}

func (a ReviewAction) nextQuestion() *Question {

	q := a.getQuestions()

	if a.in.isCmd() {
		return q.First()
	}
	next, save := q.next(a.in.msg.Text)
	if save {
		a.s.save(a.in.getNote(a.getName(), next.RelatesTo.SaveTag))
	}
	return next
}

func (a ReviewAction) askQuestion() {
	q := a.nextQuestion()
	a.s.send(q.Context...)
	a.s.sendWithKeyboard(q.QuestionText, q.makeKeyboard())
	return
}

type RemindAction struct {
	s  *session
	in *Incoming
}

func (a RemindAction) getName() string {
	return remind_action
}
func (a RemindAction) getHint() string {
	return remind_action_hint
}
func (a RemindAction) firstUp() Action {
	log.Println("remind me ...")
	a.s.save(a.in.getNote())
	return a
}

func (a RemindAction) getQuestions() Questions {

	var q struct {
		Questions `json:"QandA"`
	}

	if a.in.hasSubmisson() {
		load(default_script, &q)
		return q.Questions
	}
	load(a.getName(), &q)

	return q.Questions
}

func (a RemindAction) nextQuestion() *Question {

	q := a.getQuestions()

	if a.in.isCmd() {
		return q.First()
	}
	next, save := q.next(a.in.msg.Text)
	if save {
		a.s.save(a.in.getNote(a.getName(), next.RelatesTo.SaveTag))
	}
	return next
}

func (a RemindAction) askQuestion() {
	q := a.nextQuestion()
	a.s.send(q.Context...)
	a.s.sendWithKeyboard(q.QuestionText, q.makeKeyboard())
	return
}

type StickerChatAction struct {
	s        *session
	in       *Incoming
	stickers Stickers
}

func (a StickerChatAction) getName() string {
	return chat_action
}
func (a StickerChatAction) getHint() string {
	return ""
}

func (a StickerChatAction) firstUp() Action {
	a.stickers = LoadKnownStickers()
	return a
}

func (a StickerChatAction) getQuestions() Questions {
	var q struct {
		Questions `json:"QandA"`
	}
	load(a.getName(), &q)
	return q.Questions
}

func (a StickerChatAction) nextQuestion() *Question {

	q := a.getQuestions()

	if a.in.isSticker() {

		sticker := a.stickers.FindSticker(a.in.msg.Sticker.FileID)
		a.in.msg.Text = sticker.Meaning
		next, save := q.nextFrom(sticker.Ids...)
		if save {
			a.s.save(a.in.getNote(a.getName(), next.RelatesTo.SaveTag))
		}
		return next
	}
	next, save := q.next(a.in.msg.Text)
	if save {
		a.s.save(a.in.getNote(a.getName(), next.RelatesTo.SaveTag))
	}
	return next
}

func (a StickerChatAction) askQuestion() {
	q := a.nextQuestion()
	a.s.send(q.Context...)
	a.s.sendWithKeyboard(q.QuestionText, q.makeKeyboard())
	return
}
<commit_msg>set the session user<commit_after>package lib

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"strings"
)

type Action interface {
	getName() string
	getHint() string
	firstUp() Action
	getQuestions() Questions
	nextQuestion() *Question
	askQuestion()
}

const (
	start_action, help_action = "/start", "/help"

	say_action      = "/say"
	say_action_hint = "/say <message>"

	chat_action = "/chat"

	review_action      = "/review"
	review_action_hint = "/review <days>"

	remind_action      = "/remind"
	remind_action_hint = "/remind in <days> to <message>"
)

func NewAction(in *Incoming, s *session) Action {

	s.User = in.msg.Sender

	if in.getCmd() == say_action {
		return SayAction{in: in, s: s}
	} else if in.getCmd() == remind_action {
		return &RemindAction{in: in, s: s}
	} else if in.getCmd() == review_action {
		return &ReviewAction{in: in, s: s}
	} else if in.getCmd() == chat_action {
		return &ChatAction{in: in, s: s}
	} else if in.isSticker() {
		return &StickerChatAction{in: in, s: s}
	}
	log.Println("asked ", in.getCmd())
	return &HelpAction{in: in, s: s}
}

func load(name string, q interface{}) {

	if strings.Contains(name, "/") {
		name = strings.Split(name, "/")[1]
	}

	file, err := os.Open(fmt.Sprintf("./lib/config/%s.json", name))
	if err != nil {
		log.Panic("could not load QandA language file ", err.Error())
	}

	err = json.NewDecoder(file).Decode(&q)
	if err != nil {
		log.Panic("could not decode QandA ", err.Error())
	}

	return
}

type SayAction struct {
	s  *session
	in *Incoming
}

func (a SayAction) getName() string {
	return say_action
}
func (a SayAction) getHint() string {
	return say_action_hint
}
func (a SayAction) firstUp() Action {
	a.s.save(a.in.getNote())
	return a
}

func (a SayAction) nextQuestion() *Question {

	q := a.getQuestions()

	if a.in.isCmd() {
		return q.First()
	}

	next, save := q.next(a.in.msg.Text)
	if save {
		a.s.save(a.in.getNote(a.getName(), next.RelatesTo.SaveTag))
	}
	return next
}

func (a SayAction) askQuestion() {
	q := a.nextQuestion()
	a.s.send(q.Context...)
	a.s.sendWithKeyboard(q.QuestionText, q.makeKeyboard())
}

func (a SayAction) getQuestions() Questions {

	var q struct {
		Questions `json:"QandA"`
	}

	if a.in.hasSubmisson() {
		load(default_script, &q)
		return q.Questions
	}
	load(a.getName(), &q)
	return q.Questions
}

type HelpAction struct {
	s  *session
	in *Incoming
	q  struct {
		Questions `json:"QandA"`
	}
}

func (a HelpAction) getName() string {
	return say_action
}
func (a HelpAction) getHint() string {
	return say_action_hint
}
func (a HelpAction) firstUp() Action {
	log.Println("help first up")
	helpInfo := fmt.Sprintf("%s %s %s %s ",
		fmt.Sprintf("Hey %s! We can't doFirst it all but we can:\n\n", a.in.sender().Username),
		chat_action+" - to have a *quick chat* about what your up-to \n\n",
		say_action_hint+" - to say *anything* thats on your mind \n\n",
		review_action_hint+" - to review what has been happening \n\n",
	)
	a.s.send(helpInfo)
	return a
}

func (a HelpAction) nextQuestion() *Question {
	return nil
}

func (a HelpAction) getQuestions() Questions {
	return nil
}

func (a HelpAction) askQuestion() {
	return
}

type ChatAction struct {
	s  *session
	in *Incoming
}

func (a ChatAction) getName() string {
	return chat_action
}
func (a ChatAction) getHint() string {
	return ""
}
func (a ChatAction) firstUp() Action {
	//nothing to doFirst
	return a
}

func (a ChatAction) getQuestions() Questions {

	var q struct {
		Questions `json:"QandA"`
	}

	load(a.getName(), &q)
	return q.Questions
}

func (a ChatAction) nextQuestion() *Question {

	q := a.getQuestions()

	if a.in.isCmd() {
		return q.First()
	}
	next, save := q.next(a.in.msg.Text)
	if save {
		a.s.save(a.in.getNote(a.getName(), next.RelatesTo.SaveTag))
	}
	return next
}

func (a ChatAction) askQuestion() {
	q := a.nextQuestion()
	a.s.send(q.Context...)
	a.s.sendWithKeyboard(q.QuestionText, q.makeKeyboard())
	return
}

type ReviewAction struct {
	s  *session
	in *Incoming
	q  struct {
		Questions `json:"QandA"`
	}
}

func (a ReviewAction) getName() string {
	return review_action
}
func (a ReviewAction) getHint() string {
	return review_action_hint
}
func (a ReviewAction) firstUp() Action {
	log.Println("doFirsting review ...")
	n := a.s.getNotes(a.in.msg)

	saidTxt := fmt.Sprintf("%s you said: \n\n %s", a.in.sender().Username, n.FilterBy(said_tag).ToString())
	a.s.send(saidTxt)
	talkedTxt := fmt.Sprintf("%s we talked about: \n\n %s", a.in.sender().Username, n.FilterBy(chat_tag).ToString())
	a.s.send(talkedTxt)
	return a
}

func (a ReviewAction) getQuestions() Questions {

	var q struct {
		Questions `json:"QandA"`
	}

	load(a.getName(), &q)

	return q.Questions
}

func (a ReviewAction) nextQuestion() *Question {

	q := a.getQuestions()

	if a.in.isCmd() {
		return q.First()
	}
	next, save := q.next(a.in.msg.Text)
	if save {
		a.s.save(a.in.getNote(a.getName(), next.RelatesTo.SaveTag))
	}
	return next
}

func (a ReviewAction) askQuestion() {
	q := a.nextQuestion()
	a.s.send(q.Context...)
	a.s.sendWithKeyboard(q.QuestionText, q.makeKeyboard())
	return
}

type RemindAction struct {
	s  *session
	in *Incoming
}

func (a RemindAction) getName() string {
	return remind_action
}
func (a RemindAction) getHint() string {
	return remind_action_hint
}
func (a RemindAction) firstUp() Action {
	log.Println("remind me ...")
	a.s.save(a.in.getNote())
	return a
}

func (a RemindAction) getQuestions() Questions {

	var q struct {
		Questions `json:"QandA"`
	}

	if a.in.hasSubmisson() {
		load(default_script, &q)
		return q.Questions
	}
	load(a.getName(), &q)

	return q.Questions
}

func (a RemindAction) nextQuestion() *Question {

	q := a.getQuestions()

	if a.in.isCmd() {
		return q.First()
	}
	next, save := q.next(a.in.msg.Text)
	if save {
		a.s.save(a.in.getNote(a.getName(), next.RelatesTo.SaveTag))
	}
	return next
}

func (a RemindAction) askQuestion() {
	q := a.nextQuestion()
	a.s.send(q.Context...)
	a.s.sendWithKeyboard(q.QuestionText, q.makeKeyboard())
	return
}

type StickerChatAction struct {
	s        *session
	in       *Incoming
	stickers Stickers
}

func (a StickerChatAction) getName() string {
	return chat_action
}
func (a StickerChatAction) getHint() string {
	return ""
}

func (a StickerChatAction) firstUp() Action {
	a.stickers = LoadKnownStickers()
	return a
}

func (a StickerChatAction) getQuestions() Questions {
	var q struct {
		Questions `json:"QandA"`
	}
	load(a.getName(), &q)
	return q.Questions
}

func (a StickerChatAction) nextQuestion() *Question {

	q := a.getQuestions()

	if a.in.isSticker() {

		sticker := a.stickers.FindSticker(a.in.msg.Sticker.FileID)
		a.in.msg.Text = sticker.Meaning
		next, save := q.nextFrom(sticker.Ids...)
		if save {
			a.s.save(a.in.getNote(a.getName(), next.RelatesTo.SaveTag))
		}
		return next
	}
	next, save := q.next(a.in.msg.Text)
	if save {
		a.s.save(a.in.getNote(a.getName(), next.RelatesTo.SaveTag))
	}
	return next
}

func (a StickerChatAction) askQuestion() {
	q := a.nextQuestion()
	a.s.send(q.Context...)
	a.s.sendWithKeyboard(q.QuestionText, q.makeKeyboard())
	return
}
<|endoftext|>


<commit_before>package lib

import (
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
	"sync"
	"strconv"

	"github.com/matryer/runner"
	"github.com/xiwenc/go-berlin/utils"
	"io/ioutil"
)

type FileEntry struct {
	Path     string
	Checksum string
	Content	 []byte
}

type Status struct {
	Health	 string
}

var task *runner.Task
var cmd *exec.Cmd
var lock = sync.RWMutex{}

func RestartApp(backendRunCommand string) Status {
	log.Println("Restarting backend")
	parts := strings.Fields(backendRunCommand)
	head := parts[0]
	parts = parts[1:len(parts)]
	lock.RLock()

	if task != nil {
		task.Stop()
		cmd.Wait()
	}
	cmd = exec.Command(head, parts...)

	task = runner.Go(func(shouldStop runner.S) error {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Start()

		for {
			if shouldStop() {
				cmd.Process.Signal(syscall.SIGTERM)
				break
			}
		}
		return nil
	})
	lock.RUnlock()
	return Status{Health: "Restarting"}
}

func ListFiles() []*FileEntry {
	var store = []*FileEntry{}
	dir := "./"

	err := filepath.Walk(dir, func(path string, f os.FileInfo, err error) error {
		if f.IsDir() {
			return nil
		}
		fileEntry := FileEntry{}
		fileEntry.Path = path
		checksum, _ := utils.ChecksumsForFile(path)
		fileEntry.Checksum = checksum.SHA256
		store = append(store, &fileEntry)
		return nil
	})
	if err != nil {
		log.Println(err)
	}
	return store
}

func GetStatus() Status {
	status := Status{}

	if cmd.Process.Pid > 0 {
		status.Health = "Running"
	} else {
		status.Health = "Not-Running"
	}
	return status
}

func UploadFiles(files []FileEntry) Status {
	status := Status{}
	failed := 0
	updated := 0
	for _, fileEntry := range files {
		log.Println("Updating file: " + fileEntry.Path)
		err := ioutil.WriteFile(fileEntry.Path, fileEntry.Content, 0644)
		if err != nil {
			log.Println(err)
			failed++
		} else {
			updated++
		}
	}

	if failed > 0 {
		status.Health = "Failed to update " + strconv.Itoa(failed) + " files"
	} else {
		status.Health = "Updated " + strconv.Itoa(updated) + " files"
	}
	return status
}<commit_msg>auto-restart backend based on regex matches<commit_after>package lib

import (
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
	"sync"
	"strconv"
	"io/ioutil"
	"regexp"

	"github.com/matryer/runner"
	"github.com/xiwenc/go-berlin/utils"
)

type FileEntry struct {
	Path     string
	Checksum string
	Content	 []byte
}

type Status struct {
	Health	 string
}

var task *runner.Task
var cmd *exec.Cmd
var lock = sync.RWMutex{}
var cmdRaw = ""

const (
	ENV_RESTART_REGEX = "BERLIN_RESTART_REGEX"
	ENV_IGNORE_REGEX = "BERLIN_IGNORE_REGEX"
)

func RestartApp(backendRunCommand string) Status {
	log.Println("Restarting backend")

	if len(backendRunCommand) > 0 {
		cmdRaw = backendRunCommand
	} else {
		backendRunCommand = cmdRaw
	}

	parts := strings.Fields(backendRunCommand)
	head := parts[0]
	parts = parts[1:len(parts)]
	lock.RLock()

	if task != nil {
		task.Stop()
		cmd.Wait()
	}
	cmd = exec.Command(head, parts...)

	task = runner.Go(func(shouldStop runner.S) error {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Start()

		for {
			if shouldStop() {
				cmd.Process.Signal(syscall.SIGTERM)
				break
			}
		}
		return nil
	})
	lock.RUnlock()
	return Status{Health: "Restarting"}
}

func ListFiles() []*FileEntry {
	var store = []*FileEntry{}
	dir := "./"

	err := filepath.Walk(dir, func(path string, f os.FileInfo, err error) error {
		if f.IsDir() {
			return nil
		}
		fileEntry := FileEntry{}
		fileEntry.Path = path
		checksum, _ := utils.ChecksumsForFile(path)
		fileEntry.Checksum = checksum.SHA256
		store = append(store, &fileEntry)
		return nil
	})
	if err != nil {
		log.Println(err)
	}
	return store
}

func GetStatus() Status {
	status := Status{}

	if cmd.Process.Pid > 0 {
		status.Health = "Running"
	} else {
		status.Health = "Not-Running"
	}
	return status
}

func UploadFiles(files []FileEntry) Status {
	status := Status{}
	failed := 0
	updated := 0
	restart := false
	for _, fileEntry := range files {
		log.Println("Updating file: " + fileEntry.Path)
		err := ioutil.WriteFile(fileEntry.Path, fileEntry.Content, 0644)
		if err != nil {
			log.Println(err)
			failed++
		} else {
			if NeedsRestart(fileEntry) {
				restart = true
			}
			updated++
		}
	}

	if failed > 0 {
		status.Health = "Failed to update " + strconv.Itoa(failed) + " files"
	} else {
		status.Health = "Updated " + strconv.Itoa(updated) + " files"
	}

	if restart {
		RestartApp("")
		status.Health = "Restarting after updating " + strconv.Itoa(updated) + " files"
	}
	return status
}

func NeedsRestart(file FileEntry) bool {
	ignoreRegex := os.Getenv(ENV_IGNORE_REGEX)
	if len(ignoreRegex) > 0 {
		match, _ := regexp.MatchString(ignoreRegex, file.Path)
		if match {
			log.Println("Skipping restart for: " + file.Path)
			return false
		}
	}
	restartRegex := os.Getenv(ENV_RESTART_REGEX)
	if len(restartRegex) > 0 {
		match, _ := regexp.MatchString(restartRegex, file.Path)
		if match {
			log.Println("Requires restart for: " + file.Path)
			return true
		}
	}
	return false
}<|endoftext|>


<commit_before>package models

import (
	"time"

	"github.com/koding/bongo"
)

func (c *ChannelMessageList) BeforeCreate() error {
	c.AddedAt = time.Now()
	c.RevisedAt = time.Now()

	return c.MarkIfExempt()
}

func (c *ChannelMessageList) BeforeUpdate() error {
	c.AddedAt = time.Now()

	return c.MarkIfExempt()
}

func (c *ChannelMessageList) AfterCreate() {
	bongo.B.AfterCreate(c)
}

func (c *ChannelMessageList) AfterUpdate() {
	bongo.B.AfterUpdate(c)
}

func (c ChannelMessageList) AfterDelete() {
	bongo.B.AfterDelete(c)
}

func (c ChannelMessageList) GetId() int64 {
	return c.Id
}

func (c ChannelMessageList) TableName() string {
	return "api.channel_message_list"
}

func NewChannelMessageList() *ChannelMessageList {
	return &ChannelMessageList{}
}

func (c *ChannelMessageList) ById(id int64) error {
	return bongo.B.ById(c, id)
}

func (c *ChannelMessageList) One(q *bongo.Query) error {
	return bongo.B.One(c, c, q)
}

func (c *ChannelMessageList) Update() error {
	return bongo.B.Update(c)
}

func (c *ChannelMessageList) Some(data interface{}, q *bongo.Query) error {
	return bongo.B.Some(c, data, q)
}

func (c *ChannelMessageList) CountWithQuery(q *bongo.Query) (int, error) {
	return bongo.B.CountWithQuery(c, q)
}

func (c *ChannelMessageList) Create() error {
	return bongo.B.Create(c)
}

func (c *ChannelMessageList) Delete() error {
	return bongo.B.Delete(c)
}

func (c *ChannelMessageList) Emit(eventName string, data interface{}) error {
	return bongo.B.Emit(c.TableName()+"_"+eventName, data)
}
<commit_msg>Social: do not update added at time<commit_after>package models

import (
	"time"

	"github.com/koding/bongo"
)

func (c *ChannelMessageList) BeforeCreate() error {
	c.AddedAt = time.Now()
	c.RevisedAt = time.Now()

	return c.MarkIfExempt()
}

func (c *ChannelMessageList) BeforeUpdate() error {
	return c.MarkIfExempt()
}

func (c *ChannelMessageList) AfterCreate() {
	bongo.B.AfterCreate(c)
}

func (c *ChannelMessageList) AfterUpdate() {
	bongo.B.AfterUpdate(c)
}

func (c ChannelMessageList) AfterDelete() {
	bongo.B.AfterDelete(c)
}

func (c ChannelMessageList) GetId() int64 {
	return c.Id
}

func (c ChannelMessageList) TableName() string {
	return "api.channel_message_list"
}

func NewChannelMessageList() *ChannelMessageList {
	return &ChannelMessageList{}
}

func (c *ChannelMessageList) ById(id int64) error {
	return bongo.B.ById(c, id)
}

func (c *ChannelMessageList) One(q *bongo.Query) error {
	return bongo.B.One(c, c, q)
}

func (c *ChannelMessageList) Update() error {
	return bongo.B.Update(c)
}

func (c *ChannelMessageList) Some(data interface{}, q *bongo.Query) error {
	return bongo.B.Some(c, data, q)
}

func (c *ChannelMessageList) CountWithQuery(q *bongo.Query) (int, error) {
	return bongo.B.CountWithQuery(c, q)
}

func (c *ChannelMessageList) Create() error {
	return bongo.B.Create(c)
}

func (c *ChannelMessageList) Delete() error {
	return bongo.B.Delete(c)
}

func (c *ChannelMessageList) Emit(eventName string, data interface{}) error {
	return bongo.B.Emit(c.TableName()+"_"+eventName, data)
}
<|endoftext|>


<commit_before>package buildpackrunner

import (
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"code.cloudfoundry.org/cacheddownloader"
	"github.com/cloudfoundry/systemcerts"
	"github.com/pivotal-golang/archiver/extractor"
)

type ZipDownloader struct {
	downloader *cacheddownloader.Downloader
}

func IsZipFile(filename string) bool {
	return strings.HasSuffix(filename, ".zip")
}

func NewZipDownloader(skipSSLVerification bool) *ZipDownloader {
	return &ZipDownloader{
		downloader: cacheddownloader.NewDownloader(DOWNLOAD_TIMEOUT, 1, skipSSLVerification, systemcerts.SystemRootsPool()),
	}
}

func (z *ZipDownloader) DownloadAndExtract(u *url.URL, destination string) (uint64, error) {
	zipFile, err := ioutil.TempFile("", filepath.Base(u.Path))
	if err != nil {
		return 0, fmt.Errorf("Could not create zip file: %s", err.Error())
	}
	defer os.Remove(zipFile.Name())

	_, _, err = z.downloader.Download(u,
		func() (*os.File, error) {
			return os.OpenFile(zipFile.Name(), os.O_WRONLY, 0666)
		},
		cacheddownloader.CachingInfoType{},
		cacheddownloader.ChecksumInfoType{},
		make(chan struct{}),
	)
	if err != nil {
		return 0, fmt.Errorf("Failed to download buildpack '%s': %s", u.String(), err.Error())
	}

	fi, err := zipFile.Stat()
	if err != nil {
		return 0, fmt.Errorf("Failed to obtain the size of the buildpack '%s': %s", u.String(), err.Error())
	}

	err = extractor.NewZip().Extract(zipFile.Name(), destination)
	if err != nil {
		return 0, fmt.Errorf("Failed to extract buildpack '%s': %s", u.String(), err.Error())
	}

	return uint64(fi.Size()), nil
}
<commit_msg>Update import location of archiver<commit_after>package buildpackrunner

import (
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"code.cloudfoundry.org/archiver/extractor"
	"code.cloudfoundry.org/cacheddownloader"
	"github.com/cloudfoundry/systemcerts"
)

type ZipDownloader struct {
	downloader *cacheddownloader.Downloader
}

func IsZipFile(filename string) bool {
	return strings.HasSuffix(filename, ".zip")
}

func NewZipDownloader(skipSSLVerification bool) *ZipDownloader {
	return &ZipDownloader{
		downloader: cacheddownloader.NewDownloader(DOWNLOAD_TIMEOUT, 1, skipSSLVerification, systemcerts.SystemRootsPool()),
	}
}

func (z *ZipDownloader) DownloadAndExtract(u *url.URL, destination string) (uint64, error) {
	zipFile, err := ioutil.TempFile("", filepath.Base(u.Path))
	if err != nil {
		return 0, fmt.Errorf("Could not create zip file: %s", err.Error())
	}
	defer os.Remove(zipFile.Name())

	_, _, err = z.downloader.Download(u,
		func() (*os.File, error) {
			return os.OpenFile(zipFile.Name(), os.O_WRONLY, 0666)
		},
		cacheddownloader.CachingInfoType{},
		cacheddownloader.ChecksumInfoType{},
		make(chan struct{}),
	)
	if err != nil {
		return 0, fmt.Errorf("Failed to download buildpack '%s': %s", u.String(), err.Error())
	}

	fi, err := zipFile.Stat()
	if err != nil {
		return 0, fmt.Errorf("Failed to obtain the size of the buildpack '%s': %s", u.String(), err.Error())
	}

	err = extractor.NewZip().Extract(zipFile.Name(), destination)
	if err != nil {
		return 0, fmt.Errorf("Failed to extract buildpack '%s': %s", u.String(), err.Error())
	}

	return uint64(fi.Size()), nil
}
<|endoftext|>


<commit_before>package statictl

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"

	log "github.com/Sirupsen/logrus"
	"github.com/hjson/hjson-go"
)

// Normal replace
type DatabaseEntryStatic struct {
	Original   string
	Translated string
}

// Regex
type DatabaseEntryDynamic struct {
	RegexMatch   string
	RegexReplace string

	/*
		re := regexp.MustCompile(`((fo)(o))`)
		s := re.ReplaceAllString("foo", "$1-$1-$2-$3") // foo => foo-foo-fo-o
	*/
}

var staticDBPath = filepath.Join("database", "static")
var dynamicDBPath = filepath.Join("database", "dynamic")

func (t *Db) loadDatabases() error {
	for k, v := range databaseFiles {
		err := t.loadDatabase(k, v)
		if err != nil {
			continue
		}
	}

	return nil
}

func (t *Db) loadDatabase(tlType TranslationType, fileName string) error {
	filePathStatic := filepath.Join(staticDBPath, fileName)
	filePathDynamic := filepath.Join(dynamicDBPath, fileName)

	var db translationDB
	var dbRe []translationDBRegex
	var err error

	if !fileExists(filePathStatic) {
		log.Infof("Database %s doesn't exist, creating empty file", filePathStatic)
		createEmptyDatabase(filePathStatic, DbStatic)
	} else {
		db, err = t.loadDatabaseStatic(filePathStatic)
		if err != nil {
			log.Error(err)
		}
	}

	if !fileExists(filePathDynamic) {
		log.Infof("Database %s doesn't exist, creating empty file", filePathDynamic)
		createEmptyDatabase(filePathDynamic, DbDynamic)
	} else {
		dbRe, err = t.loadDatabaseDynamic(filePathDynamic)
		if err != nil {
			log.Error(err)
		}
	}

	t.db[tlType] = db
	t.dbRe[tlType] = dbRe

	return nil
}

func (t *Db) loadDatabaseStatic(fileName string) (translationDB, error) {
	db := make(translationDB)

	log.Infof("Parsing database %s", fileName)

	file, err := ioutil.ReadFile(fileName)
	if err != nil {
		log.Fatalf("File error: %v\n", err)
	}

	var dat []interface{}
	if err := hjson.Unmarshal(file, &dat); err != nil {
		log.Error(err)
	} else {
		// convert to JSON
		b, _ := json.Marshal(dat)

		// unmarshal
		var dbEntries []DatabaseEntryStatic
		json.Unmarshal(b, &dbEntries)

		for _, v := range dbEntries {
			db[v.Original] = v.Translated
		}
	}

	return db, nil
}

func (t *Db) loadDatabaseDynamic(fileName string) ([]translationDBRegex, error) {
	var dbRe []translationDBRegex

	log.Infof("Parsing database %s", fileName)

	file, err := ioutil.ReadFile(fileName)
	if err != nil {
		log.Fatalf("File error: %v\n", err)
	}

	var dat []interface{}
	if err := hjson.Unmarshal(file, &dat); err != nil {
		log.Error(err)
	} else {
		// convert to JSON
		b, _ := json.Marshal(dat)

		// unmarshal
		var dbEntries []DatabaseEntryDynamic
		json.Unmarshal(b, &dbEntries)

		for _, v := range dbEntries {
			if len(v.RegexMatch) < 1 {
				continue
			}

			matchRegex, err := regexp.Compile(v.RegexMatch)
			if err != nil {
				log.Errorf("Failed to compile regex\n%s", v.RegexMatch)
				continue
			}

			dbRe = append(dbRe, translationDBRegex{
				regex:       matchRegex,
				replacement: v.RegexReplace,
			})
		}
	}

	return dbRe, nil
}

func fileExists(fileName string) bool {
	_, err := os.Stat(fileName)
	if err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}

	// Probably exists but it might have other issues
	return true
}

func createEmptyDatabase(filePath string, typ DatabaseType) error {

	var content interface{}
	var fileDir string

	switch typ {
	case DbStatic:
		fileDir = staticDBPath
		content = []DatabaseEntryStatic{{
			Original:   "",
			Translated: "",
		}}

	case DbDynamic:
		fileDir = dynamicDBPath
		content = []DatabaseEntryDynamic{{
			RegexMatch:   "",
			RegexReplace: "",
		}}
	}

	err := os.MkdirAll(fileDir, 0755)
	if err != nil && !os.IsExist(err) {
		log.Error(err)
		return err
	}

	outJSON, err := hjson.Marshal(content)
	if err != nil {
		log.Error(err)
	}

	err = ioutil.WriteFile(filePath, outJSON, 0644)

	return err
}
<commit_msg>Silence parsing message<commit_after>package statictl

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"

	log "github.com/Sirupsen/logrus"
	"github.com/hjson/hjson-go"
)

// Normal replace
type DatabaseEntryStatic struct {
	Original   string
	Translated string
}

// Regex
type DatabaseEntryDynamic struct {
	RegexMatch   string
	RegexReplace string

	/*
		re := regexp.MustCompile(`((fo)(o))`)
		s := re.ReplaceAllString("foo", "$1-$1-$2-$3") // foo => foo-foo-fo-o
	*/
}

var staticDBPath = filepath.Join("database", "static")
var dynamicDBPath = filepath.Join("database", "dynamic")

func (t *Db) loadDatabases() error {
	for k, v := range databaseFiles {
		err := t.loadDatabase(k, v)
		if err != nil {
			continue
		}
	}

	return nil
}

func (t *Db) loadDatabase(tlType TranslationType, fileName string) error {
	filePathStatic := filepath.Join(staticDBPath, fileName)
	filePathDynamic := filepath.Join(dynamicDBPath, fileName)

	var db translationDB
	var dbRe []translationDBRegex
	var err error

	if !fileExists(filePathStatic) {
		log.Infof("Database %s doesn't exist, creating empty file", filePathStatic)
		createEmptyDatabase(filePathStatic, DbStatic)
	} else {
		db, err = t.loadDatabaseStatic(filePathStatic)
		if err != nil {
			log.Error(err)
		}
	}

	if !fileExists(filePathDynamic) {
		log.Infof("Database %s doesn't exist, creating empty file", filePathDynamic)
		createEmptyDatabase(filePathDynamic, DbDynamic)
	} else {
		dbRe, err = t.loadDatabaseDynamic(filePathDynamic)
		if err != nil {
			log.Error(err)
		}
	}

	t.db[tlType] = db
	t.dbRe[tlType] = dbRe

	return nil
}

func (t *Db) loadDatabaseStatic(fileName string) (translationDB, error) {
	db := make(translationDB)

	log.Infof("Parsing database %s", fileName)

	file, err := ioutil.ReadFile(fileName)
	if err != nil {
		log.Fatalf("File error: %v\n", err)
	}

	var dat []interface{}
	if err := hjson.Unmarshal(file, &dat); err != nil {
		log.Error(err)
	} else {
		// convert to JSON
		b, _ := json.Marshal(dat)

		// unmarshal
		var dbEntries []DatabaseEntryStatic
		json.Unmarshal(b, &dbEntries)

		for _, v := range dbEntries {
			db[v.Original] = v.Translated
		}
	}

	return db, nil
}

func (t *Db) loadDatabaseDynamic(fileName string) ([]translationDBRegex, error) {
	var dbRe []translationDBRegex

	log.Debugf("Parsing database %s", fileName)

	file, err := ioutil.ReadFile(fileName)
	if err != nil {
		log.Fatalf("File error: %v\n", err)
	}

	var dat []interface{}
	if err := hjson.Unmarshal(file, &dat); err != nil {
		log.Error(err)
	} else {
		// convert to JSON
		b, _ := json.Marshal(dat)

		// unmarshal
		var dbEntries []DatabaseEntryDynamic
		json.Unmarshal(b, &dbEntries)

		for _, v := range dbEntries {
			if len(v.RegexMatch) < 1 {
				continue
			}

			matchRegex, err := regexp.Compile(v.RegexMatch)
			if err != nil {
				log.Errorf("Failed to compile regex\n%s", v.RegexMatch)
				continue
			}

			dbRe = append(dbRe, translationDBRegex{
				regex:       matchRegex,
				replacement: v.RegexReplace,
			})
		}
	}

	return dbRe, nil
}

func fileExists(fileName string) bool {
	_, err := os.Stat(fileName)
	if err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}

	// Probably exists but it might have other issues
	return true
}

func createEmptyDatabase(filePath string, typ DatabaseType) error {

	var content interface{}
	var fileDir string

	switch typ {
	case DbStatic:
		fileDir = staticDBPath
		content = []DatabaseEntryStatic{{
			Original:   "",
			Translated: "",
		}}

	case DbDynamic:
		fileDir = dynamicDBPath
		content = []DatabaseEntryDynamic{{
			RegexMatch:   "",
			RegexReplace: "",
		}}
	}

	err := os.MkdirAll(fileDir, 0755)
	if err != nil && !os.IsExist(err) {
		log.Error(err)
		return err
	}

	outJSON, err := hjson.Marshal(content)
	if err != nil {
		log.Error(err)
	}

	err = ioutil.WriteFile(filePath, outJSON, 0644)

	return err
}
<|endoftext|>


<commit_before>package generator

import (
	"log"
	"sort"
	"strings"
)

// Import is a package import with the associated alias for that package.
type Import struct {
	Alias string
	Path  string
}

// AddImport creates an import with the given alias and path, and adds it to
// Fake.Imports.
func (f *Fake) AddImport(alias string, path string) Import {
	path = unvendor(strings.TrimSpace(path))
	alias = strings.TrimSpace(alias)
	for i := range f.Imports {
		if f.Imports[i].Path == path {
			return f.Imports[i]
		}
	}
	log.Printf("Adding import: %s > %s\n", alias, path)
	result := Import{
		Alias: alias,
		Path:  path,
	}
	f.Imports = append(f.Imports, result)
	return result
}

// SortImports sorts imports alphabetically.
func (f *Fake) sortImports() {
	sort.SliceStable(f.Imports, func(i, j int) bool {
		if f.Imports[i].Path == "sync" {
			return true
		}
		if f.Imports[j].Path == "sync" {
			return false
		}
		return f.Imports[i].Path < f.Imports[j].Path
	})
}

func unvendor(s string) string {
	// Devendorize for use in import statement.
	if i := strings.LastIndex(s, "/vendor/"); i >= 0 {
		return s[i+len("/vendor/"):]
	}
	if strings.HasPrefix(s, "vendor/") {
		return s[len("vendor/"):]
	}
	return s
}

func (f *Fake) hasDuplicateAliases() bool {
	hasDuplicates := false
	for _, imports := range f.aliasMap() {
		if len(imports) > 1 {
			hasDuplicates = true
			break
		}
	}
	return hasDuplicates
}

func (f *Fake) printAliases() {
	for i := range f.Imports {
		log.Printf("- %s > %s\n", f.Imports[i].Alias, f.Imports[i].Path)
	}
}

// disambiguateAliases ensures that all imports are aliased uniquely.
func (f *Fake) disambiguateAliases() {
	f.sortImports()
	if !f.hasDuplicateAliases() {
		return
	}

	log.Printf("!!! Duplicate import aliases found,...")
	log.Printf("aliases before disambiguation:\n")
	f.printAliases()
	var byAlias map[string][]Import
	for {
		byAlias = f.aliasMap()
		if !f.hasDuplicateAliases() {
			break
		}

		for i := range f.Imports {
			imports := byAlias[f.Imports[i].Alias]
			if len(imports) == 1 {
				continue
			}

			for j := 0; j < len(imports); j++ {
				if imports[j].Path == f.Imports[i].Path && j > 0 {
					f.Imports[i].Alias = f.Imports[i].Alias + string('a'+byte(j-1))
					if f.Imports[i].Path == f.TargetPackage {
						f.TargetAlias = f.Imports[i].Alias
					}
				}
			}
		}
	}

	log.Println("aliases after disambiguation:")
	f.printAliases()
}

func (f *Fake) aliasMap() map[string][]Import {
	result := map[string][]Import{}
	for i := range f.Imports {
		imports := result[f.Imports[i].Alias]
		result[f.Imports[i].Alias] = append(imports, f.Imports[i])
	}
	return result
}

func (f *Fake) importsMap() map[string]Import {
	f.disambiguateAliases()
	result := map[string]Import{}
	for i := range f.Imports {
		result[f.Imports[i].Path] = f.Imports[i]
	}
	return result
}
<commit_msg>Remove unnecessary variable<commit_after>package generator

import (
	"log"
	"sort"
	"strings"
)

// Import is a package import with the associated alias for that package.
type Import struct {
	Alias string
	Path  string
}

// AddImport creates an import with the given alias and path, and adds it to
// Fake.Imports.
func (f *Fake) AddImport(alias string, path string) Import {
	path = unvendor(strings.TrimSpace(path))
	alias = strings.TrimSpace(alias)
	for i := range f.Imports {
		if f.Imports[i].Path == path {
			return f.Imports[i]
		}
	}
	log.Printf("Adding import: %s > %s\n", alias, path)
	result := Import{
		Alias: alias,
		Path:  path,
	}
	f.Imports = append(f.Imports, result)
	return result
}

// SortImports sorts imports alphabetically.
func (f *Fake) sortImports() {
	sort.SliceStable(f.Imports, func(i, j int) bool {
		if f.Imports[i].Path == "sync" {
			return true
		}
		if f.Imports[j].Path == "sync" {
			return false
		}
		return f.Imports[i].Path < f.Imports[j].Path
	})
}

func unvendor(s string) string {
	// Devendorize for use in import statement.
	if i := strings.LastIndex(s, "/vendor/"); i >= 0 {
		return s[i+len("/vendor/"):]
	}
	if strings.HasPrefix(s, "vendor/") {
		return s[len("vendor/"):]
	}
	return s
}

func (f *Fake) hasDuplicateAliases() bool {
	for _, imports := range f.aliasMap() {
		if len(imports) > 1 {
			return true
		}
	}
	return false
}

func (f *Fake) printAliases() {
	for i := range f.Imports {
		log.Printf("- %s > %s\n", f.Imports[i].Alias, f.Imports[i].Path)
	}
}

// disambiguateAliases ensures that all imports are aliased uniquely.
func (f *Fake) disambiguateAliases() {
	f.sortImports()
	if !f.hasDuplicateAliases() {
		return
	}

	log.Printf("!!! Duplicate import aliases found,...")
	log.Printf("aliases before disambiguation:\n")
	f.printAliases()
	var byAlias map[string][]Import
	for {
		byAlias = f.aliasMap()
		if !f.hasDuplicateAliases() {
			break
		}

		for i := range f.Imports {
			imports := byAlias[f.Imports[i].Alias]
			if len(imports) == 1 {
				continue
			}

			for j := 0; j < len(imports); j++ {
				if imports[j].Path == f.Imports[i].Path && j > 0 {
					f.Imports[i].Alias = f.Imports[i].Alias + string('a'+byte(j-1))
					if f.Imports[i].Path == f.TargetPackage {
						f.TargetAlias = f.Imports[i].Alias
					}
				}
			}
		}
	}

	log.Println("aliases after disambiguation:")
	f.printAliases()
}

func (f *Fake) aliasMap() map[string][]Import {
	result := map[string][]Import{}
	for i := range f.Imports {
		imports := result[f.Imports[i].Alias]
		result[f.Imports[i].Alias] = append(imports, f.Imports[i])
	}
	return result
}

func (f *Fake) importsMap() map[string]Import {
	f.disambiguateAliases()
	result := map[string]Import{}
	for i := range f.Imports {
		result[f.Imports[i].Path] = f.Imports[i]
	}
	return result
}
<|endoftext|>


<commit_before>package lexer
<commit_msg>Add test for Lex<commit_after>package lexer

import (
	"bytes"
	"io/ioutil"
	"testing"
)

func TestLex(t *testing.T) {
	src, err := ioutil.ReadFile("fixture/modem.txt")
	if err != nil {
		t.Fatal(err)
	}
	_, err = Lex(bytes.NewReader(src))
	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package lfs

import (
	"path/filepath"
	"sync"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

const (
	batchSize = 100
)

type Transferable interface {
	Check() (*objectResource, error)
	Transfer(CopyCallback) error
	Object() *objectResource
	Oid() string
	Size() int64
	Name() string
	SetObject(*objectResource)
}

// TransferQueue provides a queue that will allow concurrent transfers.
type TransferQueue struct {
	meter         *ProgressMeter
	workers       int // Number of transfer workers to spawn
	transferKind  string
	errors        []error
	transferables map[string]Transferable
	batcher       *Batcher
	apic          chan Transferable // Channel for processing individual API requests
	transferc     chan Transferable // Channel for processing transfers
	errorc        chan error        // Channel for processing errors
	watchers      []chan string
	wait          sync.WaitGroup
	retries       map[string]interface{}
	retrylock     sync.Mutex
}

// newTransferQueue builds a TransferQueue, allowing `workers` concurrent transfers.
func newTransferQueue(files int, size int64, dryRun bool) *TransferQueue {
	q := &TransferQueue{
		meter:         NewProgressMeter(files, size, dryRun),
		apic:          make(chan Transferable, batchSize),
		transferc:     make(chan Transferable, batchSize),
		errorc:        make(chan error),
		workers:       Config.ConcurrentTransfers(),
		transferables: make(map[string]Transferable),
		retries:       make(map[string]interface{}),
	}

	q.run()

	return q
}

// Add adds a Transferable to the transfer queue.
func (q *TransferQueue) Add(t Transferable) {
	q.wait.Add(1)
	q.transferables[t.Oid()] = t

	if q.batcher != nil {
		q.batcher.Add(t)
		return
	}

	q.apic <- t
}

// Wait waits for the queue to finish processing all transfers
func (q *TransferQueue) Wait() {
	q.wait.Wait()

	if q.batcher != nil {
		q.batcher.Exit()
	}

	close(q.apic)
	close(q.transferc)
	close(q.errorc)

	for _, watcher := range q.watchers {
		close(watcher)
	}

	q.meter.Finish()
}

// Watch returns a channel where the queue will write the OID of each transfer
// as it completes. The channel will be closed when the queue finishes processing.
func (q *TransferQueue) Watch() chan string {
	c := make(chan string, batchSize)
	q.watchers = append(q.watchers, c)
	return c
}

// individualApiRoutine processes the queue of transfers one at a time by making
// a POST call for each object, feeding the results to the transfer workers.
// If configured, the object transfers can still happen concurrently, the
// sequential nature here is only for the meta POST calls.
func (q *TransferQueue) individualApiRoutine(apiWaiter chan interface{}) {
	for t := range q.apic {
		obj, err := t.Check()
		if err != nil {
			if q.canRetry(err, t.Oid()) {
				q.Add(t)
			} else {
				q.errorc <- err
			}
			q.wait.Done()
			continue
		}

		if apiWaiter != nil { // Signal to launch more individual api workers
			q.meter.Start()
			select {
			case apiWaiter <- 1:
			default:
			}
		}

		if obj != nil {
			t.SetObject(obj)
			q.meter.Add(t.Name())
			q.transferc <- t
		} else {
			q.meter.Skip(t.Size())
			q.wait.Done()
		}
	}
}

// legacyFallback is used when a batch request is made to a server that does
// not support the batch endpoint. When this happens, the Transferables are
// fed from the batcher into apic to be processed individually.
func (q *TransferQueue) legacyFallback(failedBatch []Transferable) {
	tracerx.Printf("tq: batch api not implemented, falling back to individual")

	q.launchIndividualApiRoutines()

	for _, t := range failedBatch {
		q.apic <- t
	}

	for {
		batch := q.batcher.Next()
		if batch == nil {
			break
		}

		for _, t := range batch {
			q.apic <- t
		}
	}
}

// batchApiRoutine processes the queue of transfers using the batch endpoint,
// making only one POST call for all objects. The results are then handed
// off to the transfer workers.
func (q *TransferQueue) batchApiRoutine() {
	var startProgress sync.Once
	batchNumber := 0

	for {
		batchNumber++

		batch := q.batcher.Next()
		if batch == nil {
			break
		}

		tracerx.Printf("tq: sending batch of size %d", len(batch))

		transfers := make([]*objectResource, 0, len(batch))
		for _, t := range batch {
			transfers = append(transfers, &objectResource{Oid: t.Oid(), Size: t.Size()})
		}

		objects, err := Batch(transfers, q.transferKind)
		if err != nil {
			if IsNotImplementedError(err) {
				configFile := filepath.Join(LocalGitDir, "config")
				git.Config.SetLocal(configFile, "lfs.batch", "false")

				go q.legacyFallback(batch)
				return
			}

			// TODO technically, this could go forever. Maybe we just limit it to n batch retries total.
			if q.canRetry(err, "batch") {
				tracerx.Printf("tq: resubmitting batch: %s", err)
				for _, t := range batch {
					q.Add(t)
				}
			} else {
				tracerx.Printf("Too many batch failures, erroring")
				q.errorc <- err
			}

			q.wait.Add(-len(transfers))
			continue
		}

		startProgress.Do(q.meter.Start)

		for _, o := range objects {
			if _, ok := o.Rel(q.transferKind); ok {
				// This object has an error
				if o.Error != nil {
					q.errorc <- Error(o.Error)
					q.meter.Skip(o.Size)
					q.wait.Done()
					continue
				}

				// This object needs to be transferred
				if transfer, ok := q.transferables[o.Oid]; ok {
					transfer.SetObject(o)
					q.meter.Add(transfer.Name())
					q.transferc <- transfer
				} else {
					q.meter.Skip(transfer.Size())
					q.wait.Done()
				}
			} else {
				q.meter.Skip(o.Size)
				q.wait.Done()
			}
		}
	}
}

// This goroutine collects errors returned from transfers
func (q *TransferQueue) errorCollector() {
	for err := range q.errorc {
		q.errors = append(q.errors, err)
	}
}

func (q *TransferQueue) transferWorker() {
	for transfer := range q.transferc {
		cb := func(total, read int64, current int) error {
			q.meter.TransferBytes(q.transferKind, transfer.Name(), read, total, current)
			return nil
		}

		if err := transfer.Transfer(cb); err != nil {
			if q.canRetry(err, transfer.Oid()) {
				tracerx.Printf("tq: retrying object %s", transfer.Oid())
				q.Add(transfer)
			} else {
				q.errorc <- err
			}
		} else {
			oid := transfer.Oid()
			for _, c := range q.watchers {
				c <- oid
			}
		}

		q.meter.FinishTransfer(transfer.Name())

		q.wait.Done()
	}
}

// launchIndividualApiRoutines first launches a single api worker. When it
// receives the first successful api request it launches workers - 1 more
// workers. This prevents being prompted for credentials multiple times at once
// when they're needed.
func (q *TransferQueue) launchIndividualApiRoutines() {
	go func() {
		apiWaiter := make(chan interface{})
		go q.individualApiRoutine(apiWaiter)

		<-apiWaiter

		for i := 0; i < q.workers-1; i++ {
			go q.individualApiRoutine(nil)
		}
	}()
}

// run starts the transfer queue, doing individual or batch transfers depending
// on the Config.BatchTransfer() value. run will transfer files sequentially or
// concurrently depending on the Config.ConcurrentTransfers() value.
func (q *TransferQueue) run() {
	go q.errorCollector()

	tracerx.Printf("tq: starting %d transfer workers", q.workers)
	for i := 0; i < q.workers; i++ {
		go q.transferWorker()
	}

	if Config.BatchTransfer() {
		tracerx.Printf("tq: running as batched queue, batch size of %d", batchSize)
		q.batcher = NewBatcher(batchSize)
		go q.batchApiRoutine()
	} else {
		tracerx.Printf("tq: running as individual queue")
		q.launchIndividualApiRoutines()
	}
}

func (q *TransferQueue) canRetry(err error, id string) bool {
	if !IsRetriableError(err) {
		return false
	}

	defer q.retrylock.Unlock()
	q.retrylock.Lock()
	if _, ok := q.retries[id]; ok {
		// Already retried it
		return false
	}
	q.retries[id] = struct{}{}
	return true
}

// Errors returns any errors encountered during transfer.
func (q *TransferQueue) Errors() []error {
	return q.errors
}
<commit_msg>ンンン ンンン ンン<commit_after>package lfs

import (
	"path/filepath"
	"sync"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

const (
	batchSize       = 100
	maxBatchRetries = 3
)

type Transferable interface {
	Check() (*objectResource, error)
	Transfer(CopyCallback) error
	Object() *objectResource
	Oid() string
	Size() int64
	Name() string
	SetObject(*objectResource)
}

// TransferQueue provides a queue that will allow concurrent transfers.
type TransferQueue struct {
	meter         *ProgressMeter
	workers       int // Number of transfer workers to spawn
	transferKind  string
	errors        []error
	transferables map[string]Transferable
	batcher       *Batcher
	apic          chan Transferable // Channel for processing individual API requests
	transferc     chan Transferable // Channel for processing transfers
	errorc        chan error        // Channel for processing errors
	watchers      []chan string
	wait          sync.WaitGroup
	retries       map[string]interface{}
	retrylock     sync.Mutex
}

// newTransferQueue builds a TransferQueue, allowing `workers` concurrent transfers.
func newTransferQueue(files int, size int64, dryRun bool) *TransferQueue {
	q := &TransferQueue{
		meter:         NewProgressMeter(files, size, dryRun),
		apic:          make(chan Transferable, batchSize),
		transferc:     make(chan Transferable, batchSize),
		errorc:        make(chan error),
		workers:       Config.ConcurrentTransfers(),
		transferables: make(map[string]Transferable),
		retries:       make(map[string]interface{}),
	}

	q.run()

	return q
}

// Add adds a Transferable to the transfer queue.
func (q *TransferQueue) Add(t Transferable) {
	q.wait.Add(1)
	q.transferables[t.Oid()] = t

	if q.batcher != nil {
		q.batcher.Add(t)
		return
	}

	q.apic <- t
}

// Wait waits for the queue to finish processing all transfers
func (q *TransferQueue) Wait() {
	q.wait.Wait()

	if q.batcher != nil {
		q.batcher.Exit()
	}

	close(q.apic)
	close(q.transferc)
	close(q.errorc)

	for _, watcher := range q.watchers {
		close(watcher)
	}

	q.meter.Finish()
}

// Watch returns a channel where the queue will write the OID of each transfer
// as it completes. The channel will be closed when the queue finishes processing.
func (q *TransferQueue) Watch() chan string {
	c := make(chan string, batchSize)
	q.watchers = append(q.watchers, c)
	return c
}

// individualApiRoutine processes the queue of transfers one at a time by making
// a POST call for each object, feeding the results to the transfer workers.
// If configured, the object transfers can still happen concurrently, the
// sequential nature here is only for the meta POST calls.
func (q *TransferQueue) individualApiRoutine(apiWaiter chan interface{}) {
	for t := range q.apic {
		obj, err := t.Check()
		if err != nil {
			if q.canRetry(err, t.Oid()) {
				q.Add(t)
			} else {
				q.errorc <- err
			}
			q.wait.Done()
			continue
		}

		if apiWaiter != nil { // Signal to launch more individual api workers
			q.meter.Start()
			select {
			case apiWaiter <- 1:
			default:
			}
		}

		if obj != nil {
			t.SetObject(obj)
			q.meter.Add(t.Name())
			q.transferc <- t
		} else {
			q.meter.Skip(t.Size())
			q.wait.Done()
		}
	}
}

// legacyFallback is used when a batch request is made to a server that does
// not support the batch endpoint. When this happens, the Transferables are
// fed from the batcher into apic to be processed individually.
func (q *TransferQueue) legacyFallback(failedBatch []Transferable) {
	tracerx.Printf("tq: batch api not implemented, falling back to individual")

	q.launchIndividualApiRoutines()

	for _, t := range failedBatch {
		q.apic <- t
	}

	for {
		batch := q.batcher.Next()
		if batch == nil {
			break
		}

		for _, t := range batch {
			q.apic <- t
		}
	}
}

// batchApiRoutine processes the queue of transfers using the batch endpoint,
// making only one POST call for all objects. The results are then handed
// off to the transfer workers.
func (q *TransferQueue) batchApiRoutine() {
	var startProgress sync.Once
	batchRetries := 0

	for {
		batch := q.batcher.Next()
		if batch == nil {
			break
		}

		tracerx.Printf("tq: sending batch of size %d", len(batch))

		transfers := make([]*objectResource, 0, len(batch))
		for _, t := range batch {
			transfers = append(transfers, &objectResource{Oid: t.Oid(), Size: t.Size()})
		}

		objects, err := Batch(transfers, q.transferKind)
		if err != nil {
			if IsNotImplementedError(err) {
				configFile := filepath.Join(LocalGitDir, "config")
				git.Config.SetLocal(configFile, "lfs.batch", "false")

				go q.legacyFallback(batch)
				return
			}

			// Batch operation retries should be caused by network issues. We want to
			// retry these failures, but limit it to maxBatchRetries total retries,
			// otherwise a serious network issue could cause an infinite loop of
			// retried calls.
			if IsRetriableError(err) && batchRetries <= maxBatchRetries {
				batchRetries++
				tracerx.Printf("tq: resubmitting batch: %s", err)
				for _, t := range batch {
					q.Add(t)
				}
			} else {
				q.errorc <- err
			}

			q.wait.Add(-len(transfers))
			continue
		}

		startProgress.Do(q.meter.Start)

		for _, o := range objects {
			if _, ok := o.Rel(q.transferKind); ok {
				// This object has an error
				if o.Error != nil {
					q.errorc <- Error(o.Error)
					q.meter.Skip(o.Size)
					q.wait.Done()
					continue
				}

				// This object needs to be transferred
				if transfer, ok := q.transferables[o.Oid]; ok {
					transfer.SetObject(o)
					q.meter.Add(transfer.Name())
					q.transferc <- transfer
				} else {
					q.meter.Skip(transfer.Size())
					q.wait.Done()
				}
			} else {
				q.meter.Skip(o.Size)
				q.wait.Done()
			}
		}
	}
}

// This goroutine collects errors returned from transfers
func (q *TransferQueue) errorCollector() {
	for err := range q.errorc {
		q.errors = append(q.errors, err)
	}
}

func (q *TransferQueue) transferWorker() {
	for transfer := range q.transferc {
		cb := func(total, read int64, current int) error {
			q.meter.TransferBytes(q.transferKind, transfer.Name(), read, total, current)
			return nil
		}

		if err := transfer.Transfer(cb); err != nil {
			if q.canRetry(err, transfer.Oid()) {
				tracerx.Printf("tq: retrying object %s", transfer.Oid())
				q.Add(transfer)
			} else {
				q.errorc <- err
			}
		} else {
			oid := transfer.Oid()
			for _, c := range q.watchers {
				c <- oid
			}
		}

		q.meter.FinishTransfer(transfer.Name())

		q.wait.Done()
	}
}

// launchIndividualApiRoutines first launches a single api worker. When it
// receives the first successful api request it launches workers - 1 more
// workers. This prevents being prompted for credentials multiple times at once
// when they're needed.
func (q *TransferQueue) launchIndividualApiRoutines() {
	go func() {
		apiWaiter := make(chan interface{})
		go q.individualApiRoutine(apiWaiter)

		<-apiWaiter

		for i := 0; i < q.workers-1; i++ {
			go q.individualApiRoutine(nil)
		}
	}()
}

// run starts the transfer queue, doing individual or batch transfers depending
// on the Config.BatchTransfer() value. run will transfer files sequentially or
// concurrently depending on the Config.ConcurrentTransfers() value.
func (q *TransferQueue) run() {
	go q.errorCollector()

	tracerx.Printf("tq: starting %d transfer workers", q.workers)
	for i := 0; i < q.workers; i++ {
		go q.transferWorker()
	}

	if Config.BatchTransfer() {
		tracerx.Printf("tq: running as batched queue, batch size of %d", batchSize)
		q.batcher = NewBatcher(batchSize)
		go q.batchApiRoutine()
	} else {
		tracerx.Printf("tq: running as individual queue")
		q.launchIndividualApiRoutines()
	}
}

func (q *TransferQueue) canRetry(err error, id string) bool {
	if !IsRetriableError(err) {
		return false
	}

	defer q.retrylock.Unlock()
	q.retrylock.Lock()
	if _, ok := q.retries[id]; ok {
		// Already retried it
		return false
	}
	q.retries[id] = struct{}{}
	return true
}

// Errors returns any errors encountered during transfer.
func (q *TransferQueue) Errors() []error {
	return q.errors
}
<|endoftext|>