<commit_before>package local

import (
	"context"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform/backend"
	"github.com/hashicorp/terraform/command/format"
	"github.com/hashicorp/terraform/config/module"
	"github.com/hashicorp/terraform/state"
	"github.com/hashicorp/terraform/terraform"
)

func (b *Local) opPlan(
	ctx context.Context,
	op *backend.Operation,
	runningOp *backend.RunningOperation) {
	log.Printf("[INFO] backend/local: starting Plan operation")

	if b.CLI != nil && op.Plan != nil {
		b.CLI.Output(b.Colorize().Color(
			"[reset][bold][yellow]" +
				"The plan command received a saved plan file as input. This command\n" +
				"will output the saved plan. This will not modify the already-existing\n" +
				"plan. If you wish to generate a new plan, please pass in a configuration\n" +
				"directory as an argument.\n\n"))
	}

	// A local plan requires either a plan or a module
	if op.Plan == nil && op.Module == nil && !op.Destroy {
		runningOp.Err = fmt.Errorf(strings.TrimSpace(planErrNoConfig))
		return
	}

	// If we have a nil module at this point, then set it to an empty tree
	// to avoid any potential crashes.
	if op.Module == nil {
		op.Module = module.NewEmptyTree()
	}

	// Setup our count hook that keeps track of resource changes
	countHook := new(CountHook)
	if b.ContextOpts == nil {
		b.ContextOpts = new(terraform.ContextOpts)
	}
	old := b.ContextOpts.Hooks
	defer func() { b.ContextOpts.Hooks = old }()
	b.ContextOpts.Hooks = append(b.ContextOpts.Hooks, countHook)

	// Get our context
	tfCtx, opState, err := b.context(op)
	if err != nil {
		runningOp.Err = err
		return
	}

	// context acquired the state, and therefor the lock.
	// Unlock it when the operation is complete
	defer func() {
		if s, ok := opState.(state.Locker); op.LockState && ok {
			if err := s.Unlock(); err != nil {
				log.Printf("[ERROR]: %s", err)
			}
		}
	}()

	// Setup the state
	runningOp.State = tfCtx.State()

	// If we're refreshing before plan, perform that
	if op.PlanRefresh {
		log.Printf("[INFO] backend/local: plan calling Refresh")

		if b.CLI != nil {
			b.CLI.Output(b.Colorize().Color(strings.TrimSpace(planRefreshing) + "\n"))
		}

		_, err := tfCtx.Refresh()
		if err != nil {
			runningOp.Err = errwrap.Wrapf("Error refreshing state: {{err}}", err)
			return
		}
	}

	// Perform the plan
	log.Printf("[INFO] backend/local: plan calling Plan")
	plan, err := tfCtx.Plan()
	if err != nil {
		runningOp.Err = errwrap.Wrapf("Error running plan: {{err}}", err)
		return
	}

	// Record state
	runningOp.PlanEmpty = plan.Diff.Empty()

	// Save the plan to disk
	if path := op.PlanOutPath; path != "" {
		// Write the backend if we have one
		plan.Backend = op.PlanOutBackend

		log.Printf("[INFO] backend/local: writing plan output to: %s", path)
		f, err := os.Create(path)
		if err == nil {
			err = terraform.WritePlan(plan, f)
		}
		f.Close()
		if err != nil {
			runningOp.Err = fmt.Errorf("Error writing plan file: %s", err)
			return
		}
	}

	// Perform some output tasks if we have a CLI to output to.
	if b.CLI != nil {
		if plan.Diff.Empty() {
			b.CLI.Output(b.Colorize().Color(strings.TrimSpace(planNoChanges)))
			return
		}

		if path := op.PlanOutPath; path == "" {
			b.CLI.Output(strings.TrimSpace(planHeaderNoOutput) + "\n")
		} else {
			b.CLI.Output(fmt.Sprintf(
				strings.TrimSpace(planHeaderYesOutput)+"\n",
				path))
		}

		b.CLI.Output(format.Plan(&format.PlanOpts{
			Plan:        plan,
			Color:       b.Colorize(),
			ModuleDepth: -1,
		}))

		b.CLI.Output(b.Colorize().Color(fmt.Sprintf(
			"[reset][bold]Plan:[reset] "+
				"%d to add, %d to change, %d to destroy.",
			countHook.ToAdd+countHook.ToRemoveAndAdd,
			countHook.ToChange,
			countHook.ToRemove+countHook.ToRemoveAndAdd)))
	}
}

const planErrNoConfig = `
No configuration files found!

Plan requires configuration to be present. Planning without a configuration
would mark everything for destruction, which is normally not what is desired.
If you would like to destroy everything, please run plan with the "-destroy"
flag or create a single empty configuration file. Otherwise, please create
a Terraform configuration file in the path being executed and try again.
`

const planHeaderNoOutput = `
The Terraform execution plan has been generated and is shown below.
Resources are shown in alphabetical order for quick scanning. Green resources
will be created (or destroyed and then created if an existing resource
exists), yellow resources are being changed in-place, and red resources
will be destroyed. Cyan entries are data sources to be read.

Note: You didn't specify an "-out" parameter to save this plan, so when
"apply" is called, Terraform can't guarantee this is what will execute.
`

const planHeaderYesOutput = `
The Terraform execution plan has been generated and is shown below.
Resources are shown in alphabetical order for quick scanning. Green resources
will be created (or destroyed and then created if an existing resource
exists), yellow resources are being changed in-place, and red resources
will be destroyed. Cyan entries are data sources to be read.

Your plan was also saved to the path below. Call the "apply" subcommand
with this plan file and Terraform will exactly execute this execution
plan.

Path: %s
`

const planNoChanges = `
[reset][bold][green]No changes. Infrastructure is up-to-date.[reset][green]

This means that Terraform could not detect any differences between your
configuration and real physical resources that exist. As a result, Terraform
doesn't need to do anything.
`

const planRefreshing = `
[reset][bold]Refreshing Terraform state in-memory prior to plan...[reset]
The refreshed state will be used to calculate this plan, but will not be
persisted to local or remote state storage.
`
<commit_msg>backend/local: fix could not to did not to prevent error look<commit_after>package local

import (
	"context"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform/backend"
	"github.com/hashicorp/terraform/command/format"
	"github.com/hashicorp/terraform/config/module"
	"github.com/hashicorp/terraform/state"
	"github.com/hashicorp/terraform/terraform"
)

func (b *Local) opPlan(
	ctx context.Context,
	op *backend.Operation,
	runningOp *backend.RunningOperation) {
	log.Printf("[INFO] backend/local: starting Plan operation")

	if b.CLI != nil && op.Plan != nil {
		b.CLI.Output(b.Colorize().Color(
			"[reset][bold][yellow]" +
				"The plan command received a saved plan file as input. This command\n" +
				"will output the saved plan. This will not modify the already-existing\n" +
				"plan. If you wish to generate a new plan, please pass in a configuration\n" +
				"directory as an argument.\n\n"))
	}

	// A local plan requires either a plan or a module
	if op.Plan == nil && op.Module == nil && !op.Destroy {
		runningOp.Err = fmt.Errorf(strings.TrimSpace(planErrNoConfig))
		return
	}

	// If we have a nil module at this point, then set it to an empty tree
	// to avoid any potential crashes.
	if op.Module == nil {
		op.Module = module.NewEmptyTree()
	}

	// Setup our count hook that keeps track of resource changes
	countHook := new(CountHook)
	if b.ContextOpts == nil {
		b.ContextOpts = new(terraform.ContextOpts)
	}
	old := b.ContextOpts.Hooks
	defer func() { b.ContextOpts.Hooks = old }()
	b.ContextOpts.Hooks = append(b.ContextOpts.Hooks, countHook)

	// Get our context
	tfCtx, opState, err := b.context(op)
	if err != nil {
		runningOp.Err = err
		return
	}

	// context acquired the state, and therefor the lock.
	// Unlock it when the operation is complete
	defer func() {
		if s, ok := opState.(state.Locker); op.LockState && ok {
			if err := s.Unlock(); err != nil {
				log.Printf("[ERROR]: %s", err)
			}
		}
	}()

	// Setup the state
	runningOp.State = tfCtx.State()

	// If we're refreshing before plan, perform that
	if op.PlanRefresh {
		log.Printf("[INFO] backend/local: plan calling Refresh")

		if b.CLI != nil {
			b.CLI.Output(b.Colorize().Color(strings.TrimSpace(planRefreshing) + "\n"))
		}

		_, err := tfCtx.Refresh()
		if err != nil {
			runningOp.Err = errwrap.Wrapf("Error refreshing state: {{err}}", err)
			return
		}
	}

	// Perform the plan
	log.Printf("[INFO] backend/local: plan calling Plan")
	plan, err := tfCtx.Plan()
	if err != nil {
		runningOp.Err = errwrap.Wrapf("Error running plan: {{err}}", err)
		return
	}

	// Record state
	runningOp.PlanEmpty = plan.Diff.Empty()

	// Save the plan to disk
	if path := op.PlanOutPath; path != "" {
		// Write the backend if we have one
		plan.Backend = op.PlanOutBackend

		log.Printf("[INFO] backend/local: writing plan output to: %s", path)
		f, err := os.Create(path)
		if err == nil {
			err = terraform.WritePlan(plan, f)
		}
		f.Close()
		if err != nil {
			runningOp.Err = fmt.Errorf("Error writing plan file: %s", err)
			return
		}
	}

	// Perform some output tasks if we have a CLI to output to.
	if b.CLI != nil {
		if plan.Diff.Empty() {
			b.CLI.Output(b.Colorize().Color(strings.TrimSpace(planNoChanges)))
			return
		}

		if path := op.PlanOutPath; path == "" {
			b.CLI.Output(strings.TrimSpace(planHeaderNoOutput) + "\n")
		} else {
			b.CLI.Output(fmt.Sprintf(
				strings.TrimSpace(planHeaderYesOutput)+"\n",
				path))
		}

		b.CLI.Output(format.Plan(&format.PlanOpts{
			Plan:        plan,
			Color:       b.Colorize(),
			ModuleDepth: -1,
		}))

		b.CLI.Output(b.Colorize().Color(fmt.Sprintf(
			"[reset][bold]Plan:[reset] "+
				"%d to add, %d to change, %d to destroy.",
			countHook.ToAdd+countHook.ToRemoveAndAdd,
			countHook.ToChange,
			countHook.ToRemove+countHook.ToRemoveAndAdd)))
	}
}

const planErrNoConfig = `
No configuration files found!

Plan requires configuration to be present. Planning without a configuration
would mark everything for destruction, which is normally not what is desired.
If you would like to destroy everything, please run plan with the "-destroy"
flag or create a single empty configuration file. Otherwise, please create
a Terraform configuration file in the path being executed and try again.
`

const planHeaderNoOutput = `
The Terraform execution plan has been generated and is shown below.
Resources are shown in alphabetical order for quick scanning. Green resources
will be created (or destroyed and then created if an existing resource
exists), yellow resources are being changed in-place, and red resources
will be destroyed. Cyan entries are data sources to be read.

Note: You didn't specify an "-out" parameter to save this plan, so when
"apply" is called, Terraform can't guarantee this is what will execute.
`

const planHeaderYesOutput = `
The Terraform execution plan has been generated and is shown below.
Resources are shown in alphabetical order for quick scanning. Green resources
will be created (or destroyed and then created if an existing resource
exists), yellow resources are being changed in-place, and red resources
will be destroyed. Cyan entries are data sources to be read.

Your plan was also saved to the path below. Call the "apply" subcommand
with this plan file and Terraform will exactly execute this execution
plan.

Path: %s
`

const planNoChanges = `
[reset][bold][green]No changes. Infrastructure is up-to-date.[reset][green]

This means that Terraform did not detect any differences between your
configuration and real physical resources that exist. As a result, Terraform
doesn't need to do anything.
`

const planRefreshing = `
[reset][bold]Refreshing Terraform state in-memory prior to plan...[reset]
The refreshed state will be used to calculate this plan, but will not be
persisted to local or remote state storage.
`
<|endoftext|>


<commit_before>// Package wal defines and implements a general purpose, high performance
// write-ahead-log for performing ACID transactions to disk without sacrificing
// speed or latency more than fundamentally required.
package wal

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"io"
	"math"
	"os"
	"sort"
	"sync"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/persist"
	"github.com/NebulousLabs/errors"
)

// WAL is a general purpose, high performance write-ahead-log for performing
// ACID transactions to disk without sacrificing speed or latency more than
// fundamentally required.
type WAL struct {
	// availablePages lists the offset of file pages which currently have completed or
	// voided updates in them. The pages are in no particular order.
	availablePages []uint64

	// filePageCount indicates the number of pages total in the file. If the
	// number of availablePages ever drops below the number of pages required
	// for a new transaction, then the file is extended, new pages are added,
	// and the availablePages array is updated to include the extended pages.
	filePageCount uint64

	// atomicNextTxnNum is used to give every transaction a unique transaction
	// number. The transaction will then wait until atomicTransactionCounter allows
	// the transaction to be committed. This ensures that transactions are committed
	// in the correct order.
	atomicNextTxnNum uint64

	// logFile contains all of the persistent data associated with the log.
	logFile file

	// path is the path to the underlying logFile
	path string

	// mu is used to lock the availablePages field of the wal
	mu sync.Mutex

	// syncCond is used to schedule the calls to fsync
	syncCond *sync.Cond

	// syncMu is the lock contained in syncCond and must be held before
	// changing the state of the syncCond
	syncMu sync.Mutex

	// syncCount is a counter that indicates how many transactions are
	// currently waiting for a fsync
	syncCount uint64

	// stopChan is a channel that is used to signal a shutdown
	stopChan chan struct{}

	// syncing indicates if the syncing thread is currently being executed
	syncing bool

	// recoveryComplete indicates if the caller signalled that the recovery is complete
	recoveryComplete bool

	// dependencies are used to inject special behaviour into the wal by providing
	// custom dependencies when the wal is created and calling deps.disrupt(setting).
	// The following settings are currently available
	deps dependencies
}

type (
	// SortPage is a helper struct for sorting, that contains a page and the offset
	// of the next page
	SortPage struct {
		p        *page
		nextPage uint64
	}
	// ByTxnNumber is a type that implements the sorting interface for the SortPage struct
	ByTxnNumber []SortPage
)

// Len returns the length of the slice
func (p ByTxnNumber) Len() int {
	return len(p)
}

// Swap swaps two elements of the slice
func (p ByTxnNumber) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

// Less determines if one element of the slice is less than the other
func (p ByTxnNumber) Less(i, j int) bool {
	return p[i].p.transactionNumber < p[j].p.transactionNumber
}

// allocatePages creates new pages and adds them to the available pages of the wal
func (w *WAL) allocatePages(numPages uint64) {
	// Starting at index 1 because the first page is reserved for metadata
	for i := w.filePageCount + 1; i < w.filePageCount+numPages+1; i++ {
		w.availablePages = append(w.availablePages, i*pageSize)
	}
	w.filePageCount += numPages
}

// newWal initializes and returns a wal.
func newWal(path string, deps dependencies) (u []Update, w *WAL, err error) {
	// Create a new WAL
	newWal := WAL{
		deps:     deps,
		stopChan: make(chan struct{}),
		path:     path,
	}

	// Create a condition for the wal
	newWal.syncCond = sync.NewCond(&newWal.syncMu)
	// Try opening the WAL file.
	data, err := deps.readFile(path)
	if err == nil {
		// Recover WAL and return updates
		updates, err := newWal.recover(data)
		if err != nil {
			return nil, nil, err
		}

		// Reuse the existing wal
		newWal.logFile, err = deps.openFile(path, os.O_RDWR, 0600)

		return updates, &newWal, err

	} else if !os.IsNotExist(err) {
		// the file exists but couldn't be opened
		return nil, nil, build.ExtendErr("walFile was not opened successfully", err)
	}

	// Create new empty WAL
	newWal.logFile, err = deps.create(path)
	if err != nil {
		return nil, nil, build.ExtendErr("walFile could not be created", err)
	}

	// Write the metadata to the WAL
	if err = writeWALMetadata(newWal.logFile); err != nil {
		return nil, nil, build.ExtendErr("Failed to write metadata to file", err)
	}

	// When we create a new wal we don't need the caller to signal recoveryComplete
	newWal.recoveryComplete = true
	return nil, &newWal, nil
}

// readWALMetadata reads WAL metadata from the input file, returning an error
// if the result is unexpected.
func readWALMetadata(data []byte) error {
	var md persist.Metadata
	decoder := json.NewDecoder(bytes.NewBuffer(data))
	err := decoder.Decode(&md)
	if err != nil {
		return build.ExtendErr("error reading WAL metadata", err)
	}
	if md.Header != metadata.Header {
		return errors.New("WAL metadata header does not match header found in WAL file")
	}
	if md.Version != metadata.Version {
		return errors.New("WAL metadata version does not match version found in WAL file")
	}
	return nil
}

// recover recovers a WAL and returns comitted but not finished updates
func (w *WAL) recover(data []byte) ([]Update, error) {
	// Get all the first pages to sort them by txn number
	var firstPages ByTxnNumber

	// Validate metadata
	if err := readWALMetadata(data[0:]); err != nil {
		return nil, err
	}
	// Starting at index 1 because the first page is reserved for metadata
	for i := 1; int64(i)*pageSize < int64(len(data)); i++ {
		// read the page data
		offset := int64(i) * pageSize

		// unmarshall the page
		var p page
		var nextPage uint64
		var err error
		p.offset = uint64(offset)

		// Check if the
		if offset+pageSize > int64(len(data)) {
			nextPage, err = unmarshalPage(&p, data[offset:])
		} else {
			nextPage, err = unmarshalPage(&p, data[offset:offset+pageSize])
		}
		if err != nil {
			continue
		}

		// If the page is the first page of a transaction remember it
		if p.pageStatus == pageStatusComitted {
			firstPages = append(firstPages, SortPage{p: &p, nextPage: nextPage})
		}
	}

	// Sort the first pages by transaction number
	sort.Sort(firstPages)
	// Recover the transactions in order and get their updates
	updates := []Update{}
	for _, sp := range firstPages {
		var txn Transaction
		err := unmarshalTransaction(&txn, sp.p, sp.nextPage, data)
		if err != nil {
			continue
		}
		updates = append(updates, txn.Updates...)
	}

	// If there were no updates we can savely set the recovery to complete
	if len(updates) == 0 {
		w.recoveryComplete = true
	}

	return updates, nil
}

// RecoveryComplete is called after a wal is recovered to signal that it is
// save to reset the wal
func (w *WAL) RecoveryComplete() error {
	// Marshal the pageStatusApplied
	pageAppliedBytes := make([]byte, 8)
	binary.LittleEndian.PutUint64(pageAppliedBytes, pageStatusApplied)

	// Get the length of the file.
	length, err := w.logFile.Seek(0, io.SeekEnd)
	if err != nil {
		return err
	}

	// Set all pages to applied
	for offset := int64(crypto.HashSize) + pageSize; offset < length; offset += pageSize {
		if _, err := w.logFile.WriteAt(pageAppliedBytes, offset); err != nil {
			return err
		}
	}

	w.recoveryComplete = true
	return nil
}

// managedReservePages reserves pages for a given payload. If it needs to
// allocate new pages it will do so
func (w *WAL) managedReservePages(data []byte) []page {
	// Find out how many pages are needed for the payload
	numPages := uint64(len(data) / maxPayloadSize)
	if len(data)%maxPayloadSize != 0 {
		numPages++
	}

	w.mu.Lock()
	defer w.mu.Unlock()
	// Check if enough pages are available
	if uint64(len(w.availablePages)) < numPages {
		// Add enough pages for the new transaction
		numNewPages := numPages - uint64(len(w.availablePages))
		w.allocatePages(numNewPages)

		// sanity check: the number of available pages should equal the number of required ones
		if uint64(len(w.availablePages)) != numPages {
			panic(errors.New("sanity check failed: num of available pages != num of required pages"))
		}
	}

	// Reserve some pages and remove them from the available ones
	reservedPages := w.availablePages[uint64(len(w.availablePages))-numPages:]
	w.availablePages = w.availablePages[:uint64(len(w.availablePages))-numPages]

	// Set the fields of each page
	pages := make([]page, numPages)
	for i := uint64(0); i < numPages; i++ {
		// Set offset according to the index in reservedPages
		pages[i].offset = reservedPages[i]

		// Set nextPage if the current page isn't the last one
		// otherwise let it be nil
		if i+1 < numPages {
			pages[i].nextPage = &pages[i+1]
		}

		// Set pageStatus of the first page to pageStatusWritten
		if i == 0 {
			pages[i].pageStatus = pageStatusWritten
		} else {
			pages[i].pageStatus = pageStatusOther
		}

		// Copy part of the update into the payload
		payloadsize := maxPayloadSize
		if len(data[i*maxPayloadSize:]) < payloadsize {
			payloadsize = len(data[i*maxPayloadSize:])
		}
		pages[i].payload = make([]byte, payloadsize)
		copy(pages[i].payload, data[i*maxPayloadSize:])
	}

	return pages
}

// UnmarshalPage is a helper function that unmarshals the page and returns the offset of the next one
// Note: setting offset and validating the checksum needs to be handled by the caller
func unmarshalPage(p *page, b []byte) (nextPage uint64, err error) {
	buffer := bytes.NewBuffer(b)

	// Read checksum, pageStatus, transactionNumber and nextPage
	_, err1 := buffer.Read(p.transactionChecksum[:])
	err2 := binary.Read(buffer, binary.LittleEndian, &p.pageStatus)
	err3 := binary.Read(buffer, binary.LittleEndian, &p.transactionNumber)
	err4 := binary.Read(buffer, binary.LittleEndian, &nextPage)

	// Read payloadSize
	var payloadSize uint64
	err5 := binary.Read(buffer, binary.LittleEndian, &payloadSize)

	// Read payload
	p.payload = make([]byte, payloadSize)
	_, err6 := buffer.Read(p.payload[:])

	// Check for errors
	if err1 != nil || err2 != nil || err3 != nil || err4 != nil || err5 != nil || err6 != nil {
		err = errors.Compose(errors.New("Failed to unmarshal wal page"), err1, err2, err3, err4, err5, err6)
		return
	}

	return
}

// unmarshalTransaction unmarshals a transaction starting at the first page
func unmarshalTransaction(txn *Transaction, firstPage *page, nextPageOffset uint64, logData []byte) error {
	// Set the first page of the txn
	txn.firstPage = firstPage

	p := firstPage
	npo := nextPageOffset
	var err error
	var txnPayload []byte
	// Unmarshal the pages in the order of the transaction
	for {
		// Get the payload of each page
		txnPayload = append(txnPayload, p.payload...)

		// Determine if the last page was reached and set the final page of the txn accordingly
		if npo == math.MaxUint64 {
			p.nextPage = nil
			p = nil
			break
		}

		// Set the offset for the next page before npo is overwritten
		var nextPage page
		nextPage.offset = npo

		// Unmarshal the page. The last page might not have pageSize bytes.
		if npo+pageSize > uint64(len(logData)) {
			npo, err = unmarshalPage(&nextPage, logData[npo:])
		} else {
			npo, err = unmarshalPage(&nextPage, logData[npo:npo+pageSize])
		}
		if err != nil {
			return err
		}

		// Move on to the next page
		p.nextPage = &nextPage
		p = &nextPage
	}

	// Verify the checksum before unmarshalling the updates. Otherwise we might
	// get errors later
	if err := txn.validateChecksum(); err != nil {
		return err
	}

	// Restore updates from payload
	if txn.Updates, err = unmarshalUpdates(txnPayload); err != nil {
		return build.ExtendErr("Unable to unmarshal updates", err)
	}

	// Set flags accordingly
	txn.setupComplete = true
	txn.commitComplete = true

	return nil
}

// writeWALMetadata writes WAL metadata to the input file.
func writeWALMetadata(f file) error {
	changeBytes, err := json.MarshalIndent(metadata, "", "\t")
	if err != nil {
		return build.ExtendErr("could not marshal WAL metadata", err)
	}

	// Sanity check. Metadata must not be greater than pageSize
	if len(changeBytes) > pageSize {
		panic("WAL metadata is greater than pageSize")
	}

	_, err = f.WriteAt(changeBytes, 0)
	if err != nil {
		return build.ExtendErr("unable to write WAL metadata", err)
	}
	return nil
}

// Close closes the wal and frees used resources
func (w *WAL) Close() error {
	close(w.stopChan)
	return w.logFile.Close()
}

// New will open a WAL. If the previous run did not shut down cleanly, a set of
// updates will be returned which got committed successfully to the WAL, but
// were never signaled as fully completed.
//
// If no WAL exists, a new one will be created.
//
// If in debugging mode, the WAL may return a series of updates multiple times,
// simulating multiple consecutive unclean shutdowns. If the updates are
// properly idempotent, there should be no functional difference between the
// multiple appearances and them just being loaded a single time correctly.
func New(path string) (u []Update, w *WAL, err error) {
	// Create a wal with production dependencies
	return newWal(path, prodDependencies{})
}
<commit_msg>change wal method name to not be a golang keyword<commit_after>// Package wal defines and implements a general purpose, high performance
// write-ahead-log for performing ACID transactions to disk without sacrificing
// speed or latency more than fundamentally required.
package wal

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"io"
	"math"
	"os"
	"sort"
	"sync"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/persist"
	"github.com/NebulousLabs/errors"
)

// WAL is a general purpose, high performance write-ahead-log for performing
// ACID transactions to disk without sacrificing speed or latency more than
// fundamentally required.
type WAL struct {
	// availablePages lists the offset of file pages which currently have completed or
	// voided updates in them. The pages are in no particular order.
	availablePages []uint64

	// filePageCount indicates the number of pages total in the file. If the
	// number of availablePages ever drops below the number of pages required
	// for a new transaction, then the file is extended, new pages are added,
	// and the availablePages array is updated to include the extended pages.
	filePageCount uint64

	// atomicNextTxnNum is used to give every transaction a unique transaction
	// number. The transaction will then wait until atomicTransactionCounter allows
	// the transaction to be committed. This ensures that transactions are committed
	// in the correct order.
	atomicNextTxnNum uint64

	// logFile contains all of the persistent data associated with the log.
	logFile file

	// path is the path to the underlying logFile
	path string

	// mu is used to lock the availablePages field of the wal
	mu sync.Mutex

	// syncCond is used to schedule the calls to fsync
	syncCond *sync.Cond

	// syncMu is the lock contained in syncCond and must be held before
	// changing the state of the syncCond
	syncMu sync.Mutex

	// syncCount is a counter that indicates how many transactions are
	// currently waiting for a fsync
	syncCount uint64

	// stopChan is a channel that is used to signal a shutdown
	stopChan chan struct{}

	// syncing indicates if the syncing thread is currently being executed
	syncing bool

	// recoveryComplete indicates if the caller signalled that the recovery is complete
	recoveryComplete bool

	// dependencies are used to inject special behaviour into the wal by providing
	// custom dependencies when the wal is created and calling deps.disrupt(setting).
	// The following settings are currently available
	deps dependencies
}

type (
	// SortPage is a helper struct for sorting, that contains a page and the offset
	// of the next page
	SortPage struct {
		p        *page
		nextPage uint64
	}
	// ByTxnNumber is a type that implements the sorting interface for the SortPage struct
	ByTxnNumber []SortPage
)

// Len returns the length of the slice
func (p ByTxnNumber) Len() int {
	return len(p)
}

// Swap swaps two elements of the slice
func (p ByTxnNumber) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

// Less determines if one element of the slice is less than the other
func (p ByTxnNumber) Less(i, j int) bool {
	return p[i].p.transactionNumber < p[j].p.transactionNumber
}

// allocatePages creates new pages and adds them to the available pages of the wal
func (w *WAL) allocatePages(numPages uint64) {
	// Starting at index 1 because the first page is reserved for metadata
	for i := w.filePageCount + 1; i < w.filePageCount+numPages+1; i++ {
		w.availablePages = append(w.availablePages, i*pageSize)
	}
	w.filePageCount += numPages
}

// newWal initializes and returns a wal.
func newWal(path string, deps dependencies) (u []Update, w *WAL, err error) {
	// Create a new WAL
	newWal := WAL{
		deps:     deps,
		stopChan: make(chan struct{}),
		path:     path,
	}

	// Create a condition for the wal
	newWal.syncCond = sync.NewCond(&newWal.syncMu)
	// Try opening the WAL file.
	data, err := deps.readFile(path)
	if err == nil {
		// Recover WAL and return updates
		updates, err := newWal.recoverWal(data)
		if err != nil {
			return nil, nil, err
		}

		// Reuse the existing wal
		newWal.logFile, err = deps.openFile(path, os.O_RDWR, 0600)

		return updates, &newWal, err

	} else if !os.IsNotExist(err) {
		// the file exists but couldn't be opened
		return nil, nil, build.ExtendErr("walFile was not opened successfully", err)
	}

	// Create new empty WAL
	newWal.logFile, err = deps.create(path)
	if err != nil {
		return nil, nil, build.ExtendErr("walFile could not be created", err)
	}

	// Write the metadata to the WAL
	if err = writeWALMetadata(newWal.logFile); err != nil {
		return nil, nil, build.ExtendErr("Failed to write metadata to file", err)
	}

	// When we create a new wal we don't need the caller to signal recoveryComplete
	newWal.recoveryComplete = true
	return nil, &newWal, nil
}

// readWALMetadata reads WAL metadata from the input file, returning an error
// if the result is unexpected.
func readWALMetadata(data []byte) error {
	var md persist.Metadata
	decoder := json.NewDecoder(bytes.NewBuffer(data))
	err := decoder.Decode(&md)
	if err != nil {
		return build.ExtendErr("error reading WAL metadata", err)
	}
	if md.Header != metadata.Header {
		return errors.New("WAL metadata header does not match header found in WAL file")
	}
	if md.Version != metadata.Version {
		return errors.New("WAL metadata version does not match version found in WAL file")
	}
	return nil
}

// recover recovers a WAL and returns comitted but not finished updates
func (w *WAL) recoverWal(data []byte) ([]Update, error) {
	// Get all the first pages to sort them by txn number
	var firstPages ByTxnNumber

	// Validate metadata
	if err := readWALMetadata(data[0:]); err != nil {
		return nil, err
	}
	// Starting at index 1 because the first page is reserved for metadata
	for i := 1; int64(i)*pageSize < int64(len(data)); i++ {
		// read the page data
		offset := int64(i) * pageSize

		// unmarshall the page
		var p page
		var nextPage uint64
		var err error
		p.offset = uint64(offset)

		// Check if the
		if offset+pageSize > int64(len(data)) {
			nextPage, err = unmarshalPage(&p, data[offset:])
		} else {
			nextPage, err = unmarshalPage(&p, data[offset:offset+pageSize])
		}
		if err != nil {
			continue
		}

		// If the page is the first page of a transaction remember it
		if p.pageStatus == pageStatusComitted {
			firstPages = append(firstPages, SortPage{p: &p, nextPage: nextPage})
		}
	}

	// Sort the first pages by transaction number
	sort.Sort(firstPages)
	// Recover the transactions in order and get their updates
	updates := []Update{}
	for _, sp := range firstPages {
		var txn Transaction
		err := unmarshalTransaction(&txn, sp.p, sp.nextPage, data)
		if err != nil {
			continue
		}
		updates = append(updates, txn.Updates...)
	}

	// If there were no updates we can savely set the recovery to complete
	if len(updates) == 0 {
		w.recoveryComplete = true
	}

	return updates, nil
}

// RecoveryComplete is called after a wal is recovered to signal that it is
// save to reset the wal
func (w *WAL) RecoveryComplete() error {
	// Marshal the pageStatusApplied
	pageAppliedBytes := make([]byte, 8)
	binary.LittleEndian.PutUint64(pageAppliedBytes, pageStatusApplied)

	// Get the length of the file.
	length, err := w.logFile.Seek(0, io.SeekEnd)
	if err != nil {
		return err
	}

	// Set all pages to applied.
	for offset := int64(crypto.HashSize) + pageSize; offset < length; offset += pageSize {
		if _, err := w.logFile.WriteAt(pageAppliedBytes, offset); err != nil {
			return err
		}
	}
	// Sync to lock down the obliteration.
	w.logFile.Sync()

	w.recoveryComplete = true
	return nil
}

// managedReservePages reserves pages for a given payload. If it needs to
// allocate new pages it will do so
func (w *WAL) managedReservePages(data []byte) []page {
	// Find out how many pages are needed for the payload
	numPages := uint64(len(data) / maxPayloadSize)
	if len(data)%maxPayloadSize != 0 {
		numPages++
	}

	w.mu.Lock()
	defer w.mu.Unlock()
	// Check if enough pages are available
	if uint64(len(w.availablePages)) < numPages {
		// Add enough pages for the new transaction
		numNewPages := numPages - uint64(len(w.availablePages))
		w.allocatePages(numNewPages)

		// sanity check: the number of available pages should equal the number of required ones
		if uint64(len(w.availablePages)) != numPages {
			panic(errors.New("sanity check failed: num of available pages != num of required pages"))
		}
	}

	// Reserve some pages and remove them from the available ones
	reservedPages := w.availablePages[uint64(len(w.availablePages))-numPages:]
	w.availablePages = w.availablePages[:uint64(len(w.availablePages))-numPages]

	// Set the fields of each page
	pages := make([]page, numPages)
	for i := uint64(0); i < numPages; i++ {
		// Set offset according to the index in reservedPages
		pages[i].offset = reservedPages[i]

		// Set nextPage if the current page isn't the last one
		// otherwise let it be nil
		if i+1 < numPages {
			pages[i].nextPage = &pages[i+1]
		}

		// Set pageStatus of the first page to pageStatusWritten
		if i == 0 {
			pages[i].pageStatus = pageStatusWritten
		} else {
			pages[i].pageStatus = pageStatusOther
		}

		// Copy part of the update into the payload
		payloadsize := maxPayloadSize
		if len(data[i*maxPayloadSize:]) < payloadsize {
			payloadsize = len(data[i*maxPayloadSize:])
		}
		pages[i].payload = make([]byte, payloadsize)
		copy(pages[i].payload, data[i*maxPayloadSize:])
	}

	return pages
}

// UnmarshalPage is a helper function that unmarshals the page and returns the offset of the next one
// Note: setting offset and validating the checksum needs to be handled by the caller
func unmarshalPage(p *page, b []byte) (nextPage uint64, err error) {
	buffer := bytes.NewBuffer(b)

	// Read checksum, pageStatus, transactionNumber and nextPage
	_, err1 := buffer.Read(p.transactionChecksum[:])
	err2 := binary.Read(buffer, binary.LittleEndian, &p.pageStatus)
	err3 := binary.Read(buffer, binary.LittleEndian, &p.transactionNumber)
	err4 := binary.Read(buffer, binary.LittleEndian, &nextPage)

	// Read payloadSize
	var payloadSize uint64
	err5 := binary.Read(buffer, binary.LittleEndian, &payloadSize)

	// Read payload
	p.payload = make([]byte, payloadSize)
	_, err6 := buffer.Read(p.payload[:])

	// Check for errors
	if err1 != nil || err2 != nil || err3 != nil || err4 != nil || err5 != nil || err6 != nil {
		err = errors.Compose(errors.New("Failed to unmarshal wal page"), err1, err2, err3, err4, err5, err6)
		return
	}

	return
}

// unmarshalTransaction unmarshals a transaction starting at the first page
func unmarshalTransaction(txn *Transaction, firstPage *page, nextPageOffset uint64, logData []byte) error {
	// Set the first page of the txn
	txn.firstPage = firstPage

	p := firstPage
	npo := nextPageOffset
	var err error
	var txnPayload []byte
	// Unmarshal the pages in the order of the transaction
	for {
		// Get the payload of each page
		txnPayload = append(txnPayload, p.payload...)

		// Determine if the last page was reached and set the final page of the txn accordingly
		if npo == math.MaxUint64 {
			p.nextPage = nil
			p = nil
			break
		}

		// Set the offset for the next page before npo is overwritten
		var nextPage page
		nextPage.offset = npo

		// Unmarshal the page. The last page might not have pageSize bytes.
		if npo+pageSize > uint64(len(logData)) {
			npo, err = unmarshalPage(&nextPage, logData[npo:])
		} else {
			npo, err = unmarshalPage(&nextPage, logData[npo:npo+pageSize])
		}
		if err != nil {
			return err
		}

		// Move on to the next page
		p.nextPage = &nextPage
		p = &nextPage
	}

	// Verify the checksum before unmarshalling the updates. Otherwise we might
	// get errors later
	if err := txn.validateChecksum(); err != nil {
		return err
	}

	// Restore updates from payload
	if txn.Updates, err = unmarshalUpdates(txnPayload); err != nil {
		return build.ExtendErr("Unable to unmarshal updates", err)
	}

	// Set flags accordingly
	txn.setupComplete = true
	txn.commitComplete = true

	return nil
}

// writeWALMetadata writes WAL metadata to the input file.
func writeWALMetadata(f file) error {
	changeBytes, err := json.MarshalIndent(metadata, "", "\t")
	if err != nil {
		return build.ExtendErr("could not marshal WAL metadata", err)
	}

	// Sanity check. Metadata must not be greater than pageSize
	if len(changeBytes) > pageSize {
		panic("WAL metadata is greater than pageSize")
	}

	_, err = f.WriteAt(changeBytes, 0)
	if err != nil {
		return build.ExtendErr("unable to write WAL metadata", err)
	}
	return nil
}

// Close closes the wal and frees used resources
func (w *WAL) Close() error {
	close(w.stopChan)
	return w.logFile.Close()
}

// New will open a WAL. If the previous run did not shut down cleanly, a set of
// updates will be returned which got committed successfully to the WAL, but
// were never signaled as fully completed.
//
// If no WAL exists, a new one will be created.
//
// If in debugging mode, the WAL may return a series of updates multiple times,
// simulating multiple consecutive unclean shutdowns. If the updates are
// properly idempotent, there should be no functional difference between the
// multiple appearances and them just being loaded a single time correctly.
func New(path string) (u []Update, w *WAL, err error) {
	// Create a wal with production dependencies
	return newWal(path, prodDependencies{})
}
<|endoftext|>


<commit_before>package exec

import (
	"sync"
	"time"

	"github.com/docker/docker/daemon/execdriver"
	derr "github.com/docker/docker/errors"
	"github.com/docker/docker/pkg/stringid"
	"github.com/docker/docker/runconfig"
)

// Config holds the configurations for execs. The Daemon keeps
// track of both running and finished execs so that they can be
// examined both during and after completion.
type Config struct {
	sync.Mutex
	*runconfig.StreamConfig
	ID            string
	Running       bool
	ExitCode      *int
	ProcessConfig *execdriver.ProcessConfig
	OpenStdin     bool
	OpenStderr    bool
	OpenStdout    bool
	CanRemove     bool
	ContainerID   string
	DetachKeys    []byte

	// waitStart will be closed immediately after the exec is really started.
	waitStart chan struct{}
}

// NewConfig initializes the a new exec configuration
func NewConfig() *Config {
	return &Config{
		ID:           stringid.GenerateNonCryptoID(),
		StreamConfig: runconfig.NewStreamConfig(),
		waitStart:    make(chan struct{}),
	}
}

// Store keeps track of the exec configurations.
type Store struct {
	commands map[string]*Config
	sync.RWMutex
}

// NewStore initializes a new exec store.
func NewStore() *Store {
	return &Store{commands: make(map[string]*Config, 0)}
}

// Commands returns the exec configurations in the store.
func (e *Store) Commands() map[string]*Config {
	return e.commands
}

// Add adds a new exec configuration to the store.
func (e *Store) Add(id string, Config *Config) {
	e.Lock()
	e.commands[id] = Config
	e.Unlock()
}

// Get returns an exec configuration by its id.
func (e *Store) Get(id string) *Config {
	e.RLock()
	res := e.commands[id]
	e.RUnlock()
	return res
}

// Delete removes an exec configuration from the store.
func (e *Store) Delete(id string) {
	e.Lock()
	delete(e.commands, id)
	e.Unlock()
}

// List returns the list of exec ids in the store.
func (e *Store) List() []string {
	var IDs []string
	e.RLock()
	for id := range e.commands {
		IDs = append(IDs, id)
	}
	e.RUnlock()
	return IDs
}

// Wait waits until the exec process finishes or there is an error in the error channel.
func (c *Config) Wait(cErr chan error) error {
	// Exec should not return until the process is actually running
	select {
	case <-c.waitStart:
	case err := <-cErr:
		return err
	}
	return nil
}

// Close closes the wait channel for the progress.
func (c *Config) Close() {
	close(c.waitStart)
}

// Resize changes the size of the terminal for the exec process.
func (c *Config) Resize(h, w int) error {
	select {
	case <-c.waitStart:
	case <-time.After(time.Second):
		return derr.ErrorCodeExecResize.WithArgs(c.ID)
	}
	return c.ProcessConfig.Terminal.Resize(h, w)
}
<commit_msg>Fix race condition in execCommandGC<commit_after>package exec

import (
	"sync"
	"time"

	"github.com/docker/docker/daemon/execdriver"
	derr "github.com/docker/docker/errors"
	"github.com/docker/docker/pkg/stringid"
	"github.com/docker/docker/runconfig"
)

// Config holds the configurations for execs. The Daemon keeps
// track of both running and finished execs so that they can be
// examined both during and after completion.
type Config struct {
	sync.Mutex
	*runconfig.StreamConfig
	ID            string
	Running       bool
	ExitCode      *int
	ProcessConfig *execdriver.ProcessConfig
	OpenStdin     bool
	OpenStderr    bool
	OpenStdout    bool
	CanRemove     bool
	ContainerID   string
	DetachKeys    []byte

	// waitStart will be closed immediately after the exec is really started.
	waitStart chan struct{}
}

// NewConfig initializes the a new exec configuration
func NewConfig() *Config {
	return &Config{
		ID:           stringid.GenerateNonCryptoID(),
		StreamConfig: runconfig.NewStreamConfig(),
		waitStart:    make(chan struct{}),
	}
}

// Store keeps track of the exec configurations.
type Store struct {
	commands map[string]*Config
	sync.RWMutex
}

// NewStore initializes a new exec store.
func NewStore() *Store {
	return &Store{commands: make(map[string]*Config, 0)}
}

// Commands returns the exec configurations in the store.
func (e *Store) Commands() map[string]*Config {
	e.RLock()
	commands := make(map[string]*Config, len(e.commands))
	for id, config := range e.commands {
		commands[id] = config
	}
	e.RUnlock()
	return commands
}

// Add adds a new exec configuration to the store.
func (e *Store) Add(id string, Config *Config) {
	e.Lock()
	e.commands[id] = Config
	e.Unlock()
}

// Get returns an exec configuration by its id.
func (e *Store) Get(id string) *Config {
	e.RLock()
	res := e.commands[id]
	e.RUnlock()
	return res
}

// Delete removes an exec configuration from the store.
func (e *Store) Delete(id string) {
	e.Lock()
	delete(e.commands, id)
	e.Unlock()
}

// List returns the list of exec ids in the store.
func (e *Store) List() []string {
	var IDs []string
	e.RLock()
	for id := range e.commands {
		IDs = append(IDs, id)
	}
	e.RUnlock()
	return IDs
}

// Wait waits until the exec process finishes or there is an error in the error channel.
func (c *Config) Wait(cErr chan error) error {
	// Exec should not return until the process is actually running
	select {
	case <-c.waitStart:
	case err := <-cErr:
		return err
	}
	return nil
}

// Close closes the wait channel for the progress.
func (c *Config) Close() {
	close(c.waitStart)
}

// Resize changes the size of the terminal for the exec process.
func (c *Config) Resize(h, w int) error {
	select {
	case <-c.waitStart:
	case <-time.After(time.Second):
		return derr.ErrorCodeExecResize.WithArgs(c.ID)
	}
	return c.ProcessConfig.Terminal.Resize(h, w)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/sjoerdsimons/fakemachine"
	"path"
)

type RunAction struct {
	*BaseAction
	Chroot  bool
	Script  string
	Command string
}

func (run *RunAction) PreMachine(context *YaibContext, m *fakemachine.Machine,
	args *[]string) {

	if run.Script == "" {
		return
	}

	run.Script = CleanPathAt(run.Script, context.recipeDir)
	m.AddVolume(path.Dir(run.Script))
}

func (run *RunAction) Run(context *YaibContext) {
	var command string

	run.Script = CleanPathAt(run.Script, context.recipeDir)

	if run.Script != "" {
		if run.Chroot {
			command = fmt.Sprintf("/script/%s", path.Base(run.Script))
		} else {
			command = run.Script
		}
	} else {
		command = run.Command
	}

	var err error
	if run.Chroot {
		if run.Script != "" {
			q := NewQemuHelper(*context)
			q.Setup()
			defer q.Cleanup()

			options := []string{"-q", "-D", context.rootdir}
			options = append(options, "--bind", fmt.Sprintf("%s:/script",
				path.Dir(run.Script)))
			options = append(options, command)
			RunCommand(path.Base(run.Script), "systemd-nspawn", options...)
		} else {
			err = RunCommandInChroot(*context, command, "sh", "-c", command)
		}
	} else {
		command = command + " " + context.rootdir
		RunCommand(path.Base(command), "sh", "-c", command)
	}

	if err != nil {
		panic(err)
	}
}
<commit_msg>Fix command runs<commit_after>package main

import (
	"fmt"
	"github.com/sjoerdsimons/fakemachine"
	"path"
)

type RunAction struct {
	*BaseAction
	Chroot  bool
	Script  string
	Command string
}

func (run *RunAction) PreMachine(context *YaibContext, m *fakemachine.Machine,
	args *[]string) {

	if run.Script == "" {
		return
	}

	run.Script = CleanPathAt(run.Script, context.recipeDir)
	m.AddVolume(path.Dir(run.Script))
}

func (run *RunAction) Run(context *YaibContext) {
	var command string

	if run.Script != "" {
		run.Script = CleanPathAt(run.Script, context.recipeDir)
		if run.Chroot {
			command = fmt.Sprintf("/script/%s", path.Base(run.Script))
		} else {
			command = run.Script
		}
	} else {
		command = run.Command
	}

	var err error
	if run.Chroot {
		if run.Script != "" {
			q := NewQemuHelper(*context)
			q.Setup()
			defer q.Cleanup()

			options := []string{"-q", "-D", context.rootdir}
			options = append(options, "--bind", fmt.Sprintf("%s:/script",
				path.Dir(run.Script)))
			options = append(options, command)
			RunCommand(path.Base(run.Script), "systemd-nspawn", options...)
		} else {
			err = RunCommandInChroot(*context, command, "sh", "-c", command)
		}
	} else {
		command = command + " " + context.rootdir
		RunCommand(path.Base(command), "sh", "-c", command)
	}

	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/AdRoll/goamz/aws"
	"github.com/AdRoll/goamz/kinesis"
	"github.com/AdRoll/goamz/sns"
	_ "github.com/lib/pq"
)

type dbInfo struct {
	Host     string
	Database string
	Username string
	Password string
}

type topicMessage struct {
	TopicID int    `json:"topic_id"`
	Message string `json:"message"`
}

type topicPageMessage struct {
	TopicID  int    `json:"topic_id"`
	Message  string `json:"message"`
	PageNum  int    `json:"page_num"`
	LastPage bool   `json:"last_page"`
}

func main() {
	pub, secret, _ := getSettings()
	auth := aws.Auth{AccessKey: pub, SecretKey: secret}
	K := kinesis.New(auth, aws.USEast)
	for k, v := range os.Args {
		log.Println(k, v)
	}
	log.Println("-----")
	if len(os.Args) > 1 {
		rawData := os.Args[1]

		var kpayload KinesisPayload
		err := json.Unmarshal([]byte(rawData), &kpayload)
		if err != nil {
			log.Println("Error Kinesisis Payload:", err)
		}
		for _, v := range kpayload.Records {
			log.Println("Record:", v)
			var tm topicMessage
			sDec, errDec := base64.StdEncoding.DecodeString(v.Kinesis.Data)
			if errDec != nil {
				log.Println("Error:", errDec)
			} else {
				log.Println(string(sDec))
				errJSON := json.Unmarshal(sDec, &tm)
				if errJSON != nil {
					log.Println("Error:", errJSON)
					return
				}
				pageCount := getDevicesByTopicIDPageCount(tm.TopicID)
				publishMessage("Count: 1st page sent")
				for i := 0; i < pageCount; i++ {
					tpm := topicPageMessage{}
					tpm.Message = tm.Message
					tpm.TopicID = tm.TopicID
					tpm.PageNum = i + 1
					if i+1 == pageCount {
						tpm.LastPage = true
						log.Println("LastPage is", i+1)
					}
					jsonData, jsonErr := json.Marshal(&tpm)
					if jsonErr != nil {
						log.Println("Error:", jsonErr)
						break
					}
					_, err := K.PutRecord("topic-message-page", "topic-page", jsonData, "", "")
					if tpm.LastPage {
						publishMessage(fmt.Sprintf("Count: Last Page Sent: %v", pageCount))
					}
					if err != nil {
						log.Println("Error:", err)
					}
					log.Println("Sent...", tpm)
				}
				log.Println("All done!")
			}
		}
		return

	}
	log.Println("Error: os.Args was 1 length.")
}

func getDevicesByTopicIDPageCount(topicID int) int {
	var count int
	info := getDBSettings()
	db, errCon := sql.Open("postgres", fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=require", info.Host, info.Username, info.Password, info.Database))
	defer db.Close()
	if errCon != nil {
		log.Fatal(errCon)
	}
	rows, err := db.Query(`
		select
			((count(u.endpointarn)/10000)+1) as pagecoun
		from
			subscription s ,userdevices u
		where
			s.topicid= $1 and
			s.userID=u.userid

;`, topicID)
	if err != nil {
		panic(err)
	}
	rows.Next()
	errScan := rows.Scan(&count)
	if errScan != nil {
		panic(errScan)
	}
	return count
}

func getSettings() (string, string, error) {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		return "", "", nil
	}
	settingsMap := make(map[string]string)
	json.Unmarshal(file, &settingsMap)
	return settingsMap["Access"], settingsMap["Secret"], nil
}

func getDBSettings() *dbInfo {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		log.Println("Error:", err)
		return nil
	}
	db := dbInfo{}
	err2 := json.Unmarshal(file, &db)
	if err2 != nil {
		log.Println("Error:", err2)
		return nil
	}
	return &db
}

type KinesisPayload struct {
	Records []struct {
		AwsRegion         string `json:"awsRegion"`
		EventID           string `json:"eventID"`
		EventName         string `json:"eventName"`
		EventSource       string `json:"eventSource"`
		EventSourceARN    string `json:"eventSourceARN"`
		EventVersion      string `json:"eventVersion"`
		InvokeIdentityArn string `json:"invokeIdentityArn"`
		Kinesis           struct {
			Data                 string `json:"data"`
			KinesisSchemaVersion string `json:"kinesisSchemaVersion"`
			PartitionKey         string `json:"partitionKey"`
			SequenceNumber       string `json:"sequenceNumber"`
		} `json:"kinesis"`
	} `json:"Records"`
}

func publishMessage(msg string) error {
	topicarn, topicErr := getTopicArn()
	if topicErr != nil {
		log.Println(topicErr)
		return topicErr
	}
	p, s, _ := getSettings()
	auth := aws.Auth{AccessKey: p, SecretKey: s}
	region := aws.Region{}
	region.Name = "us-east-1"
	region.SNSEndpoint = "http://sns.us-east-1.amazonaws.com"
	awssns, _ := sns.New(auth, region)
	if awssns == nil {
		return fmt.Errorf("Can't get sns reference for %v %v", auth, region)
	}
	opt := sns.PublishOptions{}
	opt.TopicArn = topicarn
	opt.Message = msg
	opt.Subject = msg
	_, pubErr := awssns.Publish(&opt)
	if pubErr != nil {
		return pubErr
	}
	return nil
}

func getTopicArn() (string, error) {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		return "", err
	}
	settingsMap := make(map[string]string)
	json.Unmarshal(file, &settingsMap)
	return settingsMap["Topicarn"], nil
}
<commit_msg>pull page count from db<commit_after>package main

import (
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"

	"github.com/AdRoll/goamz/aws"
	"github.com/AdRoll/goamz/kinesis"
	"github.com/AdRoll/goamz/sns"
	_ "github.com/lib/pq"
)

type dbInfo struct {
	Host     string
	Database string
	Username string
	Password string
}

type topicMessage struct {
	TopicID int    `json:"topic_id"`
	Message string `json:"message"`
}

type topicPageMessage struct {
	TopicID  int    `json:"topic_id"`
	Message  string `json:"message"`
	PageNum  int    `json:"page_num"`
	LastPage bool   `json:"last_page"`
}

func main() {
	pub, secret, _ := getSettings()
	auth := aws.Auth{AccessKey: pub, SecretKey: secret}
	K := kinesis.New(auth, aws.USEast)
	for k, v := range os.Args {
		log.Println(k, v)
	}
	log.Println("-----")
	if len(os.Args) > 1 {
		rawData := os.Args[1]

		var kpayload KinesisPayload
		err := json.Unmarshal([]byte(rawData), &kpayload)
		if err != nil {
			log.Println("Error Kinesisis Payload:", err)
		}
		for _, v := range kpayload.Records {
			log.Println("Record:", v)
			var tm topicMessage
			sDec, errDec := base64.StdEncoding.DecodeString(v.Kinesis.Data)
			if errDec != nil {
				log.Println("Error:", errDec)
			} else {
				log.Println(string(sDec))
				errJSON := json.Unmarshal(sDec, &tm)
				if errJSON != nil {
					log.Println("Error:", errJSON)
					return
				}
				pageCount := getDevicesByTopicIDPageCount(tm.TopicID)
				publishMessage("Count: 1st page sent")
				for i := 0; i < pageCount; i++ {
					tpm := topicPageMessage{}
					tpm.Message = tm.Message
					tpm.TopicID = tm.TopicID
					tpm.PageNum = i + 1
					if i+1 == pageCount {
						tpm.LastPage = true
						log.Println("LastPage is", i+1)
					}
					jsonData, jsonErr := json.Marshal(&tpm)
					if jsonErr != nil {
						log.Println("Error:", jsonErr)
						break
					}
					_, err := K.PutRecord("topic-message-page", "topic-page", jsonData, "", "")
					if tpm.LastPage {
						publishMessage(fmt.Sprintf("Count: Last Page Sent: %v", pageCount))
					}
					if err != nil {
						log.Println("Error:", err)
					}
					log.Println("Sent...", tpm)
				}
				log.Println("All done!")
			}
		}
		return

	}
	log.Println("Error: os.Args was 1 length.")
}

func getDevicesByTopicIDPageCount(topicID int) int {
	var count int
	info := getDBSettings()
	db, errCon := sql.Open("postgres", fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=require", info.Host, info.Username, info.Password, info.Database))
	defer db.Close()
	if errCon != nil {
		log.Fatal(errCon)
	}
	rows, err := db.Query(`
		select
			((count(u.endpointarn)/10000)+1) as pagecoun
		from
			subscription s ,userdevices u
		where
			s.topicid= $1 and
			s.userID=u.userid
;`, topicID)
	if err != nil {
		panic(err)
	}
	rows.Next()
	errScan := rows.Scan(&count)
	if errScan != nil {
		panic(errScan)
	}
	dbcount := GetPageCountFromDB()
	if dbcount > 0 {
		return dbcount
	}
	return count
}

func getSettings() (string, string, error) {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		return "", "", nil
	}
	settingsMap := make(map[string]string)
	json.Unmarshal(file, &settingsMap)
	return settingsMap["Access"], settingsMap["Secret"], nil
}

func GetPageCountFromDB() int {
	info := getDBSettings()
	db, errCon := sql.Open("postgres", fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=require", info.Host, info.Username, info.Password, info.Database))
	defer db.Close()
	if errCon != nil {
		log.Fatal(errCon)
	}
	rows, err := db.Query(`
		select
			s.value
		from
			lambdasettings s
		where
			s.name= 'pagecount'
	;`)
	if err != nil {
		panic(err)
	}
	for rows.Next() {
		var strval string
		errScan := rows.Scan(&strval)
		if errScan != nil {
			panic(errScan)
		}
		bc, cErr := strconv.Atoi(strval)
		if cErr != nil {
			panic(cErr)
		}
		return bc
	}
	return 0
}

func getDBSettings() *dbInfo {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		log.Println("Error:", err)
		return nil
	}
	db := dbInfo{}
	err2 := json.Unmarshal(file, &db)
	if err2 != nil {
		log.Println("Error:", err2)
		return nil
	}
	return &db
}

type KinesisPayload struct {
	Records []struct {
		AwsRegion         string `json:"awsRegion"`
		EventID           string `json:"eventID"`
		EventName         string `json:"eventName"`
		EventSource       string `json:"eventSource"`
		EventSourceARN    string `json:"eventSourceARN"`
		EventVersion      string `json:"eventVersion"`
		InvokeIdentityArn string `json:"invokeIdentityArn"`
		Kinesis           struct {
			Data                 string `json:"data"`
			KinesisSchemaVersion string `json:"kinesisSchemaVersion"`
			PartitionKey         string `json:"partitionKey"`
			SequenceNumber       string `json:"sequenceNumber"`
		} `json:"kinesis"`
	} `json:"Records"`
}

func publishMessage(msg string) error {
	topicarn, topicErr := getTopicArn()
	if topicErr != nil {
		log.Println(topicErr)
		return topicErr
	}
	p, s, _ := getSettings()
	auth := aws.Auth{AccessKey: p, SecretKey: s}
	region := aws.Region{}
	region.Name = "us-east-1"
	region.SNSEndpoint = "http://sns.us-east-1.amazonaws.com"
	awssns, _ := sns.New(auth, region)
	if awssns == nil {
		return fmt.Errorf("Can't get sns reference for %v %v", auth, region)
	}
	opt := sns.PublishOptions{}
	opt.TopicArn = topicarn
	opt.Message = msg
	opt.Subject = msg
	_, pubErr := awssns.Publish(&opt)
	if pubErr != nil {
		return pubErr
	}
	return nil
}

func getTopicArn() (string, error) {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		return "", err
	}
	settingsMap := make(map[string]string)
	json.Unmarshal(file, &settingsMap)
	return settingsMap["Topicarn"], nil
}
<|endoftext|>


<commit_before>package pwd

import (
	"fmt"
	"log"
	"math"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/play-with-docker/play-with-docker/config"
	"github.com/play-with-docker/play-with-docker/docker"
	"github.com/play-with-docker/play-with-docker/pwd/types"
	"github.com/twinj/uuid"
)

type sessionBuilderWriter struct {
	sessionId string
	broadcast BroadcastApi
}

func (s *sessionBuilderWriter) Write(p []byte) (n int, err error) {
	s.broadcast.BroadcastTo(s.sessionId, "session builder out", string(p))
	return len(p), nil
}

type SessionSetupConf struct {
	Instances []SessionSetupInstanceConf `json:"instances"`
}

type SessionSetupInstanceConf struct {
	Image          string `json:"image"`
	Hostname       string `json:"hostname"`
	IsSwarmManager bool   `json:"is_swarm_manager"`
	IsSwarmWorker  bool   `json:"is_swarm_worker"`
}

func (p *pwd) SessionNew(duration time.Duration, stack, stackName, imageName string) (*types.Session, error) {
	defer observeAction("SessionNew", time.Now())

	s := &types.Session{}
	s.Id = uuid.NewV4().String()
	s.Instances = map[string]*types.Instance{}
	s.CreatedAt = time.Now()
	s.ExpiresAt = s.CreatedAt.Add(duration)
	s.Ready = true
	s.Stack = stack

	if s.Stack != "" {
		s.Ready = false
	}
	if stackName == "" {
		stackName = "pwd"
	}
	s.StackName = stackName
	s.ImageName = imageName

	log.Printf("NewSession id=[%s]\n", s.Id)

	if err := p.docker.CreateNetwork(s.Id); err != nil {
		log.Println("ERROR NETWORKING")
		return nil, err
	}
	log.Printf("Network [%s] created for session [%s]\n", s.Id, s.Id)

	if err := p.prepareSession(s); err != nil {
		log.Println(err)
		return nil, err
	}

	if err := p.storage.SessionPut(s); err != nil {
		log.Println(err)
		return nil, err
	}

	p.setGauges()

	return s, nil
}

func (p *pwd) SessionClose(s *types.Session) error {
	defer observeAction("SessionClose", time.Now())

	s.Lock()
	defer s.Unlock()

	s.StopTicker()

	p.broadcast.BroadcastTo(s.Id, "session end")
	p.broadcast.BroadcastTo(s.Id, "disconnect")
	log.Printf("Starting clean up of session [%s]\n", s.Id)
	for _, i := range s.Instances {
		err := p.InstanceDelete(s, i)
		if err != nil {
			log.Println(err)
			return err
		}
	}
	// Disconnect PWD daemon from the network
	if err := p.docker.DisconnectNetwork(config.PWDContainerName, s.Id); err != nil {
		if !strings.Contains(err.Error(), "is not connected to the network") {
			log.Println("ERROR NETWORKING")
			return err
		}
	}
	log.Printf("Disconnected pwd from network [%s]\n", s.Id)
	if err := p.docker.DeleteNetwork(s.Id); err != nil {
		if !strings.Contains(err.Error(), "not found") {
			log.Println(err)
			return err
		}
	}

	err := p.storage.SessionDelete(s.Id)
	if err != nil {
		return err
	}

	log.Printf("Cleaned up session [%s]\n", s.Id)
	p.setGauges()
	return nil

}

func (p *pwd) SessionGetSmallestViewPort(s *types.Session) types.ViewPort {
	defer observeAction("SessionGetSmallestViewPort", time.Now())

	minRows := s.Clients[0].ViewPort.Rows
	minCols := s.Clients[0].ViewPort.Cols

	for _, c := range s.Clients {
		minRows = uint(math.Min(float64(minRows), float64(c.ViewPort.Rows)))
		minCols = uint(math.Min(float64(minCols), float64(c.ViewPort.Cols)))
	}

	return types.ViewPort{Rows: minRows, Cols: minCols}
}

func (p *pwd) SessionDeployStack(s *types.Session) error {
	defer observeAction("SessionDeployStack", time.Now())

	if s.Ready {
		// a stack was already deployed on this session, just ignore
		return nil
	}

	s.Ready = false
	p.broadcast.BroadcastTo(s.Id, "session ready", false)
	i, err := p.InstanceNew(s, InstanceConfig{ImageName: s.ImageName, Host: s.Host})
	if err != nil {
		log.Printf("Error creating instance for stack [%s]: %s\n", s.Stack, err)
		return err
	}

	_, fileName := filepath.Split(s.Stack)
	err = p.InstanceUploadFromUrl(i, fileName, "", s.Stack)
	if err != nil {
		log.Printf("Error uploading stack file [%s]: %s\n", s.Stack, err)
		return err
	}

	fileName = path.Base(s.Stack)
	file := fmt.Sprintf("/var/run/pwd/uploads/%s", fileName)
	cmd := fmt.Sprintf("docker swarm init --advertise-addr eth0 && docker-compose -f %s pull && docker stack deploy -c %s %s", file, file, s.StackName)

	w := sessionBuilderWriter{sessionId: s.Id, broadcast: p.broadcast}
	code, err := p.docker.ExecAttach(i.Name, []string{"sh", "-c", cmd}, &w)
	if err != nil {
		log.Printf("Error executing stack [%s]: %s\n", s.Stack, err)
		return err
	}

	log.Printf("Stack execution finished with code %d\n", code)
	s.Ready = true
	p.broadcast.BroadcastTo(s.Id, "session ready", true)
	if err := p.storage.SessionPut(s); err != nil {
		return err
	}
	return nil
}

func (p *pwd) SessionGet(sessionId string) *types.Session {
	defer observeAction("SessionGet", time.Now())

	s, _ := p.storage.SessionGet(sessionId)

	if err := p.prepareSession(s); err != nil {
		log.Println(err)
		return nil
	}
	return s
}

func (p *pwd) SessionSetup(session *types.Session, conf SessionSetupConf) error {
	defer observeAction("SessionSetup", time.Now())
	var tokens *docker.SwarmTokens = nil
	var firstSwarmManager *types.Instance = nil

	// first look for a swarm manager and create it
	for _, conf := range conf.Instances {
		if conf.IsSwarmManager {
			instanceConf := InstanceConfig{
				ImageName: conf.Image,
				Hostname:  conf.Hostname,
				Host:      session.Host,
			}
			i, err := p.InstanceNew(session, instanceConf)
			if err != nil {
				return err
			}
			if i.Docker == nil {
				dock, err := p.docker.New(i.IP, i.Cert, i.Key)
				if err != nil {
					return err
				}
				i.Docker = dock
			}
			tkns, err := i.Docker.SwarmInit()
			if err != nil {
				return err
			}
			tokens = tkns
			firstSwarmManager = i
			break
		}
	}

	// now create the rest in parallel

	wg := sync.WaitGroup{}
	for _, c := range conf.Instances {
		if firstSwarmManager != nil && c.Hostname != firstSwarmManager.Hostname {
			wg.Add(1)
			go func(c SessionSetupInstanceConf) {
				defer wg.Done()
				instanceConf := InstanceConfig{
					ImageName: c.Image,
					Hostname:  c.Hostname,
				}
				i, err := p.InstanceNew(session, instanceConf)
				if err != nil {
					log.Println(err)
					return
				}
				if c.IsSwarmManager || c.IsSwarmWorker {
					// check if we have connection to the daemon, if not, create it
					if i.Docker == nil {
						dock, err := p.docker.New(i.IP, i.Cert, i.Key)
						if err != nil {
							log.Println(err)
							return
						}
						i.Docker = dock
					}
				}

				if firstSwarmManager != nil {
					if c.IsSwarmManager {
						// this is a swarm manager
						// cluster has already been initiated, join as manager
						err := i.Docker.SwarmJoin(fmt.Sprintf("%s:2377", firstSwarmManager.IP), tokens.Manager)
						if err != nil {
							log.Println(err)
							return
						}
					}
					if c.IsSwarmWorker {
						// this is a swarm worker
						err := i.Docker.SwarmJoin(fmt.Sprintf("%s:2377", firstSwarmManager.IP), tokens.Worker)
						if err != nil {
							log.Println(err)
							return
						}
					}
				}
			}(c)
		}
	}
	wg.Wait()

	return nil
}

// This function should be called any time a session needs to be prepared:
// 1. Like when it is created
// 2. When it was loaded from storage
func (p *pwd) prepareSession(session *types.Session) error {
	session.Lock()
	defer session.Unlock()

	if session.IsPrepared() {
		return nil
	}

	p.scheduleSessionClose(session)

	// Connect PWD daemon to the new network
	if err := p.connectToNetwork(session); err != nil {
		return err
	}

	// Schedule periodic tasks
	p.tasks.Schedule(session)

	for _, i := range session.Instances {
		// wire the session back to the instance
		i.Session = session
		go p.InstanceAttachTerminal(i)
	}
	session.SetPrepared()

	return nil
}

func (p *pwd) scheduleSessionClose(s *types.Session) {
	timeLeft := s.ExpiresAt.Sub(time.Now())
	s.SetClosingTimer(time.AfterFunc(timeLeft, func() {
		p.SessionClose(s)
	}))
}

func (p *pwd) connectToNetwork(s *types.Session) error {
	ip, err := p.docker.ConnectNetwork(config.PWDContainerName, s.Id, s.PwdIpAddress)
	if err != nil {
		log.Println("ERROR NETWORKING")
		return err
	}
	s.PwdIpAddress = ip
	log.Printf("Connected %s to network [%s]\n", config.PWDContainerName, s.Id)
	return nil
}
<commit_msg>Fix path for stack files<commit_after>package pwd

import (
	"fmt"
	"log"
	"math"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/play-with-docker/play-with-docker/config"
	"github.com/play-with-docker/play-with-docker/docker"
	"github.com/play-with-docker/play-with-docker/pwd/types"
	"github.com/twinj/uuid"
)

type sessionBuilderWriter struct {
	sessionId string
	broadcast BroadcastApi
}

func (s *sessionBuilderWriter) Write(p []byte) (n int, err error) {
	s.broadcast.BroadcastTo(s.sessionId, "session builder out", string(p))
	return len(p), nil
}

type SessionSetupConf struct {
	Instances []SessionSetupInstanceConf `json:"instances"`
}

type SessionSetupInstanceConf struct {
	Image          string `json:"image"`
	Hostname       string `json:"hostname"`
	IsSwarmManager bool   `json:"is_swarm_manager"`
	IsSwarmWorker  bool   `json:"is_swarm_worker"`
}

func (p *pwd) SessionNew(duration time.Duration, stack, stackName, imageName string) (*types.Session, error) {
	defer observeAction("SessionNew", time.Now())

	s := &types.Session{}
	s.Id = uuid.NewV4().String()
	s.Instances = map[string]*types.Instance{}
	s.CreatedAt = time.Now()
	s.ExpiresAt = s.CreatedAt.Add(duration)
	s.Ready = true
	s.Stack = stack

	if s.Stack != "" {
		s.Ready = false
	}
	if stackName == "" {
		stackName = "pwd"
	}
	s.StackName = stackName
	s.ImageName = imageName

	log.Printf("NewSession id=[%s]\n", s.Id)

	if err := p.docker.CreateNetwork(s.Id); err != nil {
		log.Println("ERROR NETWORKING")
		return nil, err
	}
	log.Printf("Network [%s] created for session [%s]\n", s.Id, s.Id)

	if err := p.prepareSession(s); err != nil {
		log.Println(err)
		return nil, err
	}

	if err := p.storage.SessionPut(s); err != nil {
		log.Println(err)
		return nil, err
	}

	p.setGauges()

	return s, nil
}

func (p *pwd) SessionClose(s *types.Session) error {
	defer observeAction("SessionClose", time.Now())

	s.Lock()
	defer s.Unlock()

	s.StopTicker()

	p.broadcast.BroadcastTo(s.Id, "session end")
	p.broadcast.BroadcastTo(s.Id, "disconnect")
	log.Printf("Starting clean up of session [%s]\n", s.Id)
	for _, i := range s.Instances {
		err := p.InstanceDelete(s, i)
		if err != nil {
			log.Println(err)
			return err
		}
	}
	// Disconnect PWD daemon from the network
	if err := p.docker.DisconnectNetwork(config.PWDContainerName, s.Id); err != nil {
		if !strings.Contains(err.Error(), "is not connected to the network") {
			log.Println("ERROR NETWORKING")
			return err
		}
	}
	log.Printf("Disconnected pwd from network [%s]\n", s.Id)
	if err := p.docker.DeleteNetwork(s.Id); err != nil {
		if !strings.Contains(err.Error(), "not found") {
			log.Println(err)
			return err
		}
	}

	err := p.storage.SessionDelete(s.Id)
	if err != nil {
		return err
	}

	log.Printf("Cleaned up session [%s]\n", s.Id)
	p.setGauges()
	return nil

}

func (p *pwd) SessionGetSmallestViewPort(s *types.Session) types.ViewPort {
	defer observeAction("SessionGetSmallestViewPort", time.Now())

	minRows := s.Clients[0].ViewPort.Rows
	minCols := s.Clients[0].ViewPort.Cols

	for _, c := range s.Clients {
		minRows = uint(math.Min(float64(minRows), float64(c.ViewPort.Rows)))
		minCols = uint(math.Min(float64(minCols), float64(c.ViewPort.Cols)))
	}

	return types.ViewPort{Rows: minRows, Cols: minCols}
}

func (p *pwd) SessionDeployStack(s *types.Session) error {
	defer observeAction("SessionDeployStack", time.Now())

	if s.Ready {
		// a stack was already deployed on this session, just ignore
		return nil
	}

	s.Ready = false
	p.broadcast.BroadcastTo(s.Id, "session ready", false)
	i, err := p.InstanceNew(s, InstanceConfig{ImageName: s.ImageName, Host: s.Host})
	if err != nil {
		log.Printf("Error creating instance for stack [%s]: %s\n", s.Stack, err)
		return err
	}

	_, fileName := filepath.Split(s.Stack)
	err = p.InstanceUploadFromUrl(i, fileName, "/var/run/pwd/uploads", s.Stack)
	if err != nil {
		log.Printf("Error uploading stack file [%s]: %s\n", s.Stack, err)
		return err
	}

	fileName = path.Base(s.Stack)
	file := fmt.Sprintf("/var/run/pwd/uploads/%s", fileName)
	cmd := fmt.Sprintf("docker swarm init --advertise-addr eth0 && docker-compose -f %s pull && docker stack deploy -c %s %s", file, file, s.StackName)

	w := sessionBuilderWriter{sessionId: s.Id, broadcast: p.broadcast}
	code, err := p.docker.ExecAttach(i.Name, []string{"sh", "-c", cmd}, &w)
	if err != nil {
		log.Printf("Error executing stack [%s]: %s\n", s.Stack, err)
		return err
	}

	log.Printf("Stack execution finished with code %d\n", code)
	s.Ready = true
	p.broadcast.BroadcastTo(s.Id, "session ready", true)
	if err := p.storage.SessionPut(s); err != nil {
		return err
	}
	return nil
}

func (p *pwd) SessionGet(sessionId string) *types.Session {
	defer observeAction("SessionGet", time.Now())

	s, _ := p.storage.SessionGet(sessionId)

	if err := p.prepareSession(s); err != nil {
		log.Println(err)
		return nil
	}
	return s
}

func (p *pwd) SessionSetup(session *types.Session, conf SessionSetupConf) error {
	defer observeAction("SessionSetup", time.Now())
	var tokens *docker.SwarmTokens = nil
	var firstSwarmManager *types.Instance = nil

	// first look for a swarm manager and create it
	for _, conf := range conf.Instances {
		if conf.IsSwarmManager {
			instanceConf := InstanceConfig{
				ImageName: conf.Image,
				Hostname:  conf.Hostname,
				Host:      session.Host,
			}
			i, err := p.InstanceNew(session, instanceConf)
			if err != nil {
				return err
			}
			if i.Docker == nil {
				dock, err := p.docker.New(i.IP, i.Cert, i.Key)
				if err != nil {
					return err
				}
				i.Docker = dock
			}
			tkns, err := i.Docker.SwarmInit()
			if err != nil {
				return err
			}
			tokens = tkns
			firstSwarmManager = i
			break
		}
	}

	// now create the rest in parallel

	wg := sync.WaitGroup{}
	for _, c := range conf.Instances {
		if firstSwarmManager != nil && c.Hostname != firstSwarmManager.Hostname {
			wg.Add(1)
			go func(c SessionSetupInstanceConf) {
				defer wg.Done()
				instanceConf := InstanceConfig{
					ImageName: c.Image,
					Hostname:  c.Hostname,
				}
				i, err := p.InstanceNew(session, instanceConf)
				if err != nil {
					log.Println(err)
					return
				}
				if c.IsSwarmManager || c.IsSwarmWorker {
					// check if we have connection to the daemon, if not, create it
					if i.Docker == nil {
						dock, err := p.docker.New(i.IP, i.Cert, i.Key)
						if err != nil {
							log.Println(err)
							return
						}
						i.Docker = dock
					}
				}

				if firstSwarmManager != nil {
					if c.IsSwarmManager {
						// this is a swarm manager
						// cluster has already been initiated, join as manager
						err := i.Docker.SwarmJoin(fmt.Sprintf("%s:2377", firstSwarmManager.IP), tokens.Manager)
						if err != nil {
							log.Println(err)
							return
						}
					}
					if c.IsSwarmWorker {
						// this is a swarm worker
						err := i.Docker.SwarmJoin(fmt.Sprintf("%s:2377", firstSwarmManager.IP), tokens.Worker)
						if err != nil {
							log.Println(err)
							return
						}
					}
				}
			}(c)
		}
	}
	wg.Wait()

	return nil
}

// This function should be called any time a session needs to be prepared:
// 1. Like when it is created
// 2. When it was loaded from storage
func (p *pwd) prepareSession(session *types.Session) error {
	session.Lock()
	defer session.Unlock()

	if session.IsPrepared() {
		return nil
	}

	p.scheduleSessionClose(session)

	// Connect PWD daemon to the new network
	if err := p.connectToNetwork(session); err != nil {
		return err
	}

	// Schedule periodic tasks
	p.tasks.Schedule(session)

	for _, i := range session.Instances {
		// wire the session back to the instance
		i.Session = session
		go p.InstanceAttachTerminal(i)
	}
	session.SetPrepared()

	return nil
}

func (p *pwd) scheduleSessionClose(s *types.Session) {
	timeLeft := s.ExpiresAt.Sub(time.Now())
	s.SetClosingTimer(time.AfterFunc(timeLeft, func() {
		p.SessionClose(s)
	}))
}

func (p *pwd) connectToNetwork(s *types.Session) error {
	ip, err := p.docker.ConnectNetwork(config.PWDContainerName, s.Id, s.PwdIpAddress)
	if err != nil {
		log.Println("ERROR NETWORKING")
		return err
	}
	s.PwdIpAddress = ip
	log.Printf("Connected %s to network [%s]\n", config.PWDContainerName, s.Id)
	return nil
}
<|endoftext|>


<commit_before>package pwd

import (
	"context"
	"errors"
	"fmt"
	"log"
	"math"
	"path"
	"path/filepath"
	"sync"
	"time"

	"golang.org/x/sync/errgroup"

	"github.com/play-with-docker/play-with-docker/docker"
	"github.com/play-with-docker/play-with-docker/event"
	"github.com/play-with-docker/play-with-docker/pwd/types"
)

var preparedSessions = map[string]bool{}

type AccessDeniedError struct {
	Err error
}

func (u *AccessDeniedError) Error() string {
	return fmt.Sprintf("Acess denied error: %s", u.Err.Error())
}

func (u *AccessDeniedError) Unwrap() error {
	return u.Err
}

type sessionBuilderWriter struct {
	sessionId string
	event     event.EventApi
}

func (s *sessionBuilderWriter) Write(p []byte) (n int, err error) {
	s.event.Emit(event.SESSION_BUILDER_OUT, s.sessionId, string(p))
	return len(p), nil
}

type SessionSetupConf struct {
	Instances      []SessionSetupInstanceConf `json:"instances"`
	PlaygroundFQDN string
	DindVolumeSize string
}

type SessionSetupInstanceConf struct {
	Image          string     `json:"image"`
	Hostname       string     `json:"hostname"`
	IsSwarmManager bool       `json:"is_swarm_manager"`
	IsSwarmWorker  bool       `json:"is_swarm_worker"`
	Type           string     `json:"type"`
	Run            [][]string `json:"run"`
	Tls            bool       `json:"tls"`
}

func (p *pwd) SessionNew(ctx context.Context, config types.SessionConfig) (*types.Session, error) {
	defer observeAction("SessionNew", time.Now())

	if _, err := p.UserGet(config.UserId); errors.Is(err, userBannedError) {
		return nil, &AccessDeniedError{err}
	} else if err != nil {
		return nil, err
	}

	s := &types.Session{}
	s.Id = p.generator.NewId()
	s.CreatedAt = time.Now()
	s.ExpiresAt = s.CreatedAt.Add(config.Duration)
	s.Ready = true
	s.Stack = config.Stack
	s.UserId = config.UserId
	s.PlaygroundId = config.Playground.Id

	if s.Stack != "" {
		s.Ready = false
	}
	stackName := config.StackName
	if stackName == "" {
		stackName = "pwd"
	}
	s.StackName = stackName
	s.ImageName = config.ImageName

	log.Printf("NewSession id=[%s]\n", s.Id)
	if err := p.sessionProvisioner.SessionNew(ctx, s); err != nil {
		log.Println(err)
		return nil, err
	}

	if err := p.storage.SessionPut(s); err != nil {
		log.Println(err)
		return nil, err
	}

	p.setGauges()
	p.event.Emit(event.SESSION_NEW, s.Id)

	return s, nil
}

func (p *pwd) SessionClose(s *types.Session) error {
	defer observeAction("SessionClose", time.Now())

	log.Printf("Starting clean up of session [%s]\n", s.Id)
	g, _ := errgroup.WithContext(context.Background())
	instances, err := p.storage.InstanceFindBySessionId(s.Id)
	if err != nil {
		log.Printf("Could not find instances in session %s. Got %v\n", s.Id, err)
		return err
	}
	for _, i := range instances {
		i := i
		g.Go(func() error {
			return p.InstanceDelete(s, i)
		})
	}
	err = g.Wait()
	if err != nil {
		log.Println(err)
		return err
	}

	if err := p.sessionProvisioner.SessionClose(s); err != nil {
		log.Println(err)
		return err
	}

	err = p.storage.SessionDelete(s.Id)
	if err != nil {
		return err
	}

	log.Printf("Cleaned up session [%s]\n", s.Id)
	p.setGauges()
	p.event.Emit(event.SESSION_END, s.Id)
	return nil

}

func (p *pwd) SessionGetSmallestViewPort(sessionId string) types.ViewPort {
	defer observeAction("SessionGetSmallestViewPort", time.Now())

	clients, err := p.storage.ClientFindBySessionId(sessionId)
	if err != nil {
		log.Printf("Error finding clients for session [%s]. Got: %v\n", sessionId, err)
		return types.ViewPort{Rows: 24, Cols: 80}
	}
	if len(clients) == 0 {
		log.Printf("Session [%s] doesn't have clients. Returning default viewport\n", sessionId)
		return types.ViewPort{Rows: 24, Cols: 80}
	}
	var minRows uint
	var minCols uint

	for _, c := range clients {
		if c.ViewPort.Rows > 0 && c.ViewPort.Cols > 0 {
			minRows = uint(c.ViewPort.Rows)
			minCols = uint(c.ViewPort.Cols)
			break
		}
	}

	for _, c := range clients {
		if c.ViewPort.Rows > 0 && c.ViewPort.Cols > 0 {
			minRows = uint(math.Min(float64(minRows), float64(c.ViewPort.Rows)))
			minCols = uint(math.Min(float64(minCols), float64(c.ViewPort.Cols)))
		}
	}

	return types.ViewPort{Rows: minRows, Cols: minCols}
}

func (p *pwd) SessionDeployStack(s *types.Session) error {
	defer observeAction("SessionDeployStack", time.Now())

	if s.Ready {
		// a stack was already deployed on this session, just ignore
		return nil
	}

	s.Ready = false
	p.event.Emit(event.SESSION_READY, s.Id, false)
	i, err := p.InstanceNew(s, types.InstanceConfig{ImageName: s.ImageName, PlaygroundFQDN: s.Host, DindVolumeSize: "5G"})
	if err != nil {
		log.Printf("Error creating instance for stack [%s]: %s\n", s.Stack, err)
		return err
	}

	_, fileName := filepath.Split(s.Stack)
	err = p.InstanceUploadFromUrl(i, fileName, "/var/run/pwd/uploads", s.Stack)
	if err != nil {
		log.Printf("Error uploading stack file [%s]: %s\n", s.Stack, err)
		return err
	}

	fileName = path.Base(s.Stack)
	file := fmt.Sprintf("/var/run/pwd/uploads/%s", fileName)
	cmd := fmt.Sprintf("docker swarm init --advertise-addr eth0 && docker-compose -f %s pull && docker stack deploy -c %s %s", file, file, s.StackName)

	w := sessionBuilderWriter{sessionId: s.Id, event: p.event}

	dockerClient, err := p.dockerFactory.GetForSession(s)
	if err != nil {
		log.Println(err)
		return err
	}

	code, err := dockerClient.ExecAttach(i.Name, []string{"sh", "-c", cmd}, &w)
	if err != nil {
		log.Printf("Error executing stack [%s]: %s\n", s.Stack, err)
		return err
	}

	log.Printf("Stack execution finished with code %d\n", code)
	s.Ready = true
	p.event.Emit(event.SESSION_READY, s.Id, true)
	if err := p.storage.SessionPut(s); err != nil {
		return err
	}
	return nil
}

func (p *pwd) SessionGet(sessionId string) (*types.Session, error) {
	defer observeAction("SessionGet", time.Now())

	s, err := p.storage.SessionGet(sessionId)

	if err != nil {
		log.Println(err)
		return nil, err
	}

	return s, nil
}

func (p *pwd) SessionSetup(session *types.Session, sconf SessionSetupConf) error {
	defer observeAction("SessionSetup", time.Now())

	c := sync.NewCond(&sync.Mutex{})

	var tokens *docker.SwarmTokens = nil
	var firstSwarmManager *types.Instance = nil

	instances, err := p.storage.InstanceFindBySessionId(session.Id)
	if err != nil {
		log.Println(err)
		return err
	}
	if len(instances) > 0 {
		return sessionNotEmpty
	}

	g, ctx := errgroup.WithContext(context.Background())

	for _, conf := range sconf.Instances {
		conf := conf
		g.Go(func() error {
			instanceConf := types.InstanceConfig{
				ImageName:      conf.Image,
				Hostname:       conf.Hostname,
				PlaygroundFQDN: sconf.PlaygroundFQDN,
				Type:           conf.Type,
				Tls:            conf.Tls,
				DindVolumeSize: sconf.DindVolumeSize,
			}
			i, err := p.InstanceNew(session, instanceConf)
			if err != nil {
				return err
			}

			if conf.IsSwarmManager || conf.IsSwarmWorker {
				dockerClient, err := p.dockerFactory.GetForInstance(i)
				if err != nil {
					return err
				}
				if conf.IsSwarmManager {
					c.L.Lock()
					if firstSwarmManager == nil {
						tkns, err := dockerClient.SwarmInit(i.IP)
						if err != nil {
							log.Printf("Cannot initialize swarm on instance %s. Got: %v\n", i.Name, err)
							return err
						}
						tokens = tkns
						firstSwarmManager = i
						c.Broadcast()
						c.L.Unlock()
					} else {
						c.L.Unlock()
						if err := dockerClient.SwarmJoin(fmt.Sprintf("%s:2377", firstSwarmManager.IP), tokens.Manager); err != nil {
							log.Printf("Cannot join manager %s to swarm. Got: %v\n", i.Name, err)
							return err
						}
					}
				} else if conf.IsSwarmWorker {
					c.L.Lock()
					if firstSwarmManager == nil {
						c.Wait()
					}
					c.L.Unlock()
					err = dockerClient.SwarmJoin(fmt.Sprintf("%s:2377", firstSwarmManager.IP), tokens.Worker)
					if err != nil {
						log.Printf("Cannot join worker %s to swarm. Got: %v\n", i.Name, err)
						return err
					}
				}
			}

			for _, cmd := range conf.Run {
				errch := make(chan error)
				go func() {
					exitCode, err := p.InstanceExec(i, cmd)
					fmt.Printf("Finished execuing command [%s] on instance %s with code [%d] and err [%v]\n", cmd, i.Name, exitCode, err)

					if err != nil {
						errch <- err
					}
					if exitCode != 0 {
						errch <- fmt.Errorf("Command returned %d on instance %s", exitCode, i.IP)
					}
					errch <- nil
				}()

				// ctx.Done() could be called if the errgroup is cancelled due to a previous error. In that case, return immediately
				select {
				case err = <-errch:
					return err
				case <-ctx.Done():
					return ctx.Err()
				}
			}
			return nil
		})
	}

	if err := g.Wait(); err != nil {
		log.Println(err)
		return err
	}

	return nil
}
<commit_msg>Allow annonymous users to create sessions<commit_after>package pwd

import (
	"context"
	"errors"
	"fmt"
	"log"
	"math"
	"path"
	"path/filepath"
	"sync"
	"time"

	"golang.org/x/sync/errgroup"

	"github.com/play-with-docker/play-with-docker/docker"
	"github.com/play-with-docker/play-with-docker/event"
	"github.com/play-with-docker/play-with-docker/pwd/types"
)

var preparedSessions = map[string]bool{}

type AccessDeniedError struct {
	Err error
}

func (u *AccessDeniedError) Error() string {
	return fmt.Sprintf("Acess denied error: %s", u.Err.Error())
}

func (u *AccessDeniedError) Unwrap() error {
	return u.Err
}

type sessionBuilderWriter struct {
	sessionId string
	event     event.EventApi
}

func (s *sessionBuilderWriter) Write(p []byte) (n int, err error) {
	s.event.Emit(event.SESSION_BUILDER_OUT, s.sessionId, string(p))
	return len(p), nil
}

type SessionSetupConf struct {
	Instances      []SessionSetupInstanceConf `json:"instances"`
	PlaygroundFQDN string
	DindVolumeSize string
}

type SessionSetupInstanceConf struct {
	Image          string     `json:"image"`
	Hostname       string     `json:"hostname"`
	IsSwarmManager bool       `json:"is_swarm_manager"`
	IsSwarmWorker  bool       `json:"is_swarm_worker"`
	Type           string     `json:"type"`
	Run            [][]string `json:"run"`
	Tls            bool       `json:"tls"`
}

func (p *pwd) SessionNew(ctx context.Context, config types.SessionConfig) (*types.Session, error) {
	defer observeAction("SessionNew", time.Now())

	// Annonymous users should be also allowed to login
	if config.UserId != "" {
		if _, err := p.UserGet(config.UserId); errors.Is(err, userBannedError) {

			return nil, &AccessDeniedError{err}
		} else if err != nil {
			return nil, err
		}
	}

	s := &types.Session{}
	s.Id = p.generator.NewId()
	s.CreatedAt = time.Now()
	s.ExpiresAt = s.CreatedAt.Add(config.Duration)
	s.Ready = true
	s.Stack = config.Stack
	s.UserId = config.UserId
	s.PlaygroundId = config.Playground.Id

	if s.Stack != "" {
		s.Ready = false
	}
	stackName := config.StackName
	if stackName == "" {
		stackName = "pwd"
	}
	s.StackName = stackName
	s.ImageName = config.ImageName

	log.Printf("NewSession id=[%s]\n", s.Id)
	if err := p.sessionProvisioner.SessionNew(ctx, s); err != nil {
		log.Println(err)
		return nil, err
	}

	if err := p.storage.SessionPut(s); err != nil {
		log.Println(err)
		return nil, err
	}

	p.setGauges()
	p.event.Emit(event.SESSION_NEW, s.Id)

	return s, nil
}

func (p *pwd) SessionClose(s *types.Session) error {
	defer observeAction("SessionClose", time.Now())

	log.Printf("Starting clean up of session [%s]\n", s.Id)
	g, _ := errgroup.WithContext(context.Background())
	instances, err := p.storage.InstanceFindBySessionId(s.Id)
	if err != nil {
		log.Printf("Could not find instances in session %s. Got %v\n", s.Id, err)
		return err
	}
	for _, i := range instances {
		i := i
		g.Go(func() error {
			return p.InstanceDelete(s, i)
		})
	}
	err = g.Wait()
	if err != nil {
		log.Println(err)
		return err
	}

	if err := p.sessionProvisioner.SessionClose(s); err != nil {
		log.Println(err)
		return err
	}

	err = p.storage.SessionDelete(s.Id)
	if err != nil {
		return err
	}

	log.Printf("Cleaned up session [%s]\n", s.Id)
	p.setGauges()
	p.event.Emit(event.SESSION_END, s.Id)
	return nil

}

func (p *pwd) SessionGetSmallestViewPort(sessionId string) types.ViewPort {
	defer observeAction("SessionGetSmallestViewPort", time.Now())

	clients, err := p.storage.ClientFindBySessionId(sessionId)
	if err != nil {
		log.Printf("Error finding clients for session [%s]. Got: %v\n", sessionId, err)
		return types.ViewPort{Rows: 24, Cols: 80}
	}
	if len(clients) == 0 {
		log.Printf("Session [%s] doesn't have clients. Returning default viewport\n", sessionId)
		return types.ViewPort{Rows: 24, Cols: 80}
	}
	var minRows uint
	var minCols uint

	for _, c := range clients {
		if c.ViewPort.Rows > 0 && c.ViewPort.Cols > 0 {
			minRows = uint(c.ViewPort.Rows)
			minCols = uint(c.ViewPort.Cols)
			break
		}
	}

	for _, c := range clients {
		if c.ViewPort.Rows > 0 && c.ViewPort.Cols > 0 {
			minRows = uint(math.Min(float64(minRows), float64(c.ViewPort.Rows)))
			minCols = uint(math.Min(float64(minCols), float64(c.ViewPort.Cols)))
		}
	}

	return types.ViewPort{Rows: minRows, Cols: minCols}
}

func (p *pwd) SessionDeployStack(s *types.Session) error {
	defer observeAction("SessionDeployStack", time.Now())

	if s.Ready {
		// a stack was already deployed on this session, just ignore
		return nil
	}

	s.Ready = false
	p.event.Emit(event.SESSION_READY, s.Id, false)
	i, err := p.InstanceNew(s, types.InstanceConfig{ImageName: s.ImageName, PlaygroundFQDN: s.Host, DindVolumeSize: "5G"})
	if err != nil {
		log.Printf("Error creating instance for stack [%s]: %s\n", s.Stack, err)
		return err
	}

	_, fileName := filepath.Split(s.Stack)
	err = p.InstanceUploadFromUrl(i, fileName, "/var/run/pwd/uploads", s.Stack)
	if err != nil {
		log.Printf("Error uploading stack file [%s]: %s\n", s.Stack, err)
		return err
	}

	fileName = path.Base(s.Stack)
	file := fmt.Sprintf("/var/run/pwd/uploads/%s", fileName)
	cmd := fmt.Sprintf("docker swarm init --advertise-addr eth0 && docker-compose -f %s pull && docker stack deploy -c %s %s", file, file, s.StackName)

	w := sessionBuilderWriter{sessionId: s.Id, event: p.event}

	dockerClient, err := p.dockerFactory.GetForSession(s)
	if err != nil {
		log.Println(err)
		return err
	}

	code, err := dockerClient.ExecAttach(i.Name, []string{"sh", "-c", cmd}, &w)
	if err != nil {
		log.Printf("Error executing stack [%s]: %s\n", s.Stack, err)
		return err
	}

	log.Printf("Stack execution finished with code %d\n", code)
	s.Ready = true
	p.event.Emit(event.SESSION_READY, s.Id, true)
	if err := p.storage.SessionPut(s); err != nil {
		return err
	}
	return nil
}

func (p *pwd) SessionGet(sessionId string) (*types.Session, error) {
	defer observeAction("SessionGet", time.Now())

	s, err := p.storage.SessionGet(sessionId)

	if err != nil {
		log.Println(err)
		return nil, err
	}

	return s, nil
}

func (p *pwd) SessionSetup(session *types.Session, sconf SessionSetupConf) error {
	defer observeAction("SessionSetup", time.Now())

	c := sync.NewCond(&sync.Mutex{})

	var tokens *docker.SwarmTokens = nil
	var firstSwarmManager *types.Instance = nil

	instances, err := p.storage.InstanceFindBySessionId(session.Id)
	if err != nil {
		log.Println(err)
		return err
	}
	if len(instances) > 0 {
		return sessionNotEmpty
	}

	g, ctx := errgroup.WithContext(context.Background())

	for _, conf := range sconf.Instances {
		conf := conf
		g.Go(func() error {
			instanceConf := types.InstanceConfig{
				ImageName:      conf.Image,
				Hostname:       conf.Hostname,
				PlaygroundFQDN: sconf.PlaygroundFQDN,
				Type:           conf.Type,
				Tls:            conf.Tls,
				DindVolumeSize: sconf.DindVolumeSize,
			}
			i, err := p.InstanceNew(session, instanceConf)
			if err != nil {
				return err
			}

			if conf.IsSwarmManager || conf.IsSwarmWorker {
				dockerClient, err := p.dockerFactory.GetForInstance(i)
				if err != nil {
					return err
				}
				if conf.IsSwarmManager {
					c.L.Lock()
					if firstSwarmManager == nil {
						tkns, err := dockerClient.SwarmInit(i.IP)
						if err != nil {
							log.Printf("Cannot initialize swarm on instance %s. Got: %v\n", i.Name, err)
							return err
						}
						tokens = tkns
						firstSwarmManager = i
						c.Broadcast()
						c.L.Unlock()
					} else {
						c.L.Unlock()
						if err := dockerClient.SwarmJoin(fmt.Sprintf("%s:2377", firstSwarmManager.IP), tokens.Manager); err != nil {
							log.Printf("Cannot join manager %s to swarm. Got: %v\n", i.Name, err)
							return err
						}
					}
				} else if conf.IsSwarmWorker {
					c.L.Lock()
					if firstSwarmManager == nil {
						c.Wait()
					}
					c.L.Unlock()
					err = dockerClient.SwarmJoin(fmt.Sprintf("%s:2377", firstSwarmManager.IP), tokens.Worker)
					if err != nil {
						log.Printf("Cannot join worker %s to swarm. Got: %v\n", i.Name, err)
						return err
					}
				}
			}

			for _, cmd := range conf.Run {
				errch := make(chan error)
				go func() {
					exitCode, err := p.InstanceExec(i, cmd)
					fmt.Printf("Finished execuing command [%s] on instance %s with code [%d] and err [%v]\n", cmd, i.Name, exitCode, err)

					if err != nil {
						errch <- err
					}
					if exitCode != 0 {
						errch <- fmt.Errorf("Command returned %d on instance %s", exitCode, i.IP)
					}
					errch <- nil
				}()

				// ctx.Done() could be called if the errgroup is cancelled due to a previous error. In that case, return immediately
				select {
				case err = <-errch:
					return err
				case <-ctx.Done():
					return ctx.Err()
				}
			}
			return nil
		})
	}

	if err := g.Wait(); err != nil {
		log.Println(err)
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"http"
	"os"
	"sync"
)

type tinkerTransport struct {
	mu   sync.Mutex
	reqs int
}

func (t *tinkerTransport) RoundTrip(req *http.Request) (resp *http.Response, err os.Error) {
	t.mu.Lock()
	t.reqs++
	t.mu.Unlock()

	return http.DefaultTransport.RoundTrip(req)
}
<commit_msg>camput: let tinkerTransport's underlying transport be changed<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"http"
	"os"
	"sync"
)

type tinkerTransport struct {
	mu   sync.Mutex
	reqs int

	transport http.RoundTripper
}

func (t *tinkerTransport) RoundTrip(req *http.Request) (resp *http.Response, err os.Error) {
	t.mu.Lock()
	t.reqs++
	rt := t.transport
	t.mu.Unlock()

	if rt == nil {
		rt = http.DefaultTransport
	}

	return rt.RoundTrip(req)
}
<|endoftext|>