<commit_before>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package harness

import (
	"context"
	"io"
	"sync"
	"time"

	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/exec"
	"github.com/apache/beam/sdks/go/pkg/beam/internal/errors"
	"github.com/apache/beam/sdks/go/pkg/beam/log"
	pb "github.com/apache/beam/sdks/go/pkg/beam/model/fnexecution_v1"
)

const (
	chunkSize   = int(4e6) // Bytes to put in a single gRPC message. Max is slightly higher.
	bufElements = 20       // Number of chunks buffered per reader.
)

// ScopedDataManager scopes the global gRPC data manager to a single instruction.
// The indirection makes it easier to control access.
type ScopedDataManager struct {
	mgr    *DataChannelManager
	instID instructionID

	// TODO(herohde) 7/20/2018: capture and force close open reads/writes. However,
	// we would need the underlying Close to be idempotent or a separate method.
	closed bool
	mu     sync.Mutex
}

// NewScopedDataManager returns a ScopedDataManager for the given instruction.
func NewScopedDataManager(mgr *DataChannelManager, instID instructionID) *ScopedDataManager {
	return &ScopedDataManager{mgr: mgr, instID: instID}
}

// OpenRead opens an io.ReadCloser on the given stream.
func (s *ScopedDataManager) OpenRead(ctx context.Context, id exec.StreamID) (io.ReadCloser, error) {
	ch, err := s.open(ctx, id.Port)
	if err != nil {
		return nil, err
	}
	return ch.OpenRead(ctx, id.PtransformID, s.instID), nil
}

// OpenWrite opens an io.WriteCloser on the given stream.
func (s *ScopedDataManager) OpenWrite(ctx context.Context, id exec.StreamID) (io.WriteCloser, error) {
	ch, err := s.open(ctx, id.Port)
	if err != nil {
		return nil, err
	}
	return ch.OpenWrite(ctx, id.PtransformID, s.instID), nil
}

func (s *ScopedDataManager) open(ctx context.Context, port exec.Port) (*DataChannel, error) {
	s.mu.Lock()
	if s.closed {
		s.mu.Unlock()
		return nil, errors.Errorf("instruction %v no longer processing", s.instID)
	}
	local := s.mgr
	s.mu.Unlock()

	return local.Open(ctx, port) // don't hold lock over potentially slow operation
}

// Close prevents new IO for this instruction.
func (s *ScopedDataManager) Close() error {
	s.mu.Lock()
	s.closed = true
	s.mgr = nil
	s.mu.Unlock()
	return nil
}

// DataChannelManager manages data channels over the Data API. A fixed number of channels
// are generally used, each managing multiple logical byte streams. Thread-safe.
type DataChannelManager struct {
	ports map[string]*DataChannel
	mu    sync.Mutex // guards the ports map
}

// Open opens a R/W DataChannel over the given port.
func (m *DataChannelManager) Open(ctx context.Context, port exec.Port) (*DataChannel, error) {
	if port.URL == "" {
		panic("empty port")
	}

	m.mu.Lock()
	defer m.mu.Unlock()

	if m.ports == nil {
		m.ports = make(map[string]*DataChannel)
	}
	if con, ok := m.ports[port.URL]; ok {
		return con, nil
	}

	ch, err := newDataChannel(ctx, port)
	if err != nil {
		return nil, err
	}
	m.ports[port.URL] = ch
	return ch, nil
}

// clientID identifies a client of a connected channel.
type clientID struct {
	ptransformID string
	instID       instructionID
}

// This is a reduced version of the full gRPC interface to help with testing.
// TODO(wcn): need a compile-time assertion to make sure this stays synced with what's
// in pb.BeamFnData_DataClient
type dataClient interface {
	Send(*pb.Elements) error
	Recv() (*pb.Elements, error)
}

// DataChannel manages a single multiplexed gRPC connection over the Data API. Data is
// pushed over the channel, so data for a reader may arrive before the reader connects.
// Thread-safe.
type DataChannel struct {
	id     string
	client dataClient

	writers map[clientID]*dataWriter
	readers map[clientID]*dataReader
	// TODO: early/late closed, bad instructions, finer locks, reconnect?

	// readErr indicates a client.Recv error and is used to prevent new readers.
	readErr error

	mu sync.Mutex // guards both the readers and writers maps.
}

func newDataChannel(ctx context.Context, port exec.Port) (*DataChannel, error) {
	cc, err := dial(ctx, port.URL, 15*time.Second)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to connect to data service at %v", port.URL)
	}
	client, err := pb.NewBeamFnDataClient(cc).Data(ctx)
	if err != nil {
		cc.Close()
		return nil, errors.Wrapf(err, "failed to create data client on %v", port.URL)
	}
	return makeDataChannel(ctx, port.URL, client), nil
}

func makeDataChannel(ctx context.Context, id string, client dataClient) *DataChannel {
	ret := &DataChannel{
		id:      id,
		client:  client,
		writers: make(map[clientID]*dataWriter),
		readers: make(map[clientID]*dataReader),
	}
	go ret.read(ctx)

	return ret
}

// OpenRead returns an io.ReadCloser of the data elements for the given instruction and ptransform.
func (c *DataChannel) OpenRead(ctx context.Context, ptransformID string, instID instructionID) io.ReadCloser {
	cid := clientID{ptransformID: ptransformID, instID: instID}
	if c.readErr != nil {
		log.Errorf(ctx, "opening a reader %v on a closed channel", cid)
		return &errReader{c.readErr}
	}
	return c.makeReader(ctx, cid)
}

// OpenWrite returns an io.WriteCloser of the data elements for the given instruction and ptransform.
func (c *DataChannel) OpenWrite(ctx context.Context, ptransformID string, instID instructionID) io.WriteCloser {
	return c.makeWriter(ctx, clientID{ptransformID: ptransformID, instID: instID})
}

func (c *DataChannel) read(ctx context.Context) {
	cache := make(map[clientID]*dataReader)
	for {
		msg, err := c.client.Recv()
		if err != nil {
			// This connection is bad, so we should close and delete all extant streams.
			c.mu.Lock()
			c.readErr = err // prevent not yet opened readers from hanging.
			for _, r := range c.readers {
				log.Errorf(ctx, "DataChannel.read %v reader %v closing due to error on channel", c.id, r.id)
				if !r.completed {
					r.completed = true
					r.err = err
					close(r.buf)
				}
				delete(cache, r.id)
			}
			c.mu.Unlock()

			if err == io.EOF {
				log.Warnf(ctx, "DataChannel.read %v closed", c.id)
				return
			}
			log.Errorf(ctx, "DataChannel.read %v bad: %v", c.id, err)
			return
		}

		recordStreamReceive(msg)

		// Each message may contain segments for multiple streams, so we
		// must treat each segment in isolation. We maintain a local cache
		// to reduce lock contention.

		for _, elm := range msg.GetData() {
			id := clientID{ptransformID: elm.TransformId, instID: instructionID(elm.GetInstructionId())}

			var r *dataReader
			if local, ok := cache[id]; ok {
				r = local
			} else {
				r = c.makeReader(ctx, id)
				cache[id] = r
			}

			if r.completed {
				// The local reader has closed but the remote is still sending data.
				// Just ignore it. We keep the reader config in the cache so we don't
				// treat it as a new reader. Eventually the stream will finish and go
				// through normal teardown.
				continue
			}
			if len(elm.GetData()) == 0 {
				// Sentinel EOF segment for stream. Close buffer to signal EOF.
				r.completed = true
				close(r.buf)

				// Clean up local bookkeeping. We'll never see another message
				// for it again. We have to be careful not to remove the real
				// one, because readers may be initialized after we've seen
				// the full stream.
				delete(cache, id)
				continue
			}

			// This send is deliberately blocking, if we exceed the buffering for
			// a reader. We can't buffer the entire main input, if some user code
			// is slow (or gets stuck). If the local side closes, the reader
			// will be marked as completed and further remote data will be ignored.
			select {
			case r.buf <- elm.GetData():
			case <-r.done:
				r.completed = true
				close(r.buf)
			}
		}
	}
}

type errReader struct {
	err error
}

func (r *errReader) Read(_ []byte) (int, error) {
	return 0, r.err
}

func (r *errReader) Close() error {
	return r.err
}

func (c *DataChannel) makeReader(ctx context.Context, id clientID) *dataReader {
	c.mu.Lock()
	defer c.mu.Unlock()

	if r, ok := c.readers[id]; ok {
		return r
	}

	r := &dataReader{id: id, buf: make(chan []byte, bufElements), done: make(chan bool, 1), channel: c}
	c.readers[id] = r
	return r
}

func (c *DataChannel) removeReader(id clientID) {
	c.mu.Lock()
	delete(c.readers, id)
	c.mu.Unlock()
}

func (c *DataChannel) makeWriter(ctx context.Context, id clientID) *dataWriter {
	c.mu.Lock()
	defer c.mu.Unlock()

	if w, ok := c.writers[id]; ok {
		return w
	}

	w := &dataWriter{ch: c, id: id}
	c.writers[id] = w
	return w
}

type dataReader struct {
	id        clientID
	buf       chan []byte
	done      chan bool
	cur       []byte
	channel   *DataChannel
	completed bool
	err       error
}

func (r *dataReader) Close() error {
	r.done <- true
	r.channel.removeReader(r.id)
	return nil
}

func (r *dataReader) Read(buf []byte) (int, error) {
	if r.cur == nil {
		b, ok := <-r.buf
		if !ok {
			log.Errorf(context.TODO(), "dataReader.Read %v channel closed: %v", r.id, r.err)
			if r.err == nil {
				return 0, io.EOF
			}
			return 0, r.err
		}
		r.cur = b
	}

	n := copy(buf, r.cur)

	if len(r.cur) == n {
		r.cur = nil
	} else {
		r.cur = r.cur[n:]
	}

	return n, nil
}

// TODO(herohde) 7/20/2018: we should probably either not be tracking writers or
// make dataWriter threadsafe. Either case is likely a corruption generator.

type dataWriter struct {
	buf []byte

	id clientID
	ch *DataChannel
}

func (w *dataWriter) Close() error {
	// Don't acquire the locks as Flush will do so.
	err := w.Flush()
	if err != nil {
		return err
	}

	// Now acquire the locks since we're sending.
	w.ch.mu.Lock()
	defer w.ch.mu.Unlock()
	delete(w.ch.writers, w.id)
	msg := &pb.Elements{
		Data: []*pb.Elements_Data{
			{
				InstructionId: string(w.id.instID),
				TransformId:   w.id.ptransformID,
				// Empty data == sentinel
			},
		},
	}

	// TODO(wcn): if this send fails, we have a data channel that's lingering that
	// the runner is still waiting on. Need some way to identify these and resolve them.
	recordStreamSend(msg)
	return w.ch.client.Send(msg)
}

func (w *dataWriter) Flush() error {
	w.ch.mu.Lock()
	defer w.ch.mu.Unlock()

	if w.buf == nil {
		return nil
	}

	msg := &pb.Elements{
		Data: []*pb.Elements_Data{
			{
				InstructionId: string(w.id.instID),
				TransformId:   w.id.ptransformID,
				Data:          w.buf,
			},
		},
	}
	w.buf = nil
	recordStreamSend(msg)
	return w.ch.client.Send(msg)
}

func (w *dataWriter) Write(p []byte) (n int, err error) {
	if len(w.buf)+len(p) > chunkSize {
		l := len(w.buf)
		// We can't fit this message into the buffer. We need to flush the buffer
		if err := w.Flush(); err != nil {
			return 0, errors.Wrapf(err, "datamgr.go [%v]: error flushing buffer of length %d", w.id, l)
		}
	}

	// At this point there's room in the buffer one way or another.
	w.buf = append(w.buf, p...)
	return len(p), nil
}
<commit_msg>[Go SDK] Delete spammy log in datamgr.go<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package harness

import (
	"context"
	"io"
	"sync"
	"time"

	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/exec"
	"github.com/apache/beam/sdks/go/pkg/beam/internal/errors"
	"github.com/apache/beam/sdks/go/pkg/beam/log"
	pb "github.com/apache/beam/sdks/go/pkg/beam/model/fnexecution_v1"
)

const (
	chunkSize   = int(4e6) // Bytes to put in a single gRPC message. Max is slightly higher.
	bufElements = 20       // Number of chunks buffered per reader.
)

// ScopedDataManager scopes the global gRPC data manager to a single instruction.
// The indirection makes it easier to control access.
type ScopedDataManager struct {
	mgr    *DataChannelManager
	instID instructionID

	// TODO(herohde) 7/20/2018: capture and force close open reads/writes. However,
	// we would need the underlying Close to be idempotent or a separate method.
	closed bool
	mu     sync.Mutex
}

// NewScopedDataManager returns a ScopedDataManager for the given instruction.
func NewScopedDataManager(mgr *DataChannelManager, instID instructionID) *ScopedDataManager {
	return &ScopedDataManager{mgr: mgr, instID: instID}
}

// OpenRead opens an io.ReadCloser on the given stream.
func (s *ScopedDataManager) OpenRead(ctx context.Context, id exec.StreamID) (io.ReadCloser, error) {
	ch, err := s.open(ctx, id.Port)
	if err != nil {
		return nil, err
	}
	return ch.OpenRead(ctx, id.PtransformID, s.instID), nil
}

// OpenWrite opens an io.WriteCloser on the given stream.
func (s *ScopedDataManager) OpenWrite(ctx context.Context, id exec.StreamID) (io.WriteCloser, error) {
	ch, err := s.open(ctx, id.Port)
	if err != nil {
		return nil, err
	}
	return ch.OpenWrite(ctx, id.PtransformID, s.instID), nil
}

func (s *ScopedDataManager) open(ctx context.Context, port exec.Port) (*DataChannel, error) {
	s.mu.Lock()
	if s.closed {
		s.mu.Unlock()
		return nil, errors.Errorf("instruction %v no longer processing", s.instID)
	}
	local := s.mgr
	s.mu.Unlock()

	return local.Open(ctx, port) // don't hold lock over potentially slow operation
}

// Close prevents new IO for this instruction.
func (s *ScopedDataManager) Close() error {
	s.mu.Lock()
	s.closed = true
	s.mgr = nil
	s.mu.Unlock()
	return nil
}

// DataChannelManager manages data channels over the Data API. A fixed number of channels
// are generally used, each managing multiple logical byte streams. Thread-safe.
type DataChannelManager struct {
	ports map[string]*DataChannel
	mu    sync.Mutex // guards the ports map
}

// Open opens a R/W DataChannel over the given port.
func (m *DataChannelManager) Open(ctx context.Context, port exec.Port) (*DataChannel, error) {
	if port.URL == "" {
		panic("empty port")
	}

	m.mu.Lock()
	defer m.mu.Unlock()

	if m.ports == nil {
		m.ports = make(map[string]*DataChannel)
	}
	if con, ok := m.ports[port.URL]; ok {
		return con, nil
	}

	ch, err := newDataChannel(ctx, port)
	if err != nil {
		return nil, err
	}
	m.ports[port.URL] = ch
	return ch, nil
}

// clientID identifies a client of a connected channel.
type clientID struct {
	ptransformID string
	instID       instructionID
}

// This is a reduced version of the full gRPC interface to help with testing.
// TODO(wcn): need a compile-time assertion to make sure this stays synced with what's
// in pb.BeamFnData_DataClient
type dataClient interface {
	Send(*pb.Elements) error
	Recv() (*pb.Elements, error)
}

// DataChannel manages a single multiplexed gRPC connection over the Data API. Data is
// pushed over the channel, so data for a reader may arrive before the reader connects.
// Thread-safe.
type DataChannel struct {
	id     string
	client dataClient

	writers map[clientID]*dataWriter
	readers map[clientID]*dataReader
	// TODO: early/late closed, bad instructions, finer locks, reconnect?

	// readErr indicates a client.Recv error and is used to prevent new readers.
	readErr error

	mu sync.Mutex // guards both the readers and writers maps.
}

func newDataChannel(ctx context.Context, port exec.Port) (*DataChannel, error) {
	cc, err := dial(ctx, port.URL, 15*time.Second)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to connect to data service at %v", port.URL)
	}
	client, err := pb.NewBeamFnDataClient(cc).Data(ctx)
	if err != nil {
		cc.Close()
		return nil, errors.Wrapf(err, "failed to create data client on %v", port.URL)
	}
	return makeDataChannel(ctx, port.URL, client), nil
}

func makeDataChannel(ctx context.Context, id string, client dataClient) *DataChannel {
	ret := &DataChannel{
		id:      id,
		client:  client,
		writers: make(map[clientID]*dataWriter),
		readers: make(map[clientID]*dataReader),
	}
	go ret.read(ctx)

	return ret
}

// OpenRead returns an io.ReadCloser of the data elements for the given instruction and ptransform.
func (c *DataChannel) OpenRead(ctx context.Context, ptransformID string, instID instructionID) io.ReadCloser {
	cid := clientID{ptransformID: ptransformID, instID: instID}
	if c.readErr != nil {
		log.Errorf(ctx, "opening a reader %v on a closed channel", cid)
		return &errReader{c.readErr}
	}
	return c.makeReader(ctx, cid)
}

// OpenWrite returns an io.WriteCloser of the data elements for the given instruction and ptransform.
func (c *DataChannel) OpenWrite(ctx context.Context, ptransformID string, instID instructionID) io.WriteCloser {
	return c.makeWriter(ctx, clientID{ptransformID: ptransformID, instID: instID})
}

func (c *DataChannel) read(ctx context.Context) {
	cache := make(map[clientID]*dataReader)
	for {
		msg, err := c.client.Recv()
		if err != nil {
			// This connection is bad, so we should close and delete all extant streams.
			c.mu.Lock()
			c.readErr = err // prevent not yet opened readers from hanging.
			for _, r := range c.readers {
				log.Errorf(ctx, "DataChannel.read %v reader %v closing due to error on channel", c.id, r.id)
				if !r.completed {
					r.completed = true
					r.err = err
					close(r.buf)
				}
				delete(cache, r.id)
			}
			c.mu.Unlock()

			if err == io.EOF {
				log.Warnf(ctx, "DataChannel.read %v closed", c.id)
				return
			}
			log.Errorf(ctx, "DataChannel.read %v bad: %v", c.id, err)
			return
		}

		recordStreamReceive(msg)

		// Each message may contain segments for multiple streams, so we
		// must treat each segment in isolation. We maintain a local cache
		// to reduce lock contention.

		for _, elm := range msg.GetData() {
			id := clientID{ptransformID: elm.TransformId, instID: instructionID(elm.GetInstructionId())}

			var r *dataReader
			if local, ok := cache[id]; ok {
				r = local
			} else {
				r = c.makeReader(ctx, id)
				cache[id] = r
			}

			if r.completed {
				// The local reader has closed but the remote is still sending data.
				// Just ignore it. We keep the reader config in the cache so we don't
				// treat it as a new reader. Eventually the stream will finish and go
				// through normal teardown.
				continue
			}
			if len(elm.GetData()) == 0 {
				// Sentinel EOF segment for stream. Close buffer to signal EOF.
				r.completed = true
				close(r.buf)

				// Clean up local bookkeeping. We'll never see another message
				// for it again. We have to be careful not to remove the real
				// one, because readers may be initialized after we've seen
				// the full stream.
				delete(cache, id)
				continue
			}

			// This send is deliberately blocking, if we exceed the buffering for
			// a reader. We can't buffer the entire main input, if some user code
			// is slow (or gets stuck). If the local side closes, the reader
			// will be marked as completed and further remote data will be ignored.
			select {
			case r.buf <- elm.GetData():
			case <-r.done:
				r.completed = true
				close(r.buf)
			}
		}
	}
}

type errReader struct {
	err error
}

func (r *errReader) Read(_ []byte) (int, error) {
	return 0, r.err
}

func (r *errReader) Close() error {
	return r.err
}

func (c *DataChannel) makeReader(ctx context.Context, id clientID) *dataReader {
	c.mu.Lock()
	defer c.mu.Unlock()

	if r, ok := c.readers[id]; ok {
		return r
	}

	r := &dataReader{id: id, buf: make(chan []byte, bufElements), done: make(chan bool, 1), channel: c}
	c.readers[id] = r
	return r
}

func (c *DataChannel) removeReader(id clientID) {
	c.mu.Lock()
	delete(c.readers, id)
	c.mu.Unlock()
}

func (c *DataChannel) makeWriter(ctx context.Context, id clientID) *dataWriter {
	c.mu.Lock()
	defer c.mu.Unlock()

	if w, ok := c.writers[id]; ok {
		return w
	}

	w := &dataWriter{ch: c, id: id}
	c.writers[id] = w
	return w
}

type dataReader struct {
	id        clientID
	buf       chan []byte
	done      chan bool
	cur       []byte
	channel   *DataChannel
	completed bool
	err       error
}

func (r *dataReader) Close() error {
	r.done <- true
	r.channel.removeReader(r.id)
	return nil
}

func (r *dataReader) Read(buf []byte) (int, error) {
	if r.cur == nil {
		b, ok := <-r.buf
		if !ok {
			if r.err == nil {
				return 0, io.EOF
			}
			return 0, r.err
		}
		r.cur = b
	}

	n := copy(buf, r.cur)

	if len(r.cur) == n {
		r.cur = nil
	} else {
		r.cur = r.cur[n:]
	}

	return n, nil
}

// TODO(herohde) 7/20/2018: we should probably either not be tracking writers or
// make dataWriter threadsafe. Either case is likely a corruption generator.

type dataWriter struct {
	buf []byte

	id clientID
	ch *DataChannel
}

func (w *dataWriter) Close() error {
	// Don't acquire the locks as Flush will do so.
	err := w.Flush()
	if err != nil {
		return err
	}

	// Now acquire the locks since we're sending.
	w.ch.mu.Lock()
	defer w.ch.mu.Unlock()
	delete(w.ch.writers, w.id)
	msg := &pb.Elements{
		Data: []*pb.Elements_Data{
			{
				InstructionId: string(w.id.instID),
				TransformId:   w.id.ptransformID,
				// Empty data == sentinel
			},
		},
	}

	// TODO(wcn): if this send fails, we have a data channel that's lingering that
	// the runner is still waiting on. Need some way to identify these and resolve them.
	recordStreamSend(msg)
	return w.ch.client.Send(msg)
}

func (w *dataWriter) Flush() error {
	w.ch.mu.Lock()
	defer w.ch.mu.Unlock()

	if w.buf == nil {
		return nil
	}

	msg := &pb.Elements{
		Data: []*pb.Elements_Data{
			{
				InstructionId: string(w.id.instID),
				TransformId:   w.id.ptransformID,
				Data:          w.buf,
			},
		},
	}
	w.buf = nil
	recordStreamSend(msg)
	return w.ch.client.Send(msg)
}

func (w *dataWriter) Write(p []byte) (n int, err error) {
	if len(w.buf)+len(p) > chunkSize {
		l := len(w.buf)
		// We can't fit this message into the buffer. We need to flush the buffer
		if err := w.Flush(); err != nil {
			return 0, errors.Wrapf(err, "datamgr.go [%v]: error flushing buffer of length %d", w.id, l)
		}
	}

	// At this point there's room in the buffer one way or another.
	w.buf = append(w.buf, p...)
	return len(p), nil
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"log"
	"time"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/config"
	"github.com/jacobsa/comeback/internal/graph"
	"github.com/jacobsa/comeback/internal/registry"
	"github.com/jacobsa/comeback/internal/save"
	"github.com/jacobsa/syncutil"
)

var cmdSave = &Command{
	Name: "save",
}

var g_jobName = cmdSave.Flags.String(
	"job",
	"",
	"Job name within the config file.")

var g_discardScoreCache = cmdSave.Flags.Bool(
	"discard_score_cache",
	false,
	"If set, always recompute file hashes; don't rely on stat info.",
)

var fListOnly = cmdSave.Flags.Bool(
	"list_only",
	false,
	"If set, list the files that would be backed up but do nothing further.")

func init() {
	cmdSave.Run = runSave // Break flag-related dependency loop.
}

func saveStatePeriodically(c <-chan time.Time) {
	for _ = range c {
		log.Println("Writing out state file.")
		saveState()
	}
}

func doList(job *config.Job) (err error) {
	b := syncutil.NewBundle(context.Background())

	// Explore the file system graph, writing all non-excluded nodes into a
	// channel.
	//
	// TODO(jacobsa): Hide the use of the graph package in the unexported
	// implementation details of package save.
	graphNodes := make(chan graph.Node, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(graphNodes)
		sf := save.NewFileSystemVisitor(job.BasePath, job.Excludes)

		const parallelism = 8
		err = graph.ExploreDirectedGraph(
			ctx,
			sf,
			[]graph.Node{(*save.PathAndFileInfo)(nil)},
			graphNodes,
			parallelism)

		if err != nil {
			err = fmt.Errorf("ExploreDirectedGraph: %v", err)
			return
		}

		return
	})

	// Print out info about each node.
	b.Add(func(ctx context.Context) (err error) {
		for n := range graphNodes {
			pfi, ok := n.(*save.PathAndFileInfo)
			if !ok {
				err = fmt.Errorf("Unexpected node type: %T", n)
				return
			}

			// Skip the root node.
			if pfi == nil {
				continue
			}

			fmt.Printf("%s %d\n", pfi.Path, pfi.Info.Size())
		}

		return
	})

	err = b.Join()
	return
}

func runSave(args []string) {
	cfg := getConfig()

	// Look for the specified job.
	if *g_jobName == "" {
		log.Fatalln("You must set the -job flag.")
	}

	job, ok := cfg.Jobs[*g_jobName]
	if !ok {
		log.Fatalln("Unknown job:", *g_jobName)
	}

	// Special case: visit the file system only if --list_only is set.
	//
	// TODO(jacobsa): Integrate this into the pipeline when it exists. See issue
	// #21.
	if *fListOnly {
		err := doList(job)
		if err != nil {
			log.Fatalf("doList: %v", err)
			return
		}

		return
	}

	// Grab dependencies. Make sure to get the registry first, because otherwise
	// the user will have to wait for bucket keys to be listed before being
	// prompted for a crypto password.
	//
	// Make sure to do this before setting up state saving below, because these
	// calls may modify the state struct.
	reg := getRegistry()
	dirSaver := getDirSaver()

	// Periodically save state.
	const saveStatePeriod = 15 * time.Second
	saveStateTicker := time.NewTicker(saveStatePeriod)
	go saveStatePeriodically(saveStateTicker.C)

	// Choose a start time for the job.
	startTime := time.Now()

	// Call the directory saver.
	score, err := dirSaver.Save(job.BasePath, "", job.Excludes)
	if err != nil {
		log.Fatalln(err)
	}

	// Ensure the backup is durable.
	err = dirSaver.Flush()
	if err != nil {
		err = fmt.Errorf("dirSaver.Flush: %v", err)
		return
	}

	// Register the successful backup.
	completedJob := registry.CompletedJob{
		StartTime: startTime,
		Name:      *g_jobName,
		Score:     score,
	}

	if err = reg.RecordBackup(completedJob); err != nil {
		log.Fatalln("Recoding to registry:", err)
	}

	log.Printf(
		"Successfully backed up with score %v. Start time: %v\n",
		score.Hex(),
		startTime.UTC())

	// Store state for next time.
	saveStateTicker.Stop()
	log.Println("Writing out final state file...")
	saveState()
}
<commit_msg>Use save.List.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"log"
	"os"
	"time"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/config"
	"github.com/jacobsa/comeback/internal/registry"
	"github.com/jacobsa/comeback/internal/save"
)

var cmdSave = &Command{
	Name: "save",
}

var g_jobName = cmdSave.Flags.String(
	"job",
	"",
	"Job name within the config file.")

var g_discardScoreCache = cmdSave.Flags.Bool(
	"discard_score_cache",
	false,
	"If set, always recompute file hashes; don't rely on stat info.",
)

var fListOnly = cmdSave.Flags.Bool(
	"list_only",
	false,
	"If set, list the files that would be backed up but do nothing further.")

func init() {
	cmdSave.Run = runSave // Break flag-related dependency loop.
}

func saveStatePeriodically(c <-chan time.Time) {
	for _ = range c {
		log.Println("Writing out state file.")
		saveState()
	}
}

func doList(job *config.Job) (err error) {
	err = save.List(
		context.Background(),
		os.Stdout,
		job.BasePath,
		job.Excludes)

	if err != nil {
		err = fmt.Errorf("save.List: %v", err)
		return
	}

	return
}

func runSave(args []string) {
	cfg := getConfig()

	// Look for the specified job.
	if *g_jobName == "" {
		log.Fatalln("You must set the -job flag.")
	}

	job, ok := cfg.Jobs[*g_jobName]
	if !ok {
		log.Fatalln("Unknown job:", *g_jobName)
	}

	// Special case: visit the file system only if --list_only is set.
	//
	// TODO(jacobsa): Integrate this into the pipeline when it exists. See issue
	// #21.
	if *fListOnly {
		err := doList(job)
		if err != nil {
			log.Fatalf("doList: %v", err)
			return
		}

		return
	}

	// Grab dependencies. Make sure to get the registry first, because otherwise
	// the user will have to wait for bucket keys to be listed before being
	// prompted for a crypto password.
	//
	// Make sure to do this before setting up state saving below, because these
	// calls may modify the state struct.
	reg := getRegistry()
	dirSaver := getDirSaver()

	// Periodically save state.
	const saveStatePeriod = 15 * time.Second
	saveStateTicker := time.NewTicker(saveStatePeriod)
	go saveStatePeriodically(saveStateTicker.C)

	// Choose a start time for the job.
	startTime := time.Now()

	// Call the directory saver.
	score, err := dirSaver.Save(job.BasePath, "", job.Excludes)
	if err != nil {
		log.Fatalln(err)
	}

	// Ensure the backup is durable.
	err = dirSaver.Flush()
	if err != nil {
		err = fmt.Errorf("dirSaver.Flush: %v", err)
		return
	}

	// Register the successful backup.
	completedJob := registry.CompletedJob{
		StartTime: startTime,
		Name:      *g_jobName,
		Score:     score,
	}

	if err = reg.RecordBackup(completedJob); err != nil {
		log.Fatalln("Recoding to registry:", err)
	}

	log.Printf(
		"Successfully backed up with score %v. Start time: %v\n",
		score.Hex(),
		startTime.UTC())

	// Store state for next time.
	saveStateTicker.Stop()
	log.Println("Writing out final state file...")
	saveState()
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"github.com/kr/fs"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

var cmdSave = &Command{
	Usage: "save [-copy=false] [packages]",
	Short: "list and copy dependencies into Godeps",
	Long: `
Save writes a list of the dependencies of the named packages along
with the exact source control revision of each dependency, and copies
their source code into a subdirectory.

The dependency list is a JSON document with the following structure:

	type Godeps struct {
		ImportPath string
		GoVersion  string   // Abridged output of 'go version'.
		Packages   []string // Arguments to godep save, if any.
		Deps       []struct {
			ImportPath string
			Comment    string // Tag or description of commit.
			Rev        string // VCS-specific commit ID.
		}
	}

If -copy=false is given, the list alone is written to file Godeps.

Otherwise, the list is written to Godeps/Godeps.json, and source
code for all dependencies is copied into Godeps/_workspace.

For more about specifying packages, see 'go help packages'.
`,
	Run: runSave,
}

var saveCopy = true

func init() {
	cmdSave.Flag.BoolVar(&saveCopy, "copy", true, "copy source code")
}

func runSave(cmd *Command, args []string) {
	// Remove Godeps before listing packages, so that args
	// such as ./... don't match anything in there.
	if err := os.RemoveAll("Godeps"); err != nil {
		log.Fatalln(err)
	}
	g := &Godeps{
		ImportPath: MustLoadPackages(".")[0].ImportPath,
		GoVersion:  mustGoVersion(),
	}
	if len(args) > 0 {
		g.Packages = args
	} else {
		args = []string{"."}
	}
	a := MustLoadPackages(args...)
	err := g.Load(a)
	if err != nil {
		log.Fatalln(err)
	}
	if a := badSandboxVCS(g.Deps); a != nil && !saveCopy {
		log.Println("Unsupported sandbox VCS:", strings.Join(a, ", "))
		log.Printf("Instead, run: godep save -copy %s", strings.Join(args, " "))
		os.Exit(1)
	}
	if g.Deps == nil {
		g.Deps = make([]Dependency, 0) // produce json [], not null
	}
	manifest := "Godeps"
	if saveCopy {
		manifest = filepath.Join("Godeps", "Godeps.json")
		// We use a name starting with "_" so the go tool
		// ignores this directory when traversing packages
		// starting at the project's root. For example,
		//   godep go list ./...
		workspace := filepath.Join("Godeps", "_workspace")
		err = copySrc(workspace, g)
		if err != nil {
			log.Fatalln(err)
		}
		path := filepath.Join("Godeps", "Readme")
		err = writeFile(path, strings.TrimSpace(Readme)+"\n")
		if err != nil {
			log.Println(err)
		}
		writeVCSIgnore(workspace)
	}
	f, err := os.Create(manifest)
	if err != nil {
		log.Fatalln(err)
	}
	_, err = g.WriteTo(f)
	if err != nil {
		log.Fatalln(err)
	}
	err = f.Close()
	if err != nil {
		log.Fatalln(err)
	}
}

// badSandboxVCS returns a list of VCSes that don't work
// with the `godep go` sandbox code.
func badSandboxVCS(deps []Dependency) (a []string) {
	for _, d := range deps {
		if d.vcs.CreateCmd == "" {
			a = append(a, d.vcs.vcs.Name)
		}
	}
	sort.Strings(a)
	return uniq(a)
}

func copySrc(dir string, g *Godeps) error {
	ok := true
	for _, dep := range g.Deps {
		w := fs.Walk(dep.dir)
		for w.Step() {
			if w.Err() != nil {
				log.Println(w.Err())
				ok = false
				continue
			}
			if c := w.Stat().Name()[0]; c == '.' || c == '_' {
				// Skip directories using a rule similar to how
				// the go tool enumerates packages.
				// See $GOROOT/src/cmd/go/main.go:/matchPackagesInFs
				w.SkipDir()
			}
			if w.Stat().IsDir() {
				continue
			}
			dst := filepath.Join(dir, w.Path()[len(dep.ws)+1:])
			if err := copyFile(dst, w.Path()); err != nil {
				log.Println(err)
				ok = false
			}
		}
	}
	if !ok {
		return errors.New("error copying source code")
	}
	return nil
}

// copyFile copies a regular file from src to dst.
// dst is opened with os.Create.
func copyFile(dst, src string) error {
	err := os.MkdirAll(filepath.Dir(dst), 0777)
	if err != nil {
		return err
	}

	linkDst, err := os.Readlink(src)
	if err == nil {
		return os.Symlink(linkDst, dst)
	}

	r, err := os.Open(src)
	if err != nil {
		return err
	}
	defer r.Close()

	w, err := os.Create(dst)
	if err != nil {
		return err
	}

	_, err = io.Copy(w, r)
	err1 := w.Close()
	if err == nil {
		err = err1
	}

	return err
}

// Func writeVCSIgnore writes "ignore" files inside dir for known VCSs,
// so that dir/pkg and dir/bin don't accidentally get committed.
// It logs any errors it encounters.
func writeVCSIgnore(dir string) {
	// Currently git is the only VCS for which we know how to do this.
	// Mercurial and Bazaar have similar mechasims, but they apparently
	// require writing files outside of dir.
	const ignore = "/pkg\n/bin\n"
	name := filepath.Join(dir, ".gitignore")
	err := writeFile(name, ignore)
	if err != nil {
		log.Println(err)
	}
}

// writeFile is like ioutil.WriteFile but it creates
// intermediate directories with os.MkdirAll.
func writeFile(name, body string) error {
	err := os.MkdirAll(filepath.Dir(name), 0777)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(name, []byte(body), 0666)
}

const Readme = `
This directory tree is generated automatically by godep.

Please do not edit.

See https://github.com/kr/godep for more information.
`
<commit_msg>avoid off-by-one error in path manipulation<commit_after>package main

import (
	"errors"
	"github.com/kr/fs"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

var cmdSave = &Command{
	Usage: "save [-copy=false] [packages]",
	Short: "list and copy dependencies into Godeps",
	Long: `
Save writes a list of the dependencies of the named packages along
with the exact source control revision of each dependency, and copies
their source code into a subdirectory.

The dependency list is a JSON document with the following structure:

	type Godeps struct {
		ImportPath string
		GoVersion  string   // Abridged output of 'go version'.
		Packages   []string // Arguments to godep save, if any.
		Deps       []struct {
			ImportPath string
			Comment    string // Tag or description of commit.
			Rev        string // VCS-specific commit ID.
		}
	}

If -copy=false is given, the list alone is written to file Godeps.

Otherwise, the list is written to Godeps/Godeps.json, and source
code for all dependencies is copied into Godeps/_workspace.

For more about specifying packages, see 'go help packages'.
`,
	Run: runSave,
}

var saveCopy = true

func init() {
	cmdSave.Flag.BoolVar(&saveCopy, "copy", true, "copy source code")
}

func runSave(cmd *Command, args []string) {
	// Remove Godeps before listing packages, so that args
	// such as ./... don't match anything in there.
	if err := os.RemoveAll("Godeps"); err != nil {
		log.Fatalln(err)
	}
	g := &Godeps{
		ImportPath: MustLoadPackages(".")[0].ImportPath,
		GoVersion:  mustGoVersion(),
	}
	if len(args) > 0 {
		g.Packages = args
	} else {
		args = []string{"."}
	}
	a := MustLoadPackages(args...)
	err := g.Load(a)
	if err != nil {
		log.Fatalln(err)
	}
	if a := badSandboxVCS(g.Deps); a != nil && !saveCopy {
		log.Println("Unsupported sandbox VCS:", strings.Join(a, ", "))
		log.Printf("Instead, run: godep save -copy %s", strings.Join(args, " "))
		os.Exit(1)
	}
	if g.Deps == nil {
		g.Deps = make([]Dependency, 0) // produce json [], not null
	}
	manifest := "Godeps"
	if saveCopy {
		manifest = filepath.Join("Godeps", "Godeps.json")
		// We use a name starting with "_" so the go tool
		// ignores this directory when traversing packages
		// starting at the project's root. For example,
		//   godep go list ./...
		workspace := filepath.Join("Godeps", "_workspace")
		err = copySrc(filepath.Join(workspace, "src"), g)
		if err != nil {
			log.Fatalln(err)
		}
		path := filepath.Join("Godeps", "Readme")
		err = writeFile(path, strings.TrimSpace(Readme)+"\n")
		if err != nil {
			log.Println(err)
		}
		writeVCSIgnore(workspace)
	}
	f, err := os.Create(manifest)
	if err != nil {
		log.Fatalln(err)
	}
	_, err = g.WriteTo(f)
	if err != nil {
		log.Fatalln(err)
	}
	err = f.Close()
	if err != nil {
		log.Fatalln(err)
	}
}

// badSandboxVCS returns a list of VCSes that don't work
// with the `godep go` sandbox code.
func badSandboxVCS(deps []Dependency) (a []string) {
	for _, d := range deps {
		if d.vcs.CreateCmd == "" {
			a = append(a, d.vcs.vcs.Name)
		}
	}
	sort.Strings(a)
	return uniq(a)
}

func copySrc(dir string, g *Godeps) error {
	ok := true
	for _, dep := range g.Deps {
		srcdir := filepath.Join(dep.ws, "src")
		w := fs.Walk(dep.dir)
		for w.Step() {
			if w.Err() != nil {
				log.Println(w.Err())
				ok = false
				continue
			}
			if c := w.Stat().Name()[0]; c == '.' || c == '_' {
				// Skip directories using a rule similar to how
				// the go tool enumerates packages.
				// See $GOROOT/src/cmd/go/main.go:/matchPackagesInFs
				w.SkipDir()
			}
			if w.Stat().IsDir() {
				continue
			}
			rel, err := filepath.Rel(srcdir, w.Path())
			if err != nil { // this should never happen
				log.Println(err)
				ok = false
				continue
			}
			dst := filepath.Join(dir, rel)
			if err := copyFile(dst, w.Path()); err != nil {
				log.Println(err)
				ok = false
			}
		}
	}
	if !ok {
		return errors.New("error copying source code")
	}
	return nil
}

// copyFile copies a regular file from src to dst.
// dst is opened with os.Create.
func copyFile(dst, src string) error {
	err := os.MkdirAll(filepath.Dir(dst), 0777)
	if err != nil {
		return err
	}

	linkDst, err := os.Readlink(src)
	if err == nil {
		return os.Symlink(linkDst, dst)
	}

	r, err := os.Open(src)
	if err != nil {
		return err
	}
	defer r.Close()

	w, err := os.Create(dst)
	if err != nil {
		return err
	}

	_, err = io.Copy(w, r)
	err1 := w.Close()
	if err == nil {
		err = err1
	}

	return err
}

// Func writeVCSIgnore writes "ignore" files inside dir for known VCSs,
// so that dir/pkg and dir/bin don't accidentally get committed.
// It logs any errors it encounters.
func writeVCSIgnore(dir string) {
	// Currently git is the only VCS for which we know how to do this.
	// Mercurial and Bazaar have similar mechasims, but they apparently
	// require writing files outside of dir.
	const ignore = "/pkg\n/bin\n"
	name := filepath.Join(dir, ".gitignore")
	err := writeFile(name, ignore)
	if err != nil {
		log.Println(err)
	}
}

// writeFile is like ioutil.WriteFile but it creates
// intermediate directories with os.MkdirAll.
func writeFile(name, body string) error {
	err := os.MkdirAll(filepath.Dir(name), 0777)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(name, []byte(body), 0666)
}

const Readme = `
This directory tree is generated automatically by godep.

Please do not edit.

See https://github.com/kr/godep for more information.
`
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/brotherlogic/godiscogs"
	"golang.org/x/net/context"

	pbd "github.com/brotherlogic/godiscogs"
	"github.com/golang/protobuf/proto"
	"strconv"
	"time"
)

import pb "github.com/brotherlogic/discogssyncer/server"

// GetRelease Gets the release and metadata for the release
func (syncer *Syncer) GetRelease(id int, folder int) (*pbd.Release, *pb.ReleaseMetadata) {
	releaseData, err := ioutil.ReadFile(syncer.saveLocation + "/" + strconv.Itoa(folder) + "/" + strconv.Itoa(id) + ".release")
	if err != nil {
		return nil, nil
	}

	metadataData, _ := ioutil.ReadFile(syncer.saveLocation + "/static-metadata/" + strconv.Itoa(id) + ".metadata")
	release := &pbd.Release{}
	metadata := &pb.ReleaseMetadata{}

	proto.Unmarshal(releaseData, release)
	proto.Unmarshal(metadataData, metadata)

	return release, metadata
}

// GetMonthlySpend gets the monthly spend
func (syncer *Syncer) GetMonthlySpend(ctx context.Context, req *pb.SpendRequest) (*pb.SpendResponse, error) {
	spend := 0
	for _, rel := range syncer.relMap {
		_, metadata := syncer.GetRelease(int(rel.Id), int(rel.FolderId))
		datev := time.Unix(metadata.DateAdded, 0)
		if datev.Year() == int(req.Year) && int32(datev.Month()) == req.Month {
			spend += int(metadata.Cost)
		}
	}

	return &pb.SpendResponse{TotalSpend: int32(spend)}, nil
}

func (syncer *Syncer) saveMetadata(rel *godiscogs.Release) {
	metadataRoot := syncer.saveLocation + "/static-metadata/"
	metadataPath := metadataRoot + strconv.Itoa(int(rel.Id)) + ".metadata"
	if _, err := os.Stat(metadataRoot); os.IsNotExist(err) {
		os.MkdirAll(metadataRoot, 0777)
	}

	metadata := &pb.ReleaseMetadata{}
	if _, err := os.Stat(metadataPath); os.IsNotExist(err) {
		metadata.DateAdded = time.Now().Unix()
		metadata.DateRefreshed = time.Now().Unix()
	} else {
		data, _ := ioutil.ReadFile(metadataPath)
		proto.Unmarshal(data, metadata)
		metadata.DateRefreshed = time.Now().Unix()
	}
	data, _ := proto.Marshal(metadata)
	ioutil.WriteFile(metadataPath, data, 0644)
}

func (syncer *Syncer) deleteRelease(rel *godiscogs.Release, folder int) {
	os.Remove(syncer.saveLocation + "/" + strconv.Itoa(folder) + "/" + strconv.Itoa(int(rel.Id)) + ".release")
}

func (syncer *Syncer) saveRelease(rel *godiscogs.Release, folder int) {
	log.Printf("SAVING %v -> %v", rel, folder)

	//Check that the save folder exists
	savePath := syncer.saveLocation + "/" + strconv.Itoa(folder) + "/"
	if _, err := os.Stat(savePath); os.IsNotExist(err) {
		os.MkdirAll(savePath, 0777)
	}

	data, _ := proto.Marshal(rel)
	ioutil.WriteFile(savePath+strconv.Itoa(int(rel.Id))+".release", data, 0644)
	syncer.saveMetadata(rel)
}

type saver interface {
	GetCollection() []godiscogs.Release
	GetFolders() []godiscogs.Folder
	GetRelease(id int) (godiscogs.Release, error)
	MoveToFolder(folderID int, releaseID int, instanceID int, newFolderID int)
	AddToFolder(folderID int, releaseID int)
	SetRating(folderID int, releaseID int, instanceID int, rating int)
	GetWantlist() ([]pbd.Release, error)
	RemoveFromWantlist(releaseID int)
	AddToWantlist(releaseID int)
}

// EditWant edits a want in the wantlist
func (syncer *Syncer) EditWant(ctx context.Context, wantIn *pb.Want) (*pb.Want, error) {
	for _, want := range syncer.wants.Want {
		if want.ReleaseId == wantIn.ReleaseId {
			want.Valued = wantIn.Valued
		}
	}
	syncer.saveWantList()

	return wantIn, nil
}

// SaveCollection writes out the full collection to files.
func (syncer *Syncer) SaveCollection(retr saver) {
	releases := retr.GetCollection()
	for _, release := range releases {
		fullRelease, _ := retr.GetRelease(int(release.Id))
		fullRelease.InstanceId = release.InstanceId
		fullRelease.FolderId = release.FolderId
		fullRelease.Rating = release.Rating
		syncer.saveRelease(&fullRelease, int(release.FolderId))
	}
	folders := retr.GetFolders()
	folderList := pb.FolderList{}
	for i := range folders {
		folder := folders[i]
		folderList.Folders = append(folderList.Folders, &folder)
	}
	syncer.SaveFolders(&folderList)
}

// SyncWantlist syncs the wantlist with the server
func (syncer *Syncer) SyncWantlist() {
	wants, _ := syncer.retr.GetWantlist()

	for _, want := range wants {
		seen := false
		var val *pb.Want
		for _, swant := range syncer.wants.Want {
			if swant.ReleaseId == want.Id {
				seen = true
				val = swant
			}
		}

		if seen {
			val.Wanted = true
		} else {
			syncer.wants.Want = append(syncer.wants.Want, &pb.Want{ReleaseId: want.Id, Valued: false, Wanted: true})
		}
	}

	// Cache the want list releases
	for _, want := range syncer.wants.Want {
		release, _ := syncer.retr.GetRelease(int(want.ReleaseId))
		syncer.saveRelease(&release, -5)
	}

	syncer.saveWantList()
}

func (syncer *Syncer) getFolders() *pb.FolderList {
	data, _ := ioutil.ReadFile(syncer.saveLocation + "/metadata/folders")
	folderData := &pb.FolderList{}
	proto.Unmarshal(data, folderData)
	return folderData
}

// GetSingleRelease gets a single release
func (syncer *Syncer) GetSingleRelease(ctx context.Context, in *pbd.Release) (*pbd.Release, error) {
	if val, ok := syncer.relMap[in.Id]; ok {
		return val, nil
	}
	return nil, errors.New("Unable to find release")
}

// CollapseWantlist collapses the wantlist
func (syncer *Syncer) CollapseWantlist(ctx context.Context, in *pb.Empty) (*pb.Wantlist, error) {
	for _, want := range syncer.wants.Want {
		if !want.Valued {
			log.Printf("AVOIDING %v", want)
			syncer.retr.RemoveFromWantlist(int(want.ReleaseId))
			want.Wanted = false
		}
	}

	return &syncer.wants, nil
}

// RebuildWantlist rebuilds the wantlist
func (syncer *Syncer) RebuildWantlist(ctx context.Context, in *pb.Empty) (*pb.Wantlist, error) {
	for _, want := range syncer.wants.Want {
		syncer.retr.AddToWantlist(int(want.ReleaseId))
		want.Wanted = true
	}

	return &syncer.wants, nil
}

// MoveToFolder moves a release to the specified folder
func (syncer *Syncer) MoveToFolder(ctx context.Context, in *pb.ReleaseMove) (*pb.Empty, error) {
	syncer.retr.MoveToFolder(int(in.Release.FolderId), int(in.Release.Id), int(in.Release.InstanceId), int(in.NewFolderId))
	oldFolder := int(in.Release.FolderId)
	fullRelease, _ := syncer.retr.GetRelease(int(in.Release.Id))
	fullRelease.FolderId = int32(in.NewFolderId)
	syncer.relMap[fullRelease.Id] = &fullRelease

	syncer.Log(fmt.Sprintf("Moving %v from %v to %v", in.Release.Id, in.Release.FolderId, in.NewFolderId))
	syncer.saveRelease(&fullRelease, int(in.NewFolderId))
	syncer.deleteRelease(&fullRelease, oldFolder)
	return &pb.Empty{}, nil
}

// AddToFolder adds a release to the specified folder
func (syncer *Syncer) AddToFolder(ctx context.Context, in *pb.ReleaseMove) (*pb.Empty, error) {
	syncer.retr.AddToFolder(int(in.NewFolderId), int(in.Release.Id))
	fullRelease, _ := syncer.retr.GetRelease(int(in.Release.Id))
	fullRelease.FolderId = int32(in.NewFolderId)
	syncer.saveRelease(&fullRelease, int(in.NewFolderId))
	syncer.relMap[in.Release.Id] = &fullRelease
	return &pb.Empty{}, nil
}

// UpdateRating updates the rating of a release
func (syncer *Syncer) UpdateRating(ctx context.Context, in *pbd.Release) (*pb.Empty, error) {
	syncer.retr.SetRating(int(in.FolderId), int(in.Id), int(in.InstanceId), int(in.Rating))
	fullRelease, _ := syncer.GetRelease(int(in.Id), int(in.FolderId))
	fullRelease.Rating = int32(in.Rating)
	syncer.relMap[in.Id] = fullRelease
	syncer.saveRelease(fullRelease, int(fullRelease.FolderId))
	return &pb.Empty{}, nil
}

// UpdateMetadata updates the metadata of a given record
func (syncer *Syncer) UpdateMetadata(ctx context.Context, in *pb.MetadataUpdate) (*pb.ReleaseMetadata, error) {
	release, metadata := syncer.GetRelease(int(in.Release.Id), int(in.Release.FolderId))
	if release == nil {
		return nil, errors.New("Unable to locate release")
	}
	proto.Merge(metadata, in.Update)

	metadataRoot := syncer.saveLocation + "/static-metadata/"
	metadataPath := metadataRoot + strconv.Itoa(int(in.Release.Id)) + ".metadata"
	data, _ := proto.Marshal(metadata)
	ioutil.WriteFile(metadataPath, data, 0644)

	return metadata, nil
}

// GetWantlist gets the wantlist
func (syncer *Syncer) GetWantlist(ctx context.Context, in *pb.Empty) (*pb.Wantlist, error) {
	return &syncer.wants, nil
}

// GetMetadata gets the metadata for a given release
func (syncer *Syncer) GetMetadata(ctx context.Context, in *pbd.Release) (*pb.ReleaseMetadata, error) {
	_, metadata := syncer.GetRelease(int(in.Id), int(in.FolderId))
	return metadata, nil
}

// GetReleasesInFolder serves up the releases in a given folder
func (syncer *Syncer) GetReleasesInFolder(ctx context.Context, in *pb.FolderList) (*pb.ReleaseList, error) {

	releases := pb.ReleaseList{}
	for _, folderSpec := range in.Folders {
		folders := syncer.getFolders()
		for _, folder := range folders.Folders {
			if folder.Name == folderSpec.Name || folder.Id == folderSpec.Id {
				innerReleases := syncer.getReleases(int(folder.Id))
				releases.Releases = append(releases.Releases, innerReleases.Releases...)
			}
		}
	}

	return &releases, nil
}

func (syncer *Syncer) getReleases(folderID int) *pb.ReleaseList {
	releases := pb.ReleaseList{}
	files, _ := ioutil.ReadDir(syncer.saveLocation + "/" + strconv.Itoa(folderID) + "/")
	for _, file := range files {
		if strings.HasSuffix(file.Name(), ".release") {
			data, _ := ioutil.ReadFile(syncer.saveLocation + "/" + strconv.Itoa(folderID) + "/" + file.Name())
			release := &pbd.Release{}
			proto.Unmarshal(data, release)
			releases.Releases = append(releases.Releases, release)
		}
	}
	return &releases
}

func (syncer *Syncer) initWantlist() {
	wldata, _ := ioutil.ReadFile(syncer.saveLocation + "/metadata/wantlist")
	proto.Unmarshal(wldata, &syncer.wants)

	for _, want := range syncer.wants.Want {
		rel, _ := syncer.GetRelease(int(want.ReleaseId), -5)
		rel.FolderId = -5
		syncer.relMap[rel.Id] = rel
	}
}

func (syncer *Syncer) saveWantList() {
	data, _ := proto.Marshal(&syncer.wants)
	savePath := syncer.saveLocation + "/metadata/"
	if _, err := os.Stat(savePath); os.IsNotExist(err) {
		os.MkdirAll(savePath, 0777)
	}
	ioutil.WriteFile(savePath+"wantlist", data, 0644)
}

// SaveFolders saves out the list of folders
func (syncer *Syncer) SaveFolders(list *pb.FolderList) {
	savePath := syncer.saveLocation + "/metadata/"
	if _, err := os.Stat(savePath); os.IsNotExist(err) {
		os.MkdirAll(savePath, 0777)
	}

	data, _ := proto.Marshal(list)
	ioutil.WriteFile(savePath+"folders", data, 0644)
}

// GetCollection serves up the whole of the collection
func (syncer *Syncer) GetCollection(ctx context.Context, in *pb.Empty) (*pb.ReleaseList, error) {
	releases := pb.ReleaseList{}
	bfiles, _ := ioutil.ReadDir(syncer.saveLocation)
	for _, bfile := range bfiles {
		if bfile.IsDir() && bfile.Name() != "-5" {
			folderID, _ := strconv.Atoi(bfile.Name())
			for _, release := range syncer.getReleases(folderID).Releases {
				releases.Releases = append(releases.Releases, release)
			}
		}
	}
	return &releases, nil
}
<commit_msg>Removed save line<commit_after>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/brotherlogic/godiscogs"
	"golang.org/x/net/context"

	pbd "github.com/brotherlogic/godiscogs"
	"github.com/golang/protobuf/proto"
	"strconv"
	"time"
)

import pb "github.com/brotherlogic/discogssyncer/server"

// GetRelease Gets the release and metadata for the release
func (syncer *Syncer) GetRelease(id int, folder int) (*pbd.Release, *pb.ReleaseMetadata) {
	releaseData, err := ioutil.ReadFile(syncer.saveLocation + "/" + strconv.Itoa(folder) + "/" + strconv.Itoa(id) + ".release")
	if err != nil {
		return nil, nil
	}

	metadataData, _ := ioutil.ReadFile(syncer.saveLocation + "/static-metadata/" + strconv.Itoa(id) + ".metadata")
	release := &pbd.Release{}
	metadata := &pb.ReleaseMetadata{}

	proto.Unmarshal(releaseData, release)
	proto.Unmarshal(metadataData, metadata)

	return release, metadata
}

// GetMonthlySpend gets the monthly spend
func (syncer *Syncer) GetMonthlySpend(ctx context.Context, req *pb.SpendRequest) (*pb.SpendResponse, error) {
	spend := 0
	for _, rel := range syncer.relMap {
		_, metadata := syncer.GetRelease(int(rel.Id), int(rel.FolderId))
		datev := time.Unix(metadata.DateAdded, 0)
		if datev.Year() == int(req.Year) && int32(datev.Month()) == req.Month {
			spend += int(metadata.Cost)
		}
	}

	return &pb.SpendResponse{TotalSpend: int32(spend)}, nil
}

func (syncer *Syncer) saveMetadata(rel *godiscogs.Release) {
	metadataRoot := syncer.saveLocation + "/static-metadata/"
	metadataPath := metadataRoot + strconv.Itoa(int(rel.Id)) + ".metadata"
	if _, err := os.Stat(metadataRoot); os.IsNotExist(err) {
		os.MkdirAll(metadataRoot, 0777)
	}

	metadata := &pb.ReleaseMetadata{}
	if _, err := os.Stat(metadataPath); os.IsNotExist(err) {
		metadata.DateAdded = time.Now().Unix()
		metadata.DateRefreshed = time.Now().Unix()
	} else {
		data, _ := ioutil.ReadFile(metadataPath)
		proto.Unmarshal(data, metadata)
		metadata.DateRefreshed = time.Now().Unix()
	}
	data, _ := proto.Marshal(metadata)
	ioutil.WriteFile(metadataPath, data, 0644)
}

func (syncer *Syncer) deleteRelease(rel *godiscogs.Release, folder int) {
	os.Remove(syncer.saveLocation + "/" + strconv.Itoa(folder) + "/" + strconv.Itoa(int(rel.Id)) + ".release")
}

func (syncer *Syncer) saveRelease(rel *godiscogs.Release, folder int) {
	//Check that the save folder exists
	savePath := syncer.saveLocation + "/" + strconv.Itoa(folder) + "/"
	if _, err := os.Stat(savePath); os.IsNotExist(err) {
		os.MkdirAll(savePath, 0777)
	}

	data, _ := proto.Marshal(rel)
	ioutil.WriteFile(savePath+strconv.Itoa(int(rel.Id))+".release", data, 0644)
	syncer.saveMetadata(rel)
}

type saver interface {
	GetCollection() []godiscogs.Release
	GetFolders() []godiscogs.Folder
	GetRelease(id int) (godiscogs.Release, error)
	MoveToFolder(folderID int, releaseID int, instanceID int, newFolderID int)
	AddToFolder(folderID int, releaseID int)
	SetRating(folderID int, releaseID int, instanceID int, rating int)
	GetWantlist() ([]pbd.Release, error)
	RemoveFromWantlist(releaseID int)
	AddToWantlist(releaseID int)
}

// EditWant edits a want in the wantlist
func (syncer *Syncer) EditWant(ctx context.Context, wantIn *pb.Want) (*pb.Want, error) {
	for _, want := range syncer.wants.Want {
		if want.ReleaseId == wantIn.ReleaseId {
			want.Valued = wantIn.Valued
		}
	}
	syncer.saveWantList()

	return wantIn, nil
}

// SaveCollection writes out the full collection to files.
func (syncer *Syncer) SaveCollection(retr saver) {
	releases := retr.GetCollection()
	for _, release := range releases {
		fullRelease, _ := retr.GetRelease(int(release.Id))
		fullRelease.InstanceId = release.InstanceId
		fullRelease.FolderId = release.FolderId
		fullRelease.Rating = release.Rating
		syncer.saveRelease(&fullRelease, int(release.FolderId))
	}
	folders := retr.GetFolders()
	folderList := pb.FolderList{}
	for i := range folders {
		folder := folders[i]
		folderList.Folders = append(folderList.Folders, &folder)
	}
	syncer.SaveFolders(&folderList)
}

// SyncWantlist syncs the wantlist with the server
func (syncer *Syncer) SyncWantlist() {
	wants, _ := syncer.retr.GetWantlist()

	for _, want := range wants {
		seen := false
		var val *pb.Want
		for _, swant := range syncer.wants.Want {
			if swant.ReleaseId == want.Id {
				seen = true
				val = swant
			}
		}

		if seen {
			val.Wanted = true
		} else {
			syncer.wants.Want = append(syncer.wants.Want, &pb.Want{ReleaseId: want.Id, Valued: false, Wanted: true})
		}
	}

	// Cache the want list releases
	for _, want := range syncer.wants.Want {
		release, _ := syncer.retr.GetRelease(int(want.ReleaseId))
		syncer.saveRelease(&release, -5)
	}

	syncer.saveWantList()
}

func (syncer *Syncer) getFolders() *pb.FolderList {
	data, _ := ioutil.ReadFile(syncer.saveLocation + "/metadata/folders")
	folderData := &pb.FolderList{}
	proto.Unmarshal(data, folderData)
	return folderData
}

// GetSingleRelease gets a single release
func (syncer *Syncer) GetSingleRelease(ctx context.Context, in *pbd.Release) (*pbd.Release, error) {
	if val, ok := syncer.relMap[in.Id]; ok {
		return val, nil
	}
	return nil, errors.New("Unable to find release")
}

// CollapseWantlist collapses the wantlist
func (syncer *Syncer) CollapseWantlist(ctx context.Context, in *pb.Empty) (*pb.Wantlist, error) {
	for _, want := range syncer.wants.Want {
		if !want.Valued {
			log.Printf("AVOIDING %v", want)
			syncer.retr.RemoveFromWantlist(int(want.ReleaseId))
			want.Wanted = false
		}
	}

	return &syncer.wants, nil
}

// RebuildWantlist rebuilds the wantlist
func (syncer *Syncer) RebuildWantlist(ctx context.Context, in *pb.Empty) (*pb.Wantlist, error) {
	for _, want := range syncer.wants.Want {
		syncer.retr.AddToWantlist(int(want.ReleaseId))
		want.Wanted = true
	}

	return &syncer.wants, nil
}

// MoveToFolder moves a release to the specified folder
func (syncer *Syncer) MoveToFolder(ctx context.Context, in *pb.ReleaseMove) (*pb.Empty, error) {
	syncer.retr.MoveToFolder(int(in.Release.FolderId), int(in.Release.Id), int(in.Release.InstanceId), int(in.NewFolderId))
	oldFolder := int(in.Release.FolderId)
	fullRelease, _ := syncer.retr.GetRelease(int(in.Release.Id))
	fullRelease.FolderId = int32(in.NewFolderId)
	syncer.relMap[fullRelease.Id] = &fullRelease

	syncer.Log(fmt.Sprintf("Moving %v from %v to %v", in.Release.Id, in.Release.FolderId, in.NewFolderId))
	syncer.saveRelease(&fullRelease, int(in.NewFolderId))
	syncer.deleteRelease(&fullRelease, oldFolder)
	return &pb.Empty{}, nil
}

// AddToFolder adds a release to the specified folder
func (syncer *Syncer) AddToFolder(ctx context.Context, in *pb.ReleaseMove) (*pb.Empty, error) {
	syncer.retr.AddToFolder(int(in.NewFolderId), int(in.Release.Id))
	fullRelease, _ := syncer.retr.GetRelease(int(in.Release.Id))
	fullRelease.FolderId = int32(in.NewFolderId)
	syncer.saveRelease(&fullRelease, int(in.NewFolderId))
	syncer.relMap[in.Release.Id] = &fullRelease
	return &pb.Empty{}, nil
}

// UpdateRating updates the rating of a release
func (syncer *Syncer) UpdateRating(ctx context.Context, in *pbd.Release) (*pb.Empty, error) {
	syncer.retr.SetRating(int(in.FolderId), int(in.Id), int(in.InstanceId), int(in.Rating))
	fullRelease, _ := syncer.GetRelease(int(in.Id), int(in.FolderId))
	fullRelease.Rating = int32(in.Rating)
	syncer.relMap[in.Id] = fullRelease
	syncer.saveRelease(fullRelease, int(fullRelease.FolderId))
	return &pb.Empty{}, nil
}

// UpdateMetadata updates the metadata of a given record
func (syncer *Syncer) UpdateMetadata(ctx context.Context, in *pb.MetadataUpdate) (*pb.ReleaseMetadata, error) {
	release, metadata := syncer.GetRelease(int(in.Release.Id), int(in.Release.FolderId))
	if release == nil {
		return nil, errors.New("Unable to locate release")
	}
	proto.Merge(metadata, in.Update)

	metadataRoot := syncer.saveLocation + "/static-metadata/"
	metadataPath := metadataRoot + strconv.Itoa(int(in.Release.Id)) + ".metadata"
	data, _ := proto.Marshal(metadata)
	ioutil.WriteFile(metadataPath, data, 0644)

	return metadata, nil
}

// GetWantlist gets the wantlist
func (syncer *Syncer) GetWantlist(ctx context.Context, in *pb.Empty) (*pb.Wantlist, error) {
	return &syncer.wants, nil
}

// GetMetadata gets the metadata for a given release
func (syncer *Syncer) GetMetadata(ctx context.Context, in *pbd.Release) (*pb.ReleaseMetadata, error) {
	_, metadata := syncer.GetRelease(int(in.Id), int(in.FolderId))
	return metadata, nil
}

// GetReleasesInFolder serves up the releases in a given folder
func (syncer *Syncer) GetReleasesInFolder(ctx context.Context, in *pb.FolderList) (*pb.ReleaseList, error) {

	releases := pb.ReleaseList{}
	for _, folderSpec := range in.Folders {
		folders := syncer.getFolders()
		for _, folder := range folders.Folders {
			if folder.Name == folderSpec.Name || folder.Id == folderSpec.Id {
				innerReleases := syncer.getReleases(int(folder.Id))
				releases.Releases = append(releases.Releases, innerReleases.Releases...)
			}
		}
	}

	return &releases, nil
}

func (syncer *Syncer) getReleases(folderID int) *pb.ReleaseList {
	releases := pb.ReleaseList{}
	files, _ := ioutil.ReadDir(syncer.saveLocation + "/" + strconv.Itoa(folderID) + "/")
	for _, file := range files {
		if strings.HasSuffix(file.Name(), ".release") {
			data, _ := ioutil.ReadFile(syncer.saveLocation + "/" + strconv.Itoa(folderID) + "/" + file.Name())
			release := &pbd.Release{}
			proto.Unmarshal(data, release)
			releases.Releases = append(releases.Releases, release)
		}
	}
	return &releases
}

func (syncer *Syncer) initWantlist() {
	wldata, _ := ioutil.ReadFile(syncer.saveLocation + "/metadata/wantlist")
	proto.Unmarshal(wldata, &syncer.wants)

	for _, want := range syncer.wants.Want {
		rel, _ := syncer.GetRelease(int(want.ReleaseId), -5)
		rel.FolderId = -5
		syncer.relMap[rel.Id] = rel
	}
}

func (syncer *Syncer) saveWantList() {
	data, _ := proto.Marshal(&syncer.wants)
	savePath := syncer.saveLocation + "/metadata/"
	if _, err := os.Stat(savePath); os.IsNotExist(err) {
		os.MkdirAll(savePath, 0777)
	}
	ioutil.WriteFile(savePath+"wantlist", data, 0644)
}

// SaveFolders saves out the list of folders
func (syncer *Syncer) SaveFolders(list *pb.FolderList) {
	savePath := syncer.saveLocation + "/metadata/"
	if _, err := os.Stat(savePath); os.IsNotExist(err) {
		os.MkdirAll(savePath, 0777)
	}

	data, _ := proto.Marshal(list)
	ioutil.WriteFile(savePath+"folders", data, 0644)
}

// GetCollection serves up the whole of the collection
func (syncer *Syncer) GetCollection(ctx context.Context, in *pb.Empty) (*pb.ReleaseList, error) {
	releases := pb.ReleaseList{}
	bfiles, _ := ioutil.ReadDir(syncer.saveLocation)
	for _, bfile := range bfiles {
		if bfile.IsDir() && bfile.Name() != "-5" {
			folderID, _ := strconv.Atoi(bfile.Name())
			for _, release := range syncer.getReleases(folderID).Releases {
				releases.Releases = append(releases.Releases, release)
			}
		}
	}
	return &releases, nil
}
<|endoftext|>


<commit_before>package kember

import (
	"io"
	"time"
	"fmt"
	"strings"
	"crypto/md5"
	"encoding/hex"
)

func Valid(hash string) bool {
	if len(hash) != 32 {
		return false
	}
	for _, runeValue := range strings.ToLower(hash) {
		if ((runeValue < 'a' || runeValue > 'f') && (runeValue < '0' || runeValue > '9')) {
	        return false
	    }
	}
	return true
}

func Search(curr string, iterations int64) {
	log(0, "search(%v, %v)!", curr, iterations)
	runes := []rune(curr)
	h := md5.New()
	i := int64(0)
	for ; iterations < 0 || i < iterations; i++ {
		if i % 1000000 == 0 {
			log(i, curr)
		}
		h.Reset()
		io.WriteString(h, curr)
		sum := h.Sum(nil)
		hash := hex.EncodeToString(sum[0:16])
		if curr == hash {
			log(i, "%v == %v <-- MATCH!!!", curr, hash)
		}
		increment(runes)
		curr = string(runes)
	}
	log(i, "finished")
}

func log(i int64, msg string, args ...interface{}) {
	if len(args) > 0 {
		msg = fmt.Sprintf(msg, args...)
	}
	fmt.Printf("%8.1fM) [%v] %v\n", float64(i) / 1000000.0, time.Now().Format("2006-01-02 15:04:05 -0700 MST"), msg)
}

func increment(runes []rune) {
	runeCount := len(runes)
	pos := runeCount - 1
	for ; pos >= 0 && runes[pos] == 'f'; pos-- {}
	if pos < 0 {
		log(-1, "OVERFLOW!")
		pos = 0
	}
	for i := pos; i < runeCount; i++ {
		runes[i] = next(runes[i])
	}
}

func next(curr rune) rune {
	if curr == '9' {
		return 'a'
	} else if curr == 'f' {
		return '0'
	} else {
		return curr + 1
	}
}
<commit_msg>changed status logs to be every 10 million and removed overflow check/warning<commit_after>package kember

import (
	"io"
	"time"
	"fmt"
	"strings"
	"crypto/md5"
	"encoding/hex"
)

func Valid(hash string) bool {
	if len(hash) != 32 {
		return false
	}
	for _, runeValue := range strings.ToLower(hash) {
		if ((runeValue < 'a' || runeValue > 'f') && (runeValue < '0' || runeValue > '9')) {
	        return false
	    }
	}
	return true
}

func Search(curr string, iterations int64) {
	log(0, "search(%v, %v)!", curr, iterations)
	runes := []rune(curr)
	h := md5.New()
	i := int64(0)
	for ; iterations < 0 || i < iterations; i++ {
		if i % 10000000 == 0 {
			log(i, curr)
		}
		h.Reset()
		io.WriteString(h, curr)
		sum := h.Sum(nil)
		hash := hex.EncodeToString(sum[0:16])
		if curr == hash {
			log(i, "%v == %v <-- MATCH!!!", curr, hash)
		}
		increment(runes)
		curr = string(runes)
	}
	log(i, "finished")
}

func log(i int64, msg string, args ...interface{}) {
	if len(args) > 0 {
		msg = fmt.Sprintf(msg, args...)
	}
	fmt.Printf("%8.1fM) [%v] %v\n", float64(i) / 1000000.0, time.Now().Format("2006-01-02 15:04:05 -0700 MST"), msg)
}

func increment(runes []rune) {
	runeCount := len(runes)
	pos := runeCount - 1
	for ; pos > 0 && runes[pos] == 'f'; pos-- {}
	for i := pos; i < runeCount; i++ {
		runes[i] = next(runes[i])
	}
}

func next(curr rune) rune {
	if curr == '9' {
		return 'a'
	} else if curr == 'f' {
		return '0'
	} else {
		return curr + 1
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 ActiveState Software Inc. All rights reserved.

package tail

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/hpcloud/tail/ratelimiter"
	"github.com/hpcloud/tail/util"
	"github.com/hpcloud/tail/watch"
	"gopkg.in/tomb.v1"
)

var (
	ErrStop = fmt.Errorf("tail should now stop")
)

// Line contains the line read from the file, along with a timestamp and an error if necessary.
type Line struct {
	Text string
	Time time.Time
	Err  error // Error from tail
}

// NewLine returns a Line with present time.
func NewLine(text string) *Line {
	return &Line{text, time.Now(), nil}
}

// SeekInfo represents arguments to `os.Seek`
type SeekInfo struct {
	Offset int64
	Whence int // os.SEEK_*
}

type logger interface {
	Fatal(v ...interface{})
	Fatalf(format string, v ...interface{})
	Fatalln(v ...interface{})
	Panic(v ...interface{})
	Panicf(format string, v ...interface{})
	Panicln(v ...interface{})
	Print(v ...interface{})
	Printf(format string, v ...interface{})
	Println(v ...interface{})
}

// Config is used to specify how a file must be tailed.
type Config struct {
	// File-specifc
	Location    *SeekInfo // Seek to this location before tailing
	ReOpen      bool      // Reopen recreated files (tail -F)
	MustExist   bool      // Fail early if the file does not exist
	Poll        bool      // Poll for file changes instead of using inotify
	Pipe        bool      // Is a named pipe (mkfifo)
	RateLimiter *ratelimiter.LeakyBucket

	// Generic IO
	Follow      bool // Continue looking for new lines (tail -f)
	MaxLineSize int  // If non-zero, split longer lines into multiple lines

	// Logger, when nil, is set to tail.DefaultLogger
	// To disable logging: set field to tail.DiscardingLogger
	Logger logger
}

// Tail contains info. related to the item being tailed.
type Tail struct {
	Filename string
	Lines    chan *Line
	Config

	file   *os.File
	reader *bufio.Reader

	watcher watch.FileWatcher
	changes *watch.FileChanges

	tomb.Tomb // provides: Done, Kill, Dying

	lk sync.Mutex
}

var (
	// DefaultLogger is used when Config.Logger == nil
	DefaultLogger = log.New(os.Stderr, "", log.LstdFlags)
	// DiscardingLogger can be used to disable logging output
	DiscardingLogger = log.New(ioutil.Discard, "", 0)
)

// File begins tailing the file. Output stream is made available
// via the `Tail.Lines` channel. To handle errors during tailing,
// invoke the `Wait` or `Err` method after finishing reading from the
// `Lines` channel.
func File(filename string, config Config) (*Tail, error) {
	if config.ReOpen && !config.Follow {
		util.Fatal("cannot set ReOpen without Follow.")
	}

	t := &Tail{
		Filename: filename,
		Lines:    make(chan *Line),
		Config:   config,
	}

	// when Logger was not specified in config, use default logger
	if t.Logger == nil {
		t.Logger = log.New(os.Stderr, "", log.LstdFlags)
	}

	if t.Poll {
		t.watcher = watch.NewPollingFileWatcher(filename)
	} else {
		t.watcher = watch.NewInotifyFileWatcher(filename)
	}

	if t.MustExist {
		var err error
		t.file, err = OpenFile(t.Filename)
		if err != nil {
			return nil, err
		}
	}

	go t.tailFileSync()

	return t, nil
}

// Tell returns the file's current position, like stdio's ftell().
// But this value is not very accurate.
// it may readed one line in the chan(tail.Lines),
// so it may lost one line.
func (tail *Tail) Tell() (offset int64, err error) {
	if tail.file == nil {
		return
	}
	offset, err = tail.file.Seek(0, os.SEEK_CUR)
	if err == nil {
		tail.lk.Lock()
		offset -= int64(tail.reader.Buffered())
		tail.lk.Unlock()
	}
	return
}

// Stop stops the tailing activity.
func (tail *Tail) Stop() error {
	tail.Kill(nil)
	return tail.Wait()
}

func (tail *Tail) close() {
	close(tail.Lines)
	tail.colseFile()
}

func (tail *Tail) colseFile() {
	if tail.file != nil {
		tail.file.Close()
		tail.file = nil
	}
}

func (tail *Tail) reopen() error {
	tail.colseFile()
	for {
		var err error
		tail.file, err = OpenFile(tail.Filename)
		if err != nil {
			if os.IsNotExist(err) {
				tail.Logger.Printf("Waiting for %s to appear...", tail.Filename)
				if err := tail.watcher.BlockUntilExists(&tail.Tomb); err != nil {
					if err == tomb.ErrDying {
						return err
					}
					return fmt.Errorf("Failed to detect creation of %s: %s", tail.Filename, err)
				}
				continue
			}
			return fmt.Errorf("Unable to open file %s: %s", tail.Filename, err)
		}
		break
	}
	return nil
}

func (tail *Tail) readLine() (string, error) {
	tail.lk.Lock()
	line, err := tail.reader.ReadString('\n')
	tail.lk.Unlock()
	if err != nil {
		// Note ReadString "returns the data read before the error" in
		// case of an error, including EOF, so we return it as is. The
		// caller is expected to process it if err is EOF.
		return line, err
	}

	line = strings.TrimRight(line, "\n")

	return line, err
}

func (tail *Tail) tailFileSync() {
	defer tail.Done()
	defer tail.close()

	if !tail.MustExist {
		// deferred first open.
		err := tail.reopen()
		if err != nil {
			if err != tomb.ErrDying {
				tail.Kill(err)
			}
			return
		}
	}

	// Seek to requested location on first open of the file.
	if tail.Location != nil {
		_, err := tail.file.Seek(tail.Location.Offset, tail.Location.Whence)
		tail.Logger.Printf("Seeked %s - %+v\n", tail.Filename, tail.Location)
		if err != nil {
			tail.Killf("Seek error on %s: %s", tail.Filename, err)
			return
		}
	}

	tail.openReader()

	var offset int64
	var err error
	// Read line by line.
	for {
		// do not seek in named pipes
		if !tail.Pipe {
			// grab the position in case we need to back up in the event of a half-line
			offset, err = tail.Tell()
			if err != nil {
				tail.Kill(err)
				return
			}
		}

		line, err := tail.readLine()

		// Process `line` even if err is EOF.
		if err == nil {
			cooloff := !tail.sendLine(line)
			if cooloff {
				// Wait a second before seeking till the end of
				// file when rate limit is reached.
				msg := fmt.Sprintf(
					"Too much log activity; waiting a second " +
						"before resuming tailing")
				tail.Lines <- &Line{msg, time.Now(), fmt.Errorf(msg)}
				select {
				case <-time.After(time.Second):
				case <-tail.Dying():
					return
				}
				err = tail.seekEnd()
				if err != nil {
					tail.Kill(err)
					return
				}
			}
		} else if err == io.EOF {
			if !tail.Follow {
				if line != "" {
					tail.sendLine(line)
				}
				return
			}

			if tail.Follow && line != "" {
				// this has the potential to never return the last line if
				// it's not followed by a newline; seems a fair trade here
				err := tail.seekTo(SeekInfo{Offset: offset, Whence: 0})
				if err != nil {
					tail.Kill(err)
					return
				}
			}

			// When EOF is reached, wait for more data to become
			// available. Wait strategy is based on the `tail.watcher`
			// implementation (inotify or polling).
			err := tail.waitForChanges()
			if err != nil {
				if err != ErrStop {
					tail.Kill(err)
				}
				return
			}
		} else {
			// non-EOF error
			tail.Killf("Error reading %s: %s", tail.Filename, err)
			return
		}

		select {
		case <-tail.Dying():
			return
		default:
		}
	}
}

// waitForChanges waits until the file has been appended, deleted,
// moved or truncated. When moved or deleted - the file will be
// reopened if ReOpen is true. Truncated files are always reopened.
func (tail *Tail) waitForChanges() error {
	if tail.changes == nil {
		pos, err := tail.file.Seek(0, os.SEEK_CUR)
		if err != nil {
			return err
		}
		tail.changes, err = tail.watcher.ChangeEvents(&tail.Tomb, pos)
		if err != nil {
			return err
		}
	}

	select {
	case <-tail.changes.Modified:
		return nil
	case <-tail.changes.Deleted:
		tail.changes = nil
		if tail.ReOpen {
			// XXX: we must not log from a library.
			tail.Logger.Printf("Re-opening moved/deleted file %s ...", tail.Filename)
			if err := tail.reopen(); err != nil {
				return err
			}
			tail.Logger.Printf("Successfully reopened %s", tail.Filename)
			tail.openReader()
			return nil
		}

		tail.Logger.Printf("Stopping tail as file no longer exists: %s", tail.Filename)
		return ErrStop
	case <-tail.changes.Truncated:
		// Always reopen truncated files (Follow is true)
		tail.Logger.Printf("Re-opening truncated file %s ...", tail.Filename)
		if err := tail.reopen(); err != nil {
			return err
		}
		tail.Logger.Printf("Successfully reopened truncated %s", tail.Filename)
		tail.openReader()
		return nil
	case <-tail.Dying():
		return ErrStop
	}
}

func (tail *Tail) openReader() {
	if tail.MaxLineSize > 0 {
		// add 2 to account for newline characters
		tail.reader = bufio.NewReaderSize(tail.file, tail.MaxLineSize+2)
	} else {
		tail.reader = bufio.NewReader(tail.file)
	}
}

func (tail *Tail) seekEnd() error {
	return tail.seekTo(SeekInfo{Offset: 0, Whence: os.SEEK_END})
}

func (tail *Tail) seekTo(pos SeekInfo) error {
	_, err := tail.file.Seek(pos.Offset, pos.Whence)
	if err != nil {
		return fmt.Errorf("Seek error on %s: %s", tail.Filename, err)
	}
	// Reset the read buffer whenever the file is re-seek'ed
	tail.reader.Reset(tail.file)
	return nil
}

// sendLine sends the line(s) to Lines channel, splitting longer lines
// if necessary. Return false if rate limit is reached.
func (tail *Tail) sendLine(line string) bool {
	now := time.Now()
	lines := []string{line}

	// Split longer lines
	if tail.MaxLineSize > 0 && len(line) > tail.MaxLineSize {
		lines = util.PartitionString(line, tail.MaxLineSize)
	}

	for _, line := range lines {
		tail.Lines <- &Line{line, now, nil}
	}

	if tail.Config.RateLimiter != nil {
		ok := tail.Config.RateLimiter.Pour(uint16(len(lines)))
		if !ok {
			tail.Logger.Printf("Leaky bucket full (%v); entering 1s cooloff period.\n",
				tail.Filename)
			return false
		}
	}

	return true
}

// Cleanup removes inotify watches added by the tail package. This function is
// meant to be invoked from a process's exit handler. Linux kernel may not
// automatically remove inotify watches after the process exits.
func (tail *Tail) Cleanup() {
	watch.Cleanup(tail.Filename)
}
<commit_msg>Fixed an apparent typo.<commit_after>// Copyright (c) 2013 ActiveState Software Inc. All rights reserved.

package tail

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/hpcloud/tail/ratelimiter"
	"github.com/hpcloud/tail/util"
	"github.com/hpcloud/tail/watch"
	"gopkg.in/tomb.v1"
)

var (
	ErrStop = fmt.Errorf("tail should now stop")
)

// Line contains the line read from the file, along with a timestamp and an error if necessary.
type Line struct {
	Text string
	Time time.Time
	Err  error // Error from tail
}

// NewLine returns a Line with present time.
func NewLine(text string) *Line {
	return &Line{text, time.Now(), nil}
}

// SeekInfo represents arguments to `os.Seek`
type SeekInfo struct {
	Offset int64
	Whence int // os.SEEK_*
}

type logger interface {
	Fatal(v ...interface{})
	Fatalf(format string, v ...interface{})
	Fatalln(v ...interface{})
	Panic(v ...interface{})
	Panicf(format string, v ...interface{})
	Panicln(v ...interface{})
	Print(v ...interface{})
	Printf(format string, v ...interface{})
	Println(v ...interface{})
}

// Config is used to specify how a file must be tailed.
type Config struct {
	// File-specifc
	Location    *SeekInfo // Seek to this location before tailing
	ReOpen      bool      // Reopen recreated files (tail -F)
	MustExist   bool      // Fail early if the file does not exist
	Poll        bool      // Poll for file changes instead of using inotify
	Pipe        bool      // Is a named pipe (mkfifo)
	RateLimiter *ratelimiter.LeakyBucket

	// Generic IO
	Follow      bool // Continue looking for new lines (tail -f)
	MaxLineSize int  // If non-zero, split longer lines into multiple lines

	// Logger, when nil, is set to tail.DefaultLogger
	// To disable logging: set field to tail.DiscardingLogger
	Logger logger
}

// Tail contains info. related to the item being tailed.
type Tail struct {
	Filename string
	Lines    chan *Line
	Config

	file   *os.File
	reader *bufio.Reader

	watcher watch.FileWatcher
	changes *watch.FileChanges

	tomb.Tomb // provides: Done, Kill, Dying

	lk sync.Mutex
}

var (
	// DefaultLogger is used when Config.Logger == nil
	DefaultLogger = log.New(os.Stderr, "", log.LstdFlags)
	// DiscardingLogger can be used to disable logging output
	DiscardingLogger = log.New(ioutil.Discard, "", 0)
)

// File begins tailing the file. Output stream is made available
// via the `Tail.Lines` channel. To handle errors during tailing,
// invoke the `Wait` or `Err` method after finishing reading from the
// `Lines` channel.
func File(filename string, config Config) (*Tail, error) {
	if config.ReOpen && !config.Follow {
		util.Fatal("cannot set ReOpen without Follow.")
	}

	t := &Tail{
		Filename: filename,
		Lines:    make(chan *Line),
		Config:   config,
	}

	// when Logger was not specified in config, use default logger
	if t.Logger == nil {
		t.Logger = log.New(os.Stderr, "", log.LstdFlags)
	}

	if t.Poll {
		t.watcher = watch.NewPollingFileWatcher(filename)
	} else {
		t.watcher = watch.NewInotifyFileWatcher(filename)
	}

	if t.MustExist {
		var err error
		t.file, err = OpenFile(t.Filename)
		if err != nil {
			return nil, err
		}
	}

	go t.tailFileSync()

	return t, nil
}

// Tell returns the file's current position, like stdio's ftell().
// But this value is not very accurate.
// it may readed one line in the chan(tail.Lines),
// so it may lost one line.
func (tail *Tail) Tell() (offset int64, err error) {
	if tail.file == nil {
		return
	}
	offset, err = tail.file.Seek(0, os.SEEK_CUR)
	if err == nil {
		tail.lk.Lock()
		offset -= int64(tail.reader.Buffered())
		tail.lk.Unlock()
	}
	return
}

// Stop stops the tailing activity.
func (tail *Tail) Stop() error {
	tail.Kill(nil)
	return tail.Wait()
}

func (tail *Tail) close() {
	close(tail.Lines)
	tail.closeFile()
}

func (tail *Tail) closeFile() {
	if tail.file != nil {
		tail.file.Close()
		tail.file = nil
	}
}

func (tail *Tail) reopen() error {
	tail.closeFile()
	for {
		var err error
		tail.file, err = OpenFile(tail.Filename)
		if err != nil {
			if os.IsNotExist(err) {
				tail.Logger.Printf("Waiting for %s to appear...", tail.Filename)
				if err := tail.watcher.BlockUntilExists(&tail.Tomb); err != nil {
					if err == tomb.ErrDying {
						return err
					}
					return fmt.Errorf("Failed to detect creation of %s: %s", tail.Filename, err)
				}
				continue
			}
			return fmt.Errorf("Unable to open file %s: %s", tail.Filename, err)
		}
		break
	}
	return nil
}

func (tail *Tail) readLine() (string, error) {
	tail.lk.Lock()
	line, err := tail.reader.ReadString('\n')
	tail.lk.Unlock()
	if err != nil {
		// Note ReadString "returns the data read before the error" in
		// case of an error, including EOF, so we return it as is. The
		// caller is expected to process it if err is EOF.
		return line, err
	}

	line = strings.TrimRight(line, "\n")

	return line, err
}

func (tail *Tail) tailFileSync() {
	defer tail.Done()
	defer tail.close()

	if !tail.MustExist {
		// deferred first open.
		err := tail.reopen()
		if err != nil {
			if err != tomb.ErrDying {
				tail.Kill(err)
			}
			return
		}
	}

	// Seek to requested location on first open of the file.
	if tail.Location != nil {
		_, err := tail.file.Seek(tail.Location.Offset, tail.Location.Whence)
		tail.Logger.Printf("Seeked %s - %+v\n", tail.Filename, tail.Location)
		if err != nil {
			tail.Killf("Seek error on %s: %s", tail.Filename, err)
			return
		}
	}

	tail.openReader()

	var offset int64
	var err error
	// Read line by line.
	for {
		// do not seek in named pipes
		if !tail.Pipe {
			// grab the position in case we need to back up in the event of a half-line
			offset, err = tail.Tell()
			if err != nil {
				tail.Kill(err)
				return
			}
		}

		line, err := tail.readLine()

		// Process `line` even if err is EOF.
		if err == nil {
			cooloff := !tail.sendLine(line)
			if cooloff {
				// Wait a second before seeking till the end of
				// file when rate limit is reached.
				msg := fmt.Sprintf(
					"Too much log activity; waiting a second " +
						"before resuming tailing")
				tail.Lines <- &Line{msg, time.Now(), fmt.Errorf(msg)}
				select {
				case <-time.After(time.Second):
				case <-tail.Dying():
					return
				}
				err = tail.seekEnd()
				if err != nil {
					tail.Kill(err)
					return
				}
			}
		} else if err == io.EOF {
			if !tail.Follow {
				if line != "" {
					tail.sendLine(line)
				}
				return
			}

			if tail.Follow && line != "" {
				// this has the potential to never return the last line if
				// it's not followed by a newline; seems a fair trade here
				err := tail.seekTo(SeekInfo{Offset: offset, Whence: 0})
				if err != nil {
					tail.Kill(err)
					return
				}
			}

			// When EOF is reached, wait for more data to become
			// available. Wait strategy is based on the `tail.watcher`
			// implementation (inotify or polling).
			err := tail.waitForChanges()
			if err != nil {
				if err != ErrStop {
					tail.Kill(err)
				}
				return
			}
		} else {
			// non-EOF error
			tail.Killf("Error reading %s: %s", tail.Filename, err)
			return
		}

		select {
		case <-tail.Dying():
			return
		default:
		}
	}
}

// waitForChanges waits until the file has been appended, deleted,
// moved or truncated. When moved or deleted - the file will be
// reopened if ReOpen is true. Truncated files are always reopened.
func (tail *Tail) waitForChanges() error {
	if tail.changes == nil {
		pos, err := tail.file.Seek(0, os.SEEK_CUR)
		if err != nil {
			return err
		}
		tail.changes, err = tail.watcher.ChangeEvents(&tail.Tomb, pos)
		if err != nil {
			return err
		}
	}

	select {
	case <-tail.changes.Modified:
		return nil
	case <-tail.changes.Deleted:
		tail.changes = nil
		if tail.ReOpen {
			// XXX: we must not log from a library.
			tail.Logger.Printf("Re-opening moved/deleted file %s ...", tail.Filename)
			if err := tail.reopen(); err != nil {
				return err
			}
			tail.Logger.Printf("Successfully reopened %s", tail.Filename)
			tail.openReader()
			return nil
		}

		tail.Logger.Printf("Stopping tail as file no longer exists: %s", tail.Filename)
		return ErrStop
	case <-tail.changes.Truncated:
		// Always reopen truncated files (Follow is true)
		tail.Logger.Printf("Re-opening truncated file %s ...", tail.Filename)
		if err := tail.reopen(); err != nil {
			return err
		}
		tail.Logger.Printf("Successfully reopened truncated %s", tail.Filename)
		tail.openReader()
		return nil
	case <-tail.Dying():
		return ErrStop
	}
}

func (tail *Tail) openReader() {
	if tail.MaxLineSize > 0 {
		// add 2 to account for newline characters
		tail.reader = bufio.NewReaderSize(tail.file, tail.MaxLineSize+2)
	} else {
		tail.reader = bufio.NewReader(tail.file)
	}
}

func (tail *Tail) seekEnd() error {
	return tail.seekTo(SeekInfo{Offset: 0, Whence: os.SEEK_END})
}

func (tail *Tail) seekTo(pos SeekInfo) error {
	_, err := tail.file.Seek(pos.Offset, pos.Whence)
	if err != nil {
		return fmt.Errorf("Seek error on %s: %s", tail.Filename, err)
	}
	// Reset the read buffer whenever the file is re-seek'ed
	tail.reader.Reset(tail.file)
	return nil
}

// sendLine sends the line(s) to Lines channel, splitting longer lines
// if necessary. Return false if rate limit is reached.
func (tail *Tail) sendLine(line string) bool {
	now := time.Now()
	lines := []string{line}

	// Split longer lines
	if tail.MaxLineSize > 0 && len(line) > tail.MaxLineSize {
		lines = util.PartitionString(line, tail.MaxLineSize)
	}

	for _, line := range lines {
		tail.Lines <- &Line{line, now, nil}
	}

	if tail.Config.RateLimiter != nil {
		ok := tail.Config.RateLimiter.Pour(uint16(len(lines)))
		if !ok {
			tail.Logger.Printf("Leaky bucket full (%v); entering 1s cooloff period.\n",
				tail.Filename)
			return false
		}
	}

	return true
}

// Cleanup removes inotify watches added by the tail package. This function is
// meant to be invoked from a process's exit handler. Linux kernel may not
// automatically remove inotify watches after the process exits.
func (tail *Tail) Cleanup() {
	watch.Cleanup(tail.Filename)
}
<|endoftext|>