<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bytes

// Simple byte buffer for marshaling data.

import (
	"io"
	"os"
	"utf8"
)

// Copy from string to byte array at offset doff.  Assume there's room.
func copyString(dst []byte, doff int, str string) {
	for soff := 0; soff < len(str); soff++ {
		dst[doff] = str[soff]
		doff++
	}
}

// A Buffer is a variable-sized buffer of bytes with Read and Write methods.
// The zero value for Buffer is an empty buffer ready to use.
type Buffer struct {
	buf       []byte            // contents are the bytes buf[off : len(buf)]
	off       int               // read at &buf[off], write at &buf[len(buf)]
	runeBytes [utf8.UTFMax]byte // avoid allocation of slice on each WriteByte or Rune
	bootstrap [64]byte          // memory to hold first slice; helps small buffers (Printf) avoid allocation.
}

// Bytes returns a slice of the contents of the unread portion of the buffer;
// len(b.Bytes()) == b.Len().  If the caller changes the contents of the
// returned slice, the contents of the buffer will change provided there
// are no intervening method calls on the Buffer.
func (b *Buffer) Bytes() []byte { return b.buf[b.off:] }

// String returns the contents of the unread portion of the buffer
// as a string.  If the Buffer is a nil pointer, it returns "<nil>".
func (b *Buffer) String() string {
	if b == nil {
		// Special case, useful in debugging.
		return "<nil>"
	}
	return string(b.buf[b.off:])
}

// Len returns the number of bytes of the unread portion of the buffer;
// b.Len() == len(b.Bytes()).
func (b *Buffer) Len() int { return len(b.buf) - b.off }

// Truncate discards all but the first n unread bytes from the buffer.
// It is an error to call b.Truncate(n) with n > b.Len().
func (b *Buffer) Truncate(n int) {
	if n == 0 {
		// Reuse buffer space.
		b.off = 0
	}
	b.buf = b.buf[0 : b.off+n]
}

// Reset resets the buffer so it has no content.
// b.Reset() is the same as b.Truncate(0).
func (b *Buffer) Reset() { b.Truncate(0) }

// Grow buffer to guarantee space for n more bytes.
// Return index where bytes should be written.
func (b *Buffer) grow(n int) int {
	m := b.Len()
	// If buffer is empty, reset to recover space.
	if m == 0 && b.off != 0 {
		b.Truncate(0)
	}
	if len(b.buf)+n > cap(b.buf) {
		var buf []byte
		if b.buf == nil && n <= len(b.bootstrap) {
			buf = b.bootstrap[0:]
		} else {
			// not enough space anywhere
			buf = make([]byte, 2*cap(b.buf)+n)
			copy(buf, b.buf[b.off:])
		}
		b.buf = buf
		b.off = 0
	}
	b.buf = b.buf[0 : b.off+m+n]
	return b.off + m
}

// Write appends the contents of p to the buffer.  The return
// value n is the length of p; err is always nil.
func (b *Buffer) Write(p []byte) (n int, err os.Error) {
	m := b.grow(len(p))
	copy(b.buf[m:], p)
	return len(p), nil
}

// WriteString appends the contents of s to the buffer.  The return
// value n is the length of s; err is always nil.
func (b *Buffer) WriteString(s string) (n int, err os.Error) {
	m := b.grow(len(s))
	copyString(b.buf, m, s)
	return len(s), nil
}

// MinRead is the minimum slice size passed to a Read call by
// Buffer.ReadFrom.  As long as the Buffer has at least MinRead bytes beyond
// what is required to hold the contents of r, ReadFrom will not grow the
// underlying buffer.
const MinRead = 512

// ReadFrom reads data from r until EOF and appends it to the buffer.
// The return value n is the number of bytes read.
// Any error except os.EOF encountered during the read
// is also returned.
func (b *Buffer) ReadFrom(r io.Reader) (n int64, err os.Error) {
	// If buffer is empty, reset to recover space.
	if b.off >= len(b.buf) {
		b.Truncate(0)
	}
	for {
		if cap(b.buf)-len(b.buf) < MinRead {
			var newBuf []byte
			// can we get space without allocation?
			if b.off+cap(b.buf)-len(b.buf) >= MinRead {
				// reuse beginning of buffer
				newBuf = b.buf[0 : len(b.buf)-b.off]
			} else {
				// not enough space at end; put space on end
				newBuf = make([]byte, len(b.buf)-b.off, 2*(cap(b.buf)-b.off)+MinRead)
			}
			copy(newBuf, b.buf[b.off:])
			b.buf = newBuf
			b.off = 0
		}
		m, e := r.Read(b.buf[len(b.buf):cap(b.buf)])
		b.buf = b.buf[b.off : len(b.buf)+m]
		n += int64(m)
		if e == os.EOF {
			break
		}
		if e != nil {
			return n, e
		}
	}
	return n, nil // err is EOF, so return nil explicitly
}

// WriteTo writes data to w until the buffer is drained or an error
// occurs. The return value n is the number of bytes written.
// Any error encountered during the write is also returned.
func (b *Buffer) WriteTo(w io.Writer) (n int64, err os.Error) {
	for b.off < len(b.buf) {
		m, e := w.Write(b.buf[b.off:])
		n += int64(m)
		b.off += m
		if e != nil {
			return n, e
		}
	}
	// Buffer is now empty; reset.
	b.Truncate(0)
	return
}

// WriteByte appends the byte c to the buffer.
// The returned error is always nil, but is included
// to match bufio.Writer's WriteByte.
func (b *Buffer) WriteByte(c byte) os.Error {
	m := b.grow(1)
	b.buf[m] = c
	return nil
}

// WriteRune appends the UTF-8 encoding of Unicode
// code point r to the buffer, returning its length and
// an error, which is always nil but is included
// to match bufio.Writer's WriteRune.
func (b *Buffer) WriteRune(r int) (n int, err os.Error) {
	if r < utf8.RuneSelf {
		b.WriteByte(byte(r))
		return 1, nil
	}
	n = utf8.EncodeRune(r, b.runeBytes[0:])
	b.Write(b.runeBytes[0:n])
	return n, nil
}

// Read reads the next len(p) bytes from the buffer or until the buffer
// is drained.  The return value n is the number of bytes read.  If the
// buffer has no data to return, err is os.EOF even if len(p) is zero;
// otherwise it is nil.
func (b *Buffer) Read(p []byte) (n int, err os.Error) {
	if b.off >= len(b.buf) {
		// Buffer is empty, reset to recover space.
		b.Truncate(0)
		return 0, os.EOF
	}
	n = copy(p, b.buf[b.off:])
	b.off += n
	return
}

// Next returns a slice containing the next n bytes from the buffer,
// advancing the buffer as if the bytes had been returned by Read.
// If there are fewer than n bytes in the buffer, Next returns the entire buffer.
// The slice is only valid until the next call to a read or write method.
func (b *Buffer) Next(n int) []byte {
	m := b.Len()
	if n > m {
		n = m
	}
	data := b.buf[b.off : b.off+n]
	b.off += n
	return data
}

// ReadByte reads and returns the next byte from the buffer.
// If no byte is available, it returns error os.EOF.
func (b *Buffer) ReadByte() (c byte, err os.Error) {
	if b.off >= len(b.buf) {
		// Buffer is empty, reset to recover space.
		b.Truncate(0)
		return 0, os.EOF
	}
	c = b.buf[b.off]
	b.off++
	return c, nil
}

// ReadRune reads and returns the next UTF-8-encoded
// Unicode code point from the buffer.
// If no bytes are available, the error returned is os.EOF.
// If the bytes are an erroneous UTF-8 encoding, it
// consumes one byte and returns U+FFFD, 1.
func (b *Buffer) ReadRune() (r int, size int, err os.Error) {
	if b.off >= len(b.buf) {
		// Buffer is empty, reset to recover space.
		b.Truncate(0)
		return 0, 0, os.EOF
	}
	c := b.buf[b.off]
	if c < utf8.RuneSelf {
		b.off++
		return int(c), 1, nil
	}
	r, n := utf8.DecodeRune(b.buf[b.off:])
	b.off += n
	return r, n, nil
}

// NewBuffer creates and initializes a new Buffer using buf as its initial
// contents.  It is intended to prepare a Buffer to read existing data.  It
// can also be used to to size the internal buffer for writing.  To do that,
// buf should have the desired capacity but a length of zero.
func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} }

// NewBufferString creates and initializes a new Buffer using string s as its
// initial contents.  It is intended to prepare a buffer to read an existing
// string.
func NewBufferString(s string) *Buffer {
	buf := make([]byte, len(s))
	copyString(buf, 0, s)
	return &Buffer{buf: buf}
}
<commit_msg>bytes.Buffer.ReadFrom: fix bug. Fixes issue 852.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bytes

// Simple byte buffer for marshaling data.

import (
	"io"
	"os"
	"utf8"
)

// Copy from string to byte array at offset doff.  Assume there's room.
func copyString(dst []byte, doff int, str string) {
	for soff := 0; soff < len(str); soff++ {
		dst[doff] = str[soff]
		doff++
	}
}

// A Buffer is a variable-sized buffer of bytes with Read and Write methods.
// The zero value for Buffer is an empty buffer ready to use.
type Buffer struct {
	buf       []byte            // contents are the bytes buf[off : len(buf)]
	off       int               // read at &buf[off], write at &buf[len(buf)]
	runeBytes [utf8.UTFMax]byte // avoid allocation of slice on each WriteByte or Rune
	bootstrap [64]byte          // memory to hold first slice; helps small buffers (Printf) avoid allocation.
}

// Bytes returns a slice of the contents of the unread portion of the buffer;
// len(b.Bytes()) == b.Len().  If the caller changes the contents of the
// returned slice, the contents of the buffer will change provided there
// are no intervening method calls on the Buffer.
func (b *Buffer) Bytes() []byte { return b.buf[b.off:] }

// String returns the contents of the unread portion of the buffer
// as a string.  If the Buffer is a nil pointer, it returns "<nil>".
func (b *Buffer) String() string {
	if b == nil {
		// Special case, useful in debugging.
		return "<nil>"
	}
	return string(b.buf[b.off:])
}

// Len returns the number of bytes of the unread portion of the buffer;
// b.Len() == len(b.Bytes()).
func (b *Buffer) Len() int { return len(b.buf) - b.off }

// Truncate discards all but the first n unread bytes from the buffer.
// It is an error to call b.Truncate(n) with n > b.Len().
func (b *Buffer) Truncate(n int) {
	if n == 0 {
		// Reuse buffer space.
		b.off = 0
	}
	b.buf = b.buf[0 : b.off+n]
}

// Reset resets the buffer so it has no content.
// b.Reset() is the same as b.Truncate(0).
func (b *Buffer) Reset() { b.Truncate(0) }

// Grow buffer to guarantee space for n more bytes.
// Return index where bytes should be written.
func (b *Buffer) grow(n int) int {
	m := b.Len()
	// If buffer is empty, reset to recover space.
	if m == 0 && b.off != 0 {
		b.Truncate(0)
	}
	if len(b.buf)+n > cap(b.buf) {
		var buf []byte
		if b.buf == nil && n <= len(b.bootstrap) {
			buf = b.bootstrap[0:]
		} else {
			// not enough space anywhere
			buf = make([]byte, 2*cap(b.buf)+n)
			copy(buf, b.buf[b.off:])
		}
		b.buf = buf
		b.off = 0
	}
	b.buf = b.buf[0 : b.off+m+n]
	return b.off + m
}

// Write appends the contents of p to the buffer.  The return
// value n is the length of p; err is always nil.
func (b *Buffer) Write(p []byte) (n int, err os.Error) {
	m := b.grow(len(p))
	copy(b.buf[m:], p)
	return len(p), nil
}

// WriteString appends the contents of s to the buffer.  The return
// value n is the length of s; err is always nil.
func (b *Buffer) WriteString(s string) (n int, err os.Error) {
	m := b.grow(len(s))
	copyString(b.buf, m, s)
	return len(s), nil
}

// MinRead is the minimum slice size passed to a Read call by
// Buffer.ReadFrom.  As long as the Buffer has at least MinRead bytes beyond
// what is required to hold the contents of r, ReadFrom will not grow the
// underlying buffer.
const MinRead = 512

// ReadFrom reads data from r until EOF and appends it to the buffer.
// The return value n is the number of bytes read.
// Any error except os.EOF encountered during the read
// is also returned.
func (b *Buffer) ReadFrom(r io.Reader) (n int64, err os.Error) {
	// If buffer is empty, reset to recover space.
	if b.off >= len(b.buf) {
		b.Truncate(0)
	}
	for {
		if cap(b.buf)-len(b.buf) < MinRead {
			var newBuf []byte
			// can we get space without allocation?
			if b.off+cap(b.buf)-len(b.buf) >= MinRead {
				// reuse beginning of buffer
				newBuf = b.buf[0 : len(b.buf)-b.off]
			} else {
				// not enough space at end; put space on end
				newBuf = make([]byte, len(b.buf)-b.off, 2*(cap(b.buf)-b.off)+MinRead)
			}
			copy(newBuf, b.buf[b.off:])
			b.buf = newBuf
			b.off = 0
		}
		m, e := r.Read(b.buf[len(b.buf):cap(b.buf)])
		b.buf = b.buf[0 : len(b.buf)+m]
		n += int64(m)
		if e == os.EOF {
			break
		}
		if e != nil {
			return n, e
		}
	}
	return n, nil // err is EOF, so return nil explicitly
}

// WriteTo writes data to w until the buffer is drained or an error
// occurs. The return value n is the number of bytes written.
// Any error encountered during the write is also returned.
func (b *Buffer) WriteTo(w io.Writer) (n int64, err os.Error) {
	for b.off < len(b.buf) {
		m, e := w.Write(b.buf[b.off:])
		n += int64(m)
		b.off += m
		if e != nil {
			return n, e
		}
	}
	// Buffer is now empty; reset.
	b.Truncate(0)
	return
}

// WriteByte appends the byte c to the buffer.
// The returned error is always nil, but is included
// to match bufio.Writer's WriteByte.
func (b *Buffer) WriteByte(c byte) os.Error {
	m := b.grow(1)
	b.buf[m] = c
	return nil
}

// WriteRune appends the UTF-8 encoding of Unicode
// code point r to the buffer, returning its length and
// an error, which is always nil but is included
// to match bufio.Writer's WriteRune.
func (b *Buffer) WriteRune(r int) (n int, err os.Error) {
	if r < utf8.RuneSelf {
		b.WriteByte(byte(r))
		return 1, nil
	}
	n = utf8.EncodeRune(r, b.runeBytes[0:])
	b.Write(b.runeBytes[0:n])
	return n, nil
}

// Read reads the next len(p) bytes from the buffer or until the buffer
// is drained.  The return value n is the number of bytes read.  If the
// buffer has no data to return, err is os.EOF even if len(p) is zero;
// otherwise it is nil.
func (b *Buffer) Read(p []byte) (n int, err os.Error) {
	if b.off >= len(b.buf) {
		// Buffer is empty, reset to recover space.
		b.Truncate(0)
		return 0, os.EOF
	}
	n = copy(p, b.buf[b.off:])
	b.off += n
	return
}

// Next returns a slice containing the next n bytes from the buffer,
// advancing the buffer as if the bytes had been returned by Read.
// If there are fewer than n bytes in the buffer, Next returns the entire buffer.
// The slice is only valid until the next call to a read or write method.
func (b *Buffer) Next(n int) []byte {
	m := b.Len()
	if n > m {
		n = m
	}
	data := b.buf[b.off : b.off+n]
	b.off += n
	return data
}

// ReadByte reads and returns the next byte from the buffer.
// If no byte is available, it returns error os.EOF.
func (b *Buffer) ReadByte() (c byte, err os.Error) {
	if b.off >= len(b.buf) {
		// Buffer is empty, reset to recover space.
		b.Truncate(0)
		return 0, os.EOF
	}
	c = b.buf[b.off]
	b.off++
	return c, nil
}

// ReadRune reads and returns the next UTF-8-encoded
// Unicode code point from the buffer.
// If no bytes are available, the error returned is os.EOF.
// If the bytes are an erroneous UTF-8 encoding, it
// consumes one byte and returns U+FFFD, 1.
func (b *Buffer) ReadRune() (r int, size int, err os.Error) {
	if b.off >= len(b.buf) {
		// Buffer is empty, reset to recover space.
		b.Truncate(0)
		return 0, 0, os.EOF
	}
	c := b.buf[b.off]
	if c < utf8.RuneSelf {
		b.off++
		return int(c), 1, nil
	}
	r, n := utf8.DecodeRune(b.buf[b.off:])
	b.off += n
	return r, n, nil
}

// NewBuffer creates and initializes a new Buffer using buf as its initial
// contents.  It is intended to prepare a Buffer to read existing data.  It
// can also be used to to size the internal buffer for writing.  To do that,
// buf should have the desired capacity but a length of zero.
func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} }

// NewBufferString creates and initializes a new Buffer using string s as its
// initial contents.  It is intended to prepare a buffer to read an existing
// string.
func NewBufferString(s string) *Buffer {
	buf := make([]byte, len(s))
	copyString(buf, 0, s)
	return &Buffer{buf: buf}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kuberuntime

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math"
	"os"
	"time"

	"github.com/docker/docker/pkg/jsonlog"
	"github.com/fsnotify/fsnotify"
	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api/v1"
	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
	"k8s.io/kubernetes/pkg/util/tail"
)

// Notice that the current kuberuntime logs implementation doesn't handle
// log rotation.
// * It will not retrieve logs in rotated log file.
// * If log rotation happens when following the log:
//   * If the rotation is using create mode, we'll still follow the old file.
//   * If the rotation is using copytruncate, we'll be reading at the original position and get nothing.
// TODO(random-liu): Support log rotation.

// streamType is the type of the stream.
type streamType string

const (
	stderrType streamType = "stderr"
	stdoutType streamType = "stdout"

	// timeFormat is the time format used in the log.
	timeFormat = time.RFC3339Nano
	// blockSize is the block size used in tail.
	blockSize = 1024

	// stateCheckPeriod is the period to check container state while following
	// the container log. Kubelet should not keep following the log when the
	// container is not running.
	stateCheckPeriod = 5 * time.Second
)

var (
	// eol is the end-of-line sign in the log.
	eol = []byte{'\n'}
	// delimiter is the delimiter for timestamp and streamtype in log line.
	delimiter = []byte{' '}
)

// logMessage is the internal log type.
type logMessage struct {
	timestamp time.Time
	stream    streamType
	log       []byte
}

// reset resets the log to nil.
func (l *logMessage) reset() {
	l.timestamp = time.Time{}
	l.stream = ""
	l.log = nil
}

// logOptions is the internal type of all log options.
type logOptions struct {
	tail      int64
	bytes     int64
	since     time.Time
	follow    bool
	timestamp bool
}

// newLogOptions convert the v1.PodLogOptions to internal logOptions.
func newLogOptions(apiOpts *v1.PodLogOptions, now time.Time) *logOptions {
	opts := &logOptions{
		tail:      -1, // -1 by default which means read all logs.
		bytes:     -1, // -1 by default which means read all logs.
		follow:    apiOpts.Follow,
		timestamp: apiOpts.Timestamps,
	}
	if apiOpts.TailLines != nil {
		opts.tail = *apiOpts.TailLines
	}
	if apiOpts.LimitBytes != nil {
		opts.bytes = *apiOpts.LimitBytes
	}
	if apiOpts.SinceSeconds != nil {
		opts.since = now.Add(-time.Duration(*apiOpts.SinceSeconds) * time.Second)
	}
	if apiOpts.SinceTime != nil && apiOpts.SinceTime.After(opts.since) {
		opts.since = apiOpts.SinceTime.Time
	}
	return opts
}

// ReadLogs read the container log and redirect into stdout and stderr.
// Note that containerID is only needed when following the log, or else
// just pass in empty string "".
func (m *kubeGenericRuntimeManager) ReadLogs(path, containerID string, apiOpts *v1.PodLogOptions, stdout, stderr io.Writer) error {
	f, err := os.Open(path)
	if err != nil {
		return fmt.Errorf("failed to open log file %q: %v", path, err)
	}
	defer f.Close()

	// Convert v1.PodLogOptions into internal log options.
	opts := newLogOptions(apiOpts, time.Now())

	// Search start point based on tail line.
	start, err := tail.FindTailLineStartIndex(f, opts.tail)
	if err != nil {
		return fmt.Errorf("failed to tail %d lines of log file %q: %v", opts.tail, path, err)
	}
	if _, err := f.Seek(start, os.SEEK_SET); err != nil {
		return fmt.Errorf("failed to seek %d in log file %q: %v", start, path, err)
	}

	// Start parsing the logs.
	r := bufio.NewReader(f)
	// Do not create watcher here because it is not needed if `Follow` is false.
	var watcher *fsnotify.Watcher
	var parse parseFunc
	writer := newLogWriter(stdout, stderr, opts)
	msg := &logMessage{}
	for {
		l, err := r.ReadBytes(eol[0])
		if err != nil {
			if err != io.EOF { // This is an real error
				return fmt.Errorf("failed to read log file %q: %v", path, err)
			}
			if !opts.follow {
				// Return directly when reading to the end if not follow.
				if len(l) > 0 {
					glog.Warningf("Incomplete line in log file %q: %q", path, l)
				}
				glog.V(2).Infof("Finish parsing log file %q", path)
				return nil
			}
			// Reset seek so that if this is an incomplete line,
			// it will be read again.
			if _, err := f.Seek(-int64(len(l)), os.SEEK_CUR); err != nil {
				return fmt.Errorf("failed to reset seek in log file %q: %v", path, err)
			}
			if watcher == nil {
				// Intialize the watcher if it has not been initialized yet.
				if watcher, err = fsnotify.NewWatcher(); err != nil {
					return fmt.Errorf("failed to create fsnotify watcher: %v", err)
				}
				defer watcher.Close()
				if err := watcher.Add(f.Name()); err != nil {
					return fmt.Errorf("failed to watch file %q: %v", f.Name(), err)
				}
			}
			// Wait until the next log change.
			if found, err := m.waitLogs(containerID, watcher); !found {
				return err
			}
			continue
		}
		if parse == nil {
			// Intialize the log parsing function.
			parse, err = getParseFunc(l)
			if err != nil {
				return fmt.Errorf("failed to get parse function: %v", err)
			}
		}
		// Parse the log line.
		msg.reset()
		if err := parse(l, msg); err != nil {
			glog.Errorf("Failed with err %v when parsing log for log file %q: %q", err, path, l)
			continue
		}
		// Write the log line into the stream.
		if err := writer.write(msg); err != nil {
			if err == errMaximumWrite {
				glog.V(2).Infof("Finish parsing log file %q, hit bytes limit %d(bytes)", path, opts.bytes)
				return nil
			}
			glog.Errorf("Failed with err %v when writing log for log file %q: %+v", err, path, msg)
			return err
		}
	}
}

// waitLogs wait for the next log write. It returns a boolean and an error. The boolean
// indicates whether a new log is found; the error is error happens during waiting new logs.
func (m *kubeGenericRuntimeManager) waitLogs(id string, w *fsnotify.Watcher) (bool, error) {
	errRetry := 5
	for {
		select {
		case e := <-w.Events:
			switch e.Op {
			case fsnotify.Write:
				return true, nil
			default:
				glog.Errorf("Unexpected fsnotify event: %v, retrying...", e)
			}
		case err := <-w.Errors:
			glog.Errorf("Fsnotify watch error: %v, %d error retries remaining", err, errRetry)
			if errRetry == 0 {
				return false, err
			}
			errRetry--
		case <-time.After(stateCheckPeriod):
			s, err := m.runtimeService.ContainerStatus(id)
			if err != nil {
				return false, err
			}
			// Only keep following container log when it is running.
			if s.State != runtimeapi.ContainerState_CONTAINER_RUNNING {
				glog.Errorf("Container %q is not running (state=%q)", id, s.State)
				// Do not return error because it's normal that the container stops
				// during waiting.
				return false, nil
			}
		}
	}
}

// parseFunc is a function parsing one log line to the internal log type.
// Notice that the caller must make sure logMessage is not nil.
type parseFunc func([]byte, *logMessage) error

var parseFuncs []parseFunc = []parseFunc{
	parseCRILog,        // CRI log format parse function
	parseDockerJSONLog, // Docker JSON log format parse function
}

// parseCRILog parses logs in CRI log format. CRI Log format example:
//   2016-10-06T00:17:09.669794202Z stdout log content 1
//   2016-10-06T00:17:09.669794203Z stderr log content 2
func parseCRILog(log []byte, msg *logMessage) error {
	var err error
	// Parse timestamp
	idx := bytes.Index(log, delimiter)
	if idx < 0 {
		return fmt.Errorf("timestamp is not found")
	}
	msg.timestamp, err = time.Parse(timeFormat, string(log[:idx]))
	if err != nil {
		return fmt.Errorf("unexpected timestamp format %q: %v", timeFormat, err)
	}

	// Parse stream type
	log = log[idx+1:]
	idx = bytes.Index(log, delimiter)
	if idx < 0 {
		return fmt.Errorf("stream type is not found")
	}
	msg.stream = streamType(log[:idx])
	if msg.stream != stdoutType && msg.stream != stderrType {
		return fmt.Errorf("unexpected stream type %q", msg.stream)
	}

	// Get log content
	msg.log = log[idx+1:]

	return nil
}

// parseDockerJSONLog parses logs in Docker JSON log format. Docker JSON log format
// example:
//   {"log":"content 1","stream":"stdout","time":"2016-10-20T18:39:20.57606443Z"}
//   {"log":"content 2","stream":"stderr","time":"2016-10-20T18:39:20.57606444Z"}
func parseDockerJSONLog(log []byte, msg *logMessage) error {
	var l = &jsonlog.JSONLog{}
	l.Reset()

	// TODO: JSON decoding is fairly expensive, we should evaluate this.
	if err := json.Unmarshal(log, l); err != nil {
		return fmt.Errorf("failed with %v to unmarshal log %q", err, l)
	}
	msg.timestamp = l.Created
	msg.stream = streamType(l.Stream)
	msg.log = []byte(l.Log)
	return nil
}

// getParseFunc returns proper parse function based on the sample log line passed in.
func getParseFunc(log []byte) (parseFunc, error) {
	for _, p := range parseFuncs {
		if err := p(log, &logMessage{}); err == nil {
			return p, nil
		}
	}
	return nil, fmt.Errorf("unsupported log format: %q", log)
}

// logWriter controls the writing into the stream based on the log options.
type logWriter struct {
	stdout io.Writer
	stderr io.Writer
	opts   *logOptions
	remain int64
}

// errMaximumWrite is returned when all bytes have been written.
var errMaximumWrite = errors.New("maximum write")

// errShortWrite is returned when the message is not fully written.
var errShortWrite = errors.New("short write")

func newLogWriter(stdout io.Writer, stderr io.Writer, opts *logOptions) *logWriter {
	w := &logWriter{
		stdout: stdout,
		stderr: stderr,
		opts:   opts,
		remain: math.MaxInt64, // initialize it as infinity
	}
	if opts.bytes >= 0 {
		w.remain = opts.bytes
	}
	return w
}

// writeLogs writes logs into stdout, stderr.
func (w *logWriter) write(msg *logMessage) error {
	if msg.timestamp.Before(w.opts.since) {
		// Skip the line because it's older than since
		return nil
	}
	line := msg.log
	if w.opts.timestamp {
		prefix := append([]byte(msg.timestamp.Format(timeFormat)), delimiter[0])
		line = append(prefix, line...)
	}
	// If the line is longer than the remaining bytes, cut it.
	if int64(len(line)) > w.remain {
		line = line[:w.remain]
	}
	// Get the proper stream to write to.
	var stream io.Writer
	switch msg.stream {
	case stdoutType:
		stream = w.stdout
	case stderrType:
		stream = w.stderr
	default:
		return fmt.Errorf("unexpected stream type %q", msg.stream)
	}
	n, err := stream.Write(line)
	w.remain -= int64(n)
	if err != nil {
		return err
	}
	// If the line has not been fully written, return errShortWrite
	if n < len(line) {
		return errShortWrite
	}
	// If there are no more bytes left, return errMaximumWrite
	if w.remain <= 0 {
		return errMaximumWrite
	}
	return nil
}
<commit_msg>UPSTREAM: 55631: Parse and return the last line in the log even if it is partial<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kuberuntime

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math"
	"os"
	"time"

	"github.com/docker/docker/pkg/jsonlog"
	"github.com/fsnotify/fsnotify"
	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api/v1"
	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
	"k8s.io/kubernetes/pkg/util/tail"
)

// Notice that the current kuberuntime logs implementation doesn't handle
// log rotation.
// * It will not retrieve logs in rotated log file.
// * If log rotation happens when following the log:
//   * If the rotation is using create mode, we'll still follow the old file.
//   * If the rotation is using copytruncate, we'll be reading at the original position and get nothing.
// TODO(random-liu): Support log rotation.

// streamType is the type of the stream.
type streamType string

const (
	stderrType streamType = "stderr"
	stdoutType streamType = "stdout"

	// timeFormat is the time format used in the log.
	timeFormat = time.RFC3339Nano
	// blockSize is the block size used in tail.
	blockSize = 1024

	// stateCheckPeriod is the period to check container state while following
	// the container log. Kubelet should not keep following the log when the
	// container is not running.
	stateCheckPeriod = 5 * time.Second
)

var (
	// eol is the end-of-line sign in the log.
	eol = []byte{'\n'}
	// delimiter is the delimiter for timestamp and streamtype in log line.
	delimiter = []byte{' '}
)

// logMessage is the internal log type.
type logMessage struct {
	timestamp time.Time
	stream    streamType
	log       []byte
}

// reset resets the log to nil.
func (l *logMessage) reset() {
	l.timestamp = time.Time{}
	l.stream = ""
	l.log = nil
}

// logOptions is the internal type of all log options.
type logOptions struct {
	tail      int64
	bytes     int64
	since     time.Time
	follow    bool
	timestamp bool
}

// newLogOptions convert the v1.PodLogOptions to internal logOptions.
func newLogOptions(apiOpts *v1.PodLogOptions, now time.Time) *logOptions {
	opts := &logOptions{
		tail:      -1, // -1 by default which means read all logs.
		bytes:     -1, // -1 by default which means read all logs.
		follow:    apiOpts.Follow,
		timestamp: apiOpts.Timestamps,
	}
	if apiOpts.TailLines != nil {
		opts.tail = *apiOpts.TailLines
	}
	if apiOpts.LimitBytes != nil {
		opts.bytes = *apiOpts.LimitBytes
	}
	if apiOpts.SinceSeconds != nil {
		opts.since = now.Add(-time.Duration(*apiOpts.SinceSeconds) * time.Second)
	}
	if apiOpts.SinceTime != nil && apiOpts.SinceTime.After(opts.since) {
		opts.since = apiOpts.SinceTime.Time
	}
	return opts
}

// ReadLogs read the container log and redirect into stdout and stderr.
// Note that containerID is only needed when following the log, or else
// just pass in empty string "".
func (m *kubeGenericRuntimeManager) ReadLogs(path, containerID string, apiOpts *v1.PodLogOptions, stdout, stderr io.Writer) error {
	f, err := os.Open(path)
	if err != nil {
		return fmt.Errorf("failed to open log file %q: %v", path, err)
	}
	defer f.Close()

	// Convert v1.PodLogOptions into internal log options.
	opts := newLogOptions(apiOpts, time.Now())

	// Search start point based on tail line.
	start, err := tail.FindTailLineStartIndex(f, opts.tail)
	if err != nil {
		return fmt.Errorf("failed to tail %d lines of log file %q: %v", opts.tail, path, err)
	}
	if _, err := f.Seek(start, os.SEEK_SET); err != nil {
		return fmt.Errorf("failed to seek %d in log file %q: %v", start, path, err)
	}

	// Start parsing the logs.
	r := bufio.NewReader(f)
	// Do not create watcher here because it is not needed if `Follow` is false.
	var watcher *fsnotify.Watcher
	var parse parseFunc
	writer := newLogWriter(stdout, stderr, opts)
	msg := &logMessage{}
	for {
		l, err := r.ReadBytes(eol[0])
		if err != nil {
			if err != io.EOF { // This is an real error
				return fmt.Errorf("failed to read log file %q: %v", path, err)
			}
			if !opts.follow {
				// Return directly when reading to the end if not follow.
				if len(l) > 0 {
					glog.Warningf("Incomplete line in log file %q: %q", path, l)
					if parse == nil {
						// Intialize the log parsing function.
						parse, err = getParseFunc(l)
						if err != nil {
							return fmt.Errorf("failed to get parse function: %v", err)
						}
					}
					// Log a warning and exit if we can't parse the partial line.
					if err := parse(l, msg); err != nil {
						glog.Warningf("Failed with err %v when parsing partial line for log file %q: %q", err, path, l)
						return nil
					}
					// Write the log line into the stream.
					if err := writer.write(msg); err != nil {
						if err == errMaximumWrite {
							glog.V(2).Infof("Finish parsing log file %q, hit bytes limit %d(bytes)", path, opts.bytes)
							return nil
						}
						glog.Errorf("Failed with err %v when writing partial log for log file %q: %+v", err, path, msg)
						return err
					}
				}
				glog.V(2).Infof("Finish parsing log file %q", path)
				return nil
			}
			// Reset seek so that if this is an incomplete line,
			// it will be read again.
			if _, err := f.Seek(-int64(len(l)), os.SEEK_CUR); err != nil {
				return fmt.Errorf("failed to reset seek in log file %q: %v", path, err)
			}
			if watcher == nil {
				// Intialize the watcher if it has not been initialized yet.
				if watcher, err = fsnotify.NewWatcher(); err != nil {
					return fmt.Errorf("failed to create fsnotify watcher: %v", err)
				}
				defer watcher.Close()
				if err := watcher.Add(f.Name()); err != nil {
					return fmt.Errorf("failed to watch file %q: %v", f.Name(), err)
				}
			}
			// Wait until the next log change.
			if found, err := m.waitLogs(containerID, watcher); !found {
				return err
			}
			continue
		}
		if parse == nil {
			// Intialize the log parsing function.
			parse, err = getParseFunc(l)
			if err != nil {
				return fmt.Errorf("failed to get parse function: %v", err)
			}
		}
		// Parse the log line.
		msg.reset()
		if err := parse(l, msg); err != nil {
			glog.Errorf("Failed with err %v when parsing log for log file %q: %q", err, path, l)
			continue
		}
		// Write the log line into the stream.
		if err := writer.write(msg); err != nil {
			if err == errMaximumWrite {
				glog.V(2).Infof("Finish parsing log file %q, hit bytes limit %d(bytes)", path, opts.bytes)
				return nil
			}
			glog.Errorf("Failed with err %v when writing log for log file %q: %+v", err, path, msg)
			return err
		}
	}
}

// waitLogs wait for the next log write. It returns a boolean and an error. The boolean
// indicates whether a new log is found; the error is error happens during waiting new logs.
func (m *kubeGenericRuntimeManager) waitLogs(id string, w *fsnotify.Watcher) (bool, error) {
	errRetry := 5
	for {
		select {
		case e := <-w.Events:
			switch e.Op {
			case fsnotify.Write:
				return true, nil
			default:
				glog.Errorf("Unexpected fsnotify event: %v, retrying...", e)
			}
		case err := <-w.Errors:
			glog.Errorf("Fsnotify watch error: %v, %d error retries remaining", err, errRetry)
			if errRetry == 0 {
				return false, err
			}
			errRetry--
		case <-time.After(stateCheckPeriod):
			s, err := m.runtimeService.ContainerStatus(id)
			if err != nil {
				return false, err
			}
			// Only keep following container log when it is running.
			if s.State != runtimeapi.ContainerState_CONTAINER_RUNNING {
				glog.Errorf("Container %q is not running (state=%q)", id, s.State)
				// Do not return error because it's normal that the container stops
				// during waiting.
				return false, nil
			}
		}
	}
}

// parseFunc is a function parsing one log line to the internal log type.
// Notice that the caller must make sure logMessage is not nil.
type parseFunc func([]byte, *logMessage) error

var parseFuncs []parseFunc = []parseFunc{
	parseCRILog,        // CRI log format parse function
	parseDockerJSONLog, // Docker JSON log format parse function
}

// parseCRILog parses logs in CRI log format. CRI Log format example:
//   2016-10-06T00:17:09.669794202Z stdout log content 1
//   2016-10-06T00:17:09.669794203Z stderr log content 2
func parseCRILog(log []byte, msg *logMessage) error {
	var err error
	// Parse timestamp
	idx := bytes.Index(log, delimiter)
	if idx < 0 {
		return fmt.Errorf("timestamp is not found")
	}
	msg.timestamp, err = time.Parse(timeFormat, string(log[:idx]))
	if err != nil {
		return fmt.Errorf("unexpected timestamp format %q: %v", timeFormat, err)
	}

	// Parse stream type
	log = log[idx+1:]
	idx = bytes.Index(log, delimiter)
	if idx < 0 {
		return fmt.Errorf("stream type is not found")
	}
	msg.stream = streamType(log[:idx])
	if msg.stream != stdoutType && msg.stream != stderrType {
		return fmt.Errorf("unexpected stream type %q", msg.stream)
	}

	// Get log content
	msg.log = log[idx+1:]

	return nil
}

// parseDockerJSONLog parses logs in Docker JSON log format. Docker JSON log format
// example:
//   {"log":"content 1","stream":"stdout","time":"2016-10-20T18:39:20.57606443Z"}
//   {"log":"content 2","stream":"stderr","time":"2016-10-20T18:39:20.57606444Z"}
func parseDockerJSONLog(log []byte, msg *logMessage) error {
	var l = &jsonlog.JSONLog{}
	l.Reset()

	// TODO: JSON decoding is fairly expensive, we should evaluate this.
	if err := json.Unmarshal(log, l); err != nil {
		return fmt.Errorf("failed with %v to unmarshal log %q", err, l)
	}
	msg.timestamp = l.Created
	msg.stream = streamType(l.Stream)
	msg.log = []byte(l.Log)
	return nil
}

// getParseFunc returns proper parse function based on the sample log line passed in.
func getParseFunc(log []byte) (parseFunc, error) {
	for _, p := range parseFuncs {
		if err := p(log, &logMessage{}); err == nil {
			return p, nil
		}
	}
	return nil, fmt.Errorf("unsupported log format: %q", log)
}

// logWriter controls the writing into the stream based on the log options.
type logWriter struct {
	stdout io.Writer
	stderr io.Writer
	opts   *logOptions
	remain int64
}

// errMaximumWrite is returned when all bytes have been written.
var errMaximumWrite = errors.New("maximum write")

// errShortWrite is returned when the message is not fully written.
var errShortWrite = errors.New("short write")

func newLogWriter(stdout io.Writer, stderr io.Writer, opts *logOptions) *logWriter {
	w := &logWriter{
		stdout: stdout,
		stderr: stderr,
		opts:   opts,
		remain: math.MaxInt64, // initialize it as infinity
	}
	if opts.bytes >= 0 {
		w.remain = opts.bytes
	}
	return w
}

// writeLogs writes logs into stdout, stderr.
func (w *logWriter) write(msg *logMessage) error {
	if msg.timestamp.Before(w.opts.since) {
		// Skip the line because it's older than since
		return nil
	}
	line := msg.log
	if w.opts.timestamp {
		prefix := append([]byte(msg.timestamp.Format(timeFormat)), delimiter[0])
		line = append(prefix, line...)
	}
	// If the line is longer than the remaining bytes, cut it.
	if int64(len(line)) > w.remain {
		line = line[:w.remain]
	}
	// Get the proper stream to write to.
	var stream io.Writer
	switch msg.stream {
	case stdoutType:
		stream = w.stdout
	case stderrType:
		stream = w.stderr
	default:
		return fmt.Errorf("unexpected stream type %q", msg.stream)
	}
	n, err := stream.Write(line)
	w.remain -= int64(n)
	if err != nil {
		return err
	}
	// If the line has not been fully written, return errShortWrite
	if n < len(line) {
		return errShortWrite
	}
	// If there are no more bytes left, return errMaximumWrite
	if w.remain <= 0 {
		return errMaximumWrite
	}
	return nil
}
<|endoftext|>


<commit_before><commit_msg>Better logdb package comment.<commit_after><|endoftext|>


<commit_before>// +build linux

package linux

import (
	"os"
	"reflect"
	"testing"
	"time"

	"github.com/mackerelio/mackerel-agent/metrics"
)

func TestDiskGenerator(t *testing.T) {
	if os.Getenv("TRAVIS") != "" {
		// ref. https://github.com/travis-ci/travis-ci/issues/2627
		t.Skipf("Skip: can't access `/proc/diskstats` in Travis environment.")
	}

	g := &DiskGenerator{Interval: 1 * time.Second}
	values, err := g.Generate()
	if err != nil {
		t.Errorf("should not raise error: %v", err)
	}

	metrics := []string{
		"reads", "writes",
	}

	if _, ok := values["disk.sda.reads.delta"]; !ok {
		t.Skipf("Skip: this node does not have sda device")
	}

	for _, metric := range metrics {
		if value, ok := values["disk.sda."+metric+".delta"]; !ok {
			t.Errorf("Value for disk.sda.%s.delta should be collected", metric)
		} else {
			t.Logf("Disk '%s' delta collected: %+v", metric, value)
		}
	}

	for _, key := range metrics {
		if value, ok := values["disk.loop0."+key+".delta"]; ok {
			t.Errorf("Value for disk.loop0.%s should not be collected but got %v. The value won't change.", key, value)
		}
	}
}

func TestParseDiskStats(t *testing.T) {
	// insert empty line intentionally
	out := []byte(`202       1 xvda1 750193 3037 28116978 368712 16600606 7233846 424712632 23987908 0 2355636 24345740

202       2 xvda2 1641 9310 87552 1252 6365 3717 80664 24192 0 15040 25428`)

	var emptyMapping map[string]string
	result, err := parseDiskStats(out, emptyMapping)
	if err != nil {
		t.Errorf("error should be nil but: %s", err)
	}

	expect := metrics.Values{
		"disk.xvda1.reads":          750193,
		"disk.xvda1.readsMerged":    3037,
		"disk.xvda1.sectorsRead":    28116978,
		"disk.xvda1.readTime":       368712,
		"disk.xvda1.writes":         16600606,
		"disk.xvda1.writesMerged":   7233846,
		"disk.xvda1.sectorsWritten": 424712632,
		"disk.xvda1.writeTime":      23987908,
		"disk.xvda1.ioInProgress":   0,
		"disk.xvda1.ioTime":         2355636,
		"disk.xvda1.ioTimeWeighted": 24345740,
		"disk.xvda2.reads":          1641,
		"disk.xvda2.readsMerged":    9310,
		"disk.xvda2.sectorsRead":    87552,
		"disk.xvda2.readTime":       1252,
		"disk.xvda2.writes":         6365,
		"disk.xvda2.writesMerged":   3717,
		"disk.xvda2.sectorsWritten": 80664,
		"disk.xvda2.writeTime":      24192,
		"disk.xvda2.ioInProgress":   0,
		"disk.xvda2.ioTime":         15040,
		"disk.xvda2.ioTimeWeighted": 25428,
	}
	if !reflect.DeepEqual(result, expect) {
		t.Errorf("result is not expected one: %+v", result)
	}

	mapping := map[string]string{
		"xvda1": "_some_mount",
		"xvda3": "_nonused_mount",
	}
	resultWithMapping, err := parseDiskStats(out, mapping)
	if err != nil {
		t.Errorf("error should be nil but: %s", err)
	}

	expectWithMapping := metrics.Values{
		"disk._some_mount.reads":          750193,
		"disk._some_mount.readsMerged":    3037,
		"disk._some_mount.sectorsRead":    28116978,
		"disk._some_mount.readTime":       368712,
		"disk._some_mount.writes":         16600606,
		"disk._some_mount.writesMerged":   7233846,
		"disk._some_mount.sectorsWritten": 424712632,
		"disk._some_mount.writeTime":      23987908,
		"disk._some_mount.ioInProgress":   0,
		"disk._some_mount.ioTime":         2355636,
		"disk._some_mount.ioTimeWeighted": 24345740,
		"disk.xvda2.reads":                1641,
		"disk.xvda2.readsMerged":          9310,
		"disk.xvda2.sectorsRead":          87552,
		"disk.xvda2.readTime":             1252,
		"disk.xvda2.writes":               6365,
		"disk.xvda2.writesMerged":         3717,
		"disk.xvda2.sectorsWritten":       80664,
		"disk.xvda2.writeTime":            24192,
		"disk.xvda2.ioInProgress":         0,
		"disk.xvda2.ioTime":               15040,
		"disk.xvda2.ioTimeWeighted":       25428,
	}
	if !reflect.DeepEqual(resultWithMapping, expectWithMapping) {
		t.Errorf("result is not expected one: %+v", resultWithMapping)
	}
}
<commit_msg>try to stop skipping TestDiskGenerator<commit_after>// +build linux

package linux

import (
	"reflect"
	"testing"
	"time"

	"github.com/mackerelio/mackerel-agent/metrics"
)

func TestDiskGenerator(t *testing.T) {
	g := &DiskGenerator{Interval: 1 * time.Second}
	values, err := g.Generate()
	if err != nil {
		t.Errorf("should not raise error: %v", err)
	}

	metrics := []string{
		"reads", "writes",
	}

	if _, ok := values["disk.sda.reads.delta"]; !ok {
		t.Skipf("Skip: this node does not have sda device")
	}

	for _, metric := range metrics {
		if value, ok := values["disk.sda."+metric+".delta"]; !ok {
			t.Errorf("Value for disk.sda.%s.delta should be collected", metric)
		} else {
			t.Logf("Disk '%s' delta collected: %+v", metric, value)
		}
	}

	for _, key := range metrics {
		if value, ok := values["disk.loop0."+key+".delta"]; ok {
			t.Errorf("Value for disk.loop0.%s should not be collected but got %v. The value won't change.", key, value)
		}
	}
}

func TestParseDiskStats(t *testing.T) {
	// insert empty line intentionally
	out := []byte(`202       1 xvda1 750193 3037 28116978 368712 16600606 7233846 424712632 23987908 0 2355636 24345740

202       2 xvda2 1641 9310 87552 1252 6365 3717 80664 24192 0 15040 25428`)

	var emptyMapping map[string]string
	result, err := parseDiskStats(out, emptyMapping)
	if err != nil {
		t.Errorf("error should be nil but: %s", err)
	}

	expect := metrics.Values{
		"disk.xvda1.reads":          750193,
		"disk.xvda1.readsMerged":    3037,
		"disk.xvda1.sectorsRead":    28116978,
		"disk.xvda1.readTime":       368712,
		"disk.xvda1.writes":         16600606,
		"disk.xvda1.writesMerged":   7233846,
		"disk.xvda1.sectorsWritten": 424712632,
		"disk.xvda1.writeTime":      23987908,
		"disk.xvda1.ioInProgress":   0,
		"disk.xvda1.ioTime":         2355636,
		"disk.xvda1.ioTimeWeighted": 24345740,
		"disk.xvda2.reads":          1641,
		"disk.xvda2.readsMerged":    9310,
		"disk.xvda2.sectorsRead":    87552,
		"disk.xvda2.readTime":       1252,
		"disk.xvda2.writes":         6365,
		"disk.xvda2.writesMerged":   3717,
		"disk.xvda2.sectorsWritten": 80664,
		"disk.xvda2.writeTime":      24192,
		"disk.xvda2.ioInProgress":   0,
		"disk.xvda2.ioTime":         15040,
		"disk.xvda2.ioTimeWeighted": 25428,
	}
	if !reflect.DeepEqual(result, expect) {
		t.Errorf("result is not expected one: %+v", result)
	}

	mapping := map[string]string{
		"xvda1": "_some_mount",
		"xvda3": "_nonused_mount",
	}
	resultWithMapping, err := parseDiskStats(out, mapping)
	if err != nil {
		t.Errorf("error should be nil but: %s", err)
	}

	expectWithMapping := metrics.Values{
		"disk._some_mount.reads":          750193,
		"disk._some_mount.readsMerged":    3037,
		"disk._some_mount.sectorsRead":    28116978,
		"disk._some_mount.readTime":       368712,
		"disk._some_mount.writes":         16600606,
		"disk._some_mount.writesMerged":   7233846,
		"disk._some_mount.sectorsWritten": 424712632,
		"disk._some_mount.writeTime":      23987908,
		"disk._some_mount.ioInProgress":   0,
		"disk._some_mount.ioTime":         2355636,
		"disk._some_mount.ioTimeWeighted": 24345740,
		"disk.xvda2.reads":                1641,
		"disk.xvda2.readsMerged":          9310,
		"disk.xvda2.sectorsRead":          87552,
		"disk.xvda2.readTime":             1252,
		"disk.xvda2.writes":               6365,
		"disk.xvda2.writesMerged":         3717,
		"disk.xvda2.sectorsWritten":       80664,
		"disk.xvda2.writeTime":            24192,
		"disk.xvda2.ioInProgress":         0,
		"disk.xvda2.ioTime":               15040,
		"disk.xvda2.ioTimeWeighted":       25428,
	}
	if !reflect.DeepEqual(resultWithMapping, expectWithMapping) {
		t.Errorf("result is not expected one: %+v", resultWithMapping)
	}
}
<|endoftext|>


<commit_before>// Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan.
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/

// See page 16.
//!+

// Fetch prints the content found at each specified URL.
package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
)

func main() {
	for _, url := range os.Args[1:] {
		resp, err := http.Get(url)
		if err != nil {
			fmt.Fprintf(os.Stderr, "fetch: %v\n", err)
			os.Exit(1)
		}
		b, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			fmt.Fprintf(os.Stderr, "fetch: reading %s: %v\n", url, err)
			os.Exit(1)
		}
		fmt.Printf("%s", b)
	}
}

//!-
<commit_msg>modified main.go<commit_after>// Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan.
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/

// See page 16.
//!+

// Fetch prints the content found at each specified URL.
package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
)

func main() {
	for _, url := range os.Args[1:] {
		resp, err := http.Get(url)
		if err != nil {
			fmt.Fprintf(os.Stderr, "fetch: %v\n", err)
			os.Exit(1)
		}
		b, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			fmt.Fprintf(os.Stderr, "fetch: reading %s: %v\n", url, err)
			os.Exit(1)
		}
		fmt.Printf("Status: %s\n", resp.Status)
		fmt.Printf("Body: \n%s\n", b)
	}
}

//!-
<|endoftext|>


<commit_before>package persistentData

import (
	"github.com/sandstorm/dokku-enterprise-plugin/core/manifest"
	"fmt"
	"strconv"
	"github.com/mholt/archiver"
	"log"
	"github.com/sandstorm/dokku-enterprise-plugin/core/utility"
	"strings"
)

func ImportPersistentData(applicationName string, manifestWrapper manifest.ManifestWrapper, persistentDataFilePath, importTempDir string) {
	persistentDataDir := importTempDir + "/persistent-data"

	// Extracting tar.gz
	err := archiver.TarGz.Open(persistentDataFilePath, importTempDir)
	if err != nil {
		log.Fatalf("ERROR: could extract tar.gz file, error was: %v", err)
	}

	// MARIADB
	for i, mariadbNameWithPlaceholder := range manifestWrapper.Manifest.Mariadb {
		mariadbName := manifest.ReplacePlaceholderWithAppName(mariadbNameWithPlaceholder, applicationName)

		fmt.Printf("Importing database %s from %s", mariadbName, persistentDataDir + "/mariadb/" + strconv.Itoa(i) + ".sql")

		utility.ExecCommand("dokku", "mariadb:import", mariadbName, "< " + persistentDataDir + "/mariadb/" + strconv.Itoa(i) + ".sql")
	}

	// VOLUME
	allDockerOptions := manifestWrapper.GetDockerOptions()
	for _, option := range allDockerOptions {
		if (option[0:3] == "-v ") {
			volumeParts := strings.SplitN(option[3:], ":", 2)

			sourceDirectory := persistentDataDir + "/volume" + volumeParts[0]
			targetDirectory := manifest.ReplacePlaceholderWithAppName(volumeParts[0], applicationName)

			utility.CopyAndOverrideDirectory(sourceDirectory, targetDirectory)
		}
	}
}
<commit_msg>TASK: Make mariaDB import more robust<commit_after>package persistentData

import (
	"github.com/sandstorm/dokku-enterprise-plugin/core/manifest"
	"strconv"
	"github.com/mholt/archiver"
	"log"
	"github.com/sandstorm/dokku-enterprise-plugin/core/utility"
	"strings"
	"os"
	"fmt"
)

func ImportPersistentData(applicationName string, manifestWrapper manifest.ManifestWrapper, persistentDataFilePath, importTempDir string) {
	persistentDataDir := importTempDir + "/persistent-data"

	// Extracting tar.gz
	err := archiver.TarGz.Open(persistentDataFilePath, importTempDir)
	if err != nil {
		log.Fatalf("ERROR: could extract tar.gz file, error was: %v", err)
	}

	// MARIADB
	for i, mariadbNameWithPlaceholder := range manifestWrapper.Manifest.Mariadb {
		mariadbName := manifest.ReplacePlaceholderWithAppName(mariadbNameWithPlaceholder, applicationName)

		file, err := os.Open(persistentDataDir + "/mariadb/" + strconv.Itoa(i) + ".sql")
		if err != nil {
			log.Fatalf("ERROR: could not get file handler for database file, error was: %v", err)
		}

		utility.ExecCommandWithStdIn(file,"dokku", "mariadb:import", mariadbName)

		file.Close()
	}

	// VOLUME
	allDockerOptions := manifestWrapper.GetDockerOptions()
	for _, option := range allDockerOptions {
		if (option[0:3] == "-v ") {
			volumeParts := strings.SplitN(option[3:], ":", 2)

			sourceDirectory := persistentDataDir + "/volume" + volumeParts[0]
			targetDirectory := manifest.ReplacePlaceholderWithAppName(volumeParts[0], applicationName)

			utility.CopyAndOverrideDirectory(sourceDirectory, targetDirectory)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/dgryski/go-topk"
)

func main() {

	k := flag.Int("n", 500, "k")
	f := flag.String("f", "", "file to read")
	counts := flag.Bool("c", false, "each item has a count associated with it")

	flag.Parse()

	var r io.Reader

	if *f == "" {
		r = os.Stdin
	} else {
		var err error
		r, err = os.Open(*f)
		if err != nil {
			log.Fatal(err)
		}
	}

	tk := topk.New(*k)
	sc := bufio.NewScanner(r)

	for sc.Scan() {
		line := sc.Text()

		var count int
		var item string

		if *counts {
			fields := strings.Fields(line)
			cint, err := strconv.Atoi(fields[1])
			if err != nil {
				log.Println("failed to parse count: ", fields[1], ":", err)
				continue
			}
			item = fields[0]
			count = cint
		} else {
			item = line
			count = 1
		}

		tk.Insert(item, count)
	}

	if err := sc.Err(); err != nil {
		log.Fatal(err)
	}

	for _, v := range tk.Keys() {
		fmt.Println(v.Key, v.Count, v.Error)
	}
}
<commit_msg>Fix import<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/seiflotfy/skizze/counters/wrappers/topk/go-topk"
)

func main() {

	k := flag.Int("n", 500, "k")
	f := flag.String("f", "", "file to read")
	counts := flag.Bool("c", false, "each item has a count associated with it")

	flag.Parse()

	var r io.Reader

	if *f == "" {
		r = os.Stdin
	} else {
		var err error
		r, err = os.Open(*f)
		if err != nil {
			log.Fatal(err)
		}
	}

	tk := topk.New(*k)
	sc := bufio.NewScanner(r)

	for sc.Scan() {
		line := sc.Text()

		var count int
		var item string

		if *counts {
			fields := strings.Fields(line)
			cint, err := strconv.Atoi(fields[1])
			if err != nil {
				log.Println("failed to parse count: ", fields[1], ":", err)
				continue
			}
			item = fields[0]
			count = cint
		} else {
			item = line
			count = 1
		}

		tk.Insert(item, count)
	}

	if err := sc.Err(); err != nil {
		log.Fatal(err)
	}

	for _, v := range tk.Keys() {
		fmt.Println(v.Key, v.Count, v.Error)
	}
}
<|endoftext|>


<commit_before>// Application that builds chromium with or without patches and uploads the build
// to Google Storage.
package main

import (
	"flag"
	"fmt"
	"time"

	"go.skia.org/infra/go/sklog"

	"go.skia.org/infra/ct/go/ctfe/chromium_builds"
	"go.skia.org/infra/ct/go/frontend"
	"go.skia.org/infra/ct/go/master_scripts/master_common"
	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

var (
	emails         = flag.String("emails", "", "The comma separated email addresses to notify when the task is picked up and completes.")
	gaeTaskID      = flag.Int64("gae_task_id", -1, "The key of the App Engine task. This task will be updated when the task is completed.")
	runID          = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")
	targetPlatform = flag.String("target_platform", util.PLATFORM_ANDROID, "The platform the benchmark will run on (Android / Linux).")
	chromiumHash   = flag.String("chromium_hash", "", "The Chromium commit hash the checkout should be synced to. If not specified then Chromium's ToT hash is used.")
	skiaHash       = flag.String("skia_hash", "", "The Skia commit hash the checkout should be synced to. If not specified then Skia's LKGR hash is used (the hash in Chromium's DEPS file).")

	taskCompletedSuccessfully = false
	chromiumBuildTimestamp    = ""
)

func sendEmail(recipients []string) {
	emailSubject := "Chromium build task has completed"
	failureHtml := ""
	if !taskCompletedSuccessfully {
		emailSubject += " with failures"
		failureHtml = util.GetFailureEmailHtml(*runID)
	}
	bodyTemplate := `
	The Cluster telemetry queued task to create a new chromium build has completed. %s.

	%s
	You can schedule more runs here.

	Thanks!
	`
	emailBody := fmt.Sprintf(bodyTemplate, util.GetSwarmingLogsLink(*runID), failureHtml, frontend.ChromiumBuildTasksWebapp)
	if err := util.SendEmail(recipients, emailSubject, emailBody); err != nil {
		sklog.Errorf("Error while sending email: %s", err)
		return
	}
}

func updateWebappTask() {
	vars := chromium_builds.UpdateVars{}
	vars.Id = *gaeTaskID
	vars.SetCompleted(taskCompletedSuccessfully)
	skutil.LogErr(frontend.UpdateWebappTaskV2(&vars))
}

func main() {
	defer common.LogPanic()
	master_common.Init()

	// Send start email.
	emailsArr := util.ParseEmails(*emails)
	emailsArr = append(emailsArr, util.CtAdmins...)
	if len(emailsArr) == 0 {
		sklog.Error("At least one email address must be specified")
		return
	}
	skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&chromium_builds.UpdateVars{}, *gaeTaskID))
	skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Build chromium", *runID, ""))
	// Ensure webapp is updated and completion email is sent even if task fails.
	defer updateWebappTask()
	defer sendEmail(emailsArr)
	// Finish with glog flush and how long the task took.
	defer util.TimeTrack(time.Now(), "Running build chromium")
	defer sklog.Flush()

	if *chromiumHash == "" {
		sklog.Error("Must specify --chromium_hash")
		return
	}
	if *skiaHash == "" {
		sklog.Error("Must specify --skia_hash")
		return
	}

	// Create the required chromium build.
	// Note: chromium_builds.CreateChromiumBuildOnSwarming specifies the
	//       "-DSK_WHITELIST_SERIALIZED_TYPEFACES" flag only when *runID is empty.
	//       Since builds created by this master script will be consumed only by the
	//       capture_skps tasks (which require that flag) specify runID as empty here.
	chromiumBuilds, err := util.TriggerBuildRepoSwarmingTask("build_chromium", "", "chromium", "Linux", []string{*chromiumHash, *skiaHash}, []string{}, true /*singleBuild*/, 3*time.Hour, 1*time.Hour)
	if err != nil {
		sklog.Errorf("Error encountered when swarming build repo task: %s", err)
		return
	}
	if len(chromiumBuilds) != 1 {
		sklog.Errorf("Expected 1 build but instead got %d: %v", len(chromiumBuilds), chromiumBuilds)
		return
	}

	taskCompletedSuccessfully = true
}
<commit_msg>Missing runID in build_chromium swarming task<commit_after>// Application that builds chromium with or without patches and uploads the build
// to Google Storage.
package main

import (
	"flag"
	"fmt"
	"time"

	"go.skia.org/infra/go/sklog"

	"go.skia.org/infra/ct/go/ctfe/chromium_builds"
	"go.skia.org/infra/ct/go/frontend"
	"go.skia.org/infra/ct/go/master_scripts/master_common"
	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

var (
	emails         = flag.String("emails", "", "The comma separated email addresses to notify when the task is picked up and completes.")
	gaeTaskID      = flag.Int64("gae_task_id", -1, "The key of the App Engine task. This task will be updated when the task is completed.")
	runID          = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")
	targetPlatform = flag.String("target_platform", util.PLATFORM_ANDROID, "The platform the benchmark will run on (Android / Linux).")
	chromiumHash   = flag.String("chromium_hash", "", "The Chromium commit hash the checkout should be synced to. If not specified then Chromium's ToT hash is used.")
	skiaHash       = flag.String("skia_hash", "", "The Skia commit hash the checkout should be synced to. If not specified then Skia's LKGR hash is used (the hash in Chromium's DEPS file).")

	taskCompletedSuccessfully = false
	chromiumBuildTimestamp    = ""
)

func sendEmail(recipients []string) {
	emailSubject := "Chromium build task has completed"
	failureHtml := ""
	if !taskCompletedSuccessfully {
		emailSubject += " with failures"
		failureHtml = util.GetFailureEmailHtml(*runID)
	}
	bodyTemplate := `
	The Cluster telemetry queued task to create a new chromium build has completed. %s.

	%s
	You can schedule more runs here.

	Thanks!
	`
	emailBody := fmt.Sprintf(bodyTemplate, util.GetSwarmingLogsLink(*runID), failureHtml, frontend.ChromiumBuildTasksWebapp)
	if err := util.SendEmail(recipients, emailSubject, emailBody); err != nil {
		sklog.Errorf("Error while sending email: %s", err)
		return
	}
}

func updateWebappTask() {
	vars := chromium_builds.UpdateVars{}
	vars.Id = *gaeTaskID
	vars.SetCompleted(taskCompletedSuccessfully)
	skutil.LogErr(frontend.UpdateWebappTaskV2(&vars))
}

func main() {
	defer common.LogPanic()
	master_common.Init()

	// Send start email.
	emailsArr := util.ParseEmails(*emails)
	emailsArr = append(emailsArr, util.CtAdmins...)
	if len(emailsArr) == 0 {
		sklog.Error("At least one email address must be specified")
		return
	}
	skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&chromium_builds.UpdateVars{}, *gaeTaskID))
	skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Build chromium", *runID, ""))
	// Ensure webapp is updated and completion email is sent even if task fails.
	defer updateWebappTask()
	defer sendEmail(emailsArr)
	// Finish with glog flush and how long the task took.
	defer util.TimeTrack(time.Now(), "Running build chromium")
	defer sklog.Flush()

	if *chromiumHash == "" {
		sklog.Error("Must specify --chromium_hash")
		return
	}
	if *skiaHash == "" {
		sklog.Error("Must specify --skia_hash")
		return
	}

	// Create the required chromium build.
	// Note: chromium_builds.CreateChromiumBuildOnSwarming specifies the
	//       "-DSK_WHITELIST_SERIALIZED_TYPEFACES" flag only when *runID is empty.
	//       Since builds created by this master script will be consumed only by the
	//       capture_skps tasks (which require that flag) specify runID as empty here.
	chromiumBuilds, err := util.TriggerBuildRepoSwarmingTask("build_chromium", *runID, "chromium", "Linux", []string{*chromiumHash, *skiaHash}, []string{}, true /*singleBuild*/, 3*time.Hour, 1*time.Hour)
	if err != nil {
		sklog.Errorf("Error encountered when swarming build repo task: %s", err)
		return
	}
	if len(chromiumBuilds) != 1 {
		sklog.Errorf("Expected 1 build but instead got %d: %v", len(chromiumBuilds), chromiumBuilds)
		return
	}

	taskCompletedSuccessfully = true
}
<|endoftext|>


<commit_before>package mango

import (
	"fmt"
	"http"
	"log"
	"net/textproto"
	"os"
)

type Request struct {
	*http.Request
}

type Status int
type Body string

type Headers http.Header

func (h Headers) Add(key, value string) {
	textproto.MIMEHeader(h).Add(key, value)
}

func (h Headers) Set(key, value string) {
	textproto.MIMEHeader(h).Set(key, value)
}

func (h Headers) Get(key string) string {
	return textproto.MIMEHeader(h).Get(key)
}

func (h Headers) Del(key string) {
	textproto.MIMEHeader(h).Del(key)
}

type Env map[string]interface{}

func (this Env) Logger() *log.Logger {
	if this["mango.logger"] == nil {
		this["mango.logger"] = log.New(os.Stdout, "", log.Ldate|log.Ltime)
	}

	return this["mango.logger"].(*log.Logger)
}

func (this Env) Request() *Request {
	return this["mango.request"].(*Request)
}

func (this Env) Session() map[string]interface{} {
	return this["mango.session"].(map[string]interface{})
}

// This is the core app the user has written
type App func(Env) (Status, Headers, Body)

// These are pieces of middleware,
// which 'wrap' around the core App
// (and each other)
type Middleware func(Env, App) (Status, Headers, Body)

// Bundle a given list of Middleware pieces into a App
func bundle(r ...Middleware) App {
	if len(r) <= 1 {
		// Terminate the innermost piece of Middleware
		// Basically stops it from recursing any further.
		return func(input Env) (Status, Headers, Body) {
			return r[0](input, func(Env) (Status, Headers, Body) {
				panic("Core Mango App should never call it's upstream function.")
			})
		}
	}
	return wrap(r[0], bundle(r[1:]...))
}

// Attach a piece of Middleware to the outside
// of a App. This wraps the inner App
// inside the outer Middleware.
func wrap(middleware Middleware, app App) App {
	return func(input Env) (Status, Headers, Body) {
		return middleware(input, app)
	}
}

// Convert a App into Middleware
// We convert the core app into a Middleware
// so we can pass it to Bundle as part of the
// stack. Because the App does not call its
// upstream method, the resulting Middleware
// will just ignore any upstream passed to it.
func middlewareify(app App) Middleware {
	return func(input Env, upstream App) (Status, Headers, Body) {
		return app(input)
	}
}

type Stack struct {
	Address    string
	middleware []Middleware
	app        App
}

func Version() []int {
	return []int{0, 3, 4}
}

func VersionString() string {
	v := Version()
	return fmt.Sprintf("%d.%02d.%02d", v[0], v[1], v[2])
}

func (this *Stack) Middleware(middleware ...Middleware) {
	this.middleware = middleware
}

func (this *Stack) Compile(app App) App {
	this.app = app
	return bundle(append(this.middleware, middlewareify(this.app))...)
}

func (this *Stack) HandlerFunc(app App) http.HandlerFunc {
	compiled_app := this.Compile(app)
	return func(w http.ResponseWriter, r *http.Request) {
		env := make(map[string]interface{})
		env["mango.request"] = &Request{r}
		env["mango.version"] = Version()

		status, headers, body := compiled_app(env)

		for key, values := range headers {
			for _, value := range values {
				w.Header().Add(key, value)
			}
		}
		w.WriteHeader(int(status))
		fmt.Fprintf(w, string(body))
	}
}

func (this *Stack) Run(app App) os.Error {
	if this.Address == "" {
		this.Address = "0.0.0.0:8000"
	}
	fmt.Println("Starting Mango Stack On:", this.Address)
	http.HandleFunc("/", this.HandlerFunc(app))
	return http.ListenAndServe(this.Address, nil)
}
<commit_msg>Bumped version to 0.03.05<commit_after>package mango

import (
	"fmt"
	"http"
	"log"
	"net/textproto"
	"os"
)

type Request struct {
	*http.Request
}

type Status int
type Body string

type Headers http.Header

func (h Headers) Add(key, value string) {
	textproto.MIMEHeader(h).Add(key, value)
}

func (h Headers) Set(key, value string) {
	textproto.MIMEHeader(h).Set(key, value)
}

func (h Headers) Get(key string) string {
	return textproto.MIMEHeader(h).Get(key)
}

func (h Headers) Del(key string) {
	textproto.MIMEHeader(h).Del(key)
}

type Env map[string]interface{}

func (this Env) Logger() *log.Logger {
	if this["mango.logger"] == nil {
		this["mango.logger"] = log.New(os.Stdout, "", log.Ldate|log.Ltime)
	}

	return this["mango.logger"].(*log.Logger)
}

func (this Env) Request() *Request {
	return this["mango.request"].(*Request)
}

func (this Env) Session() map[string]interface{} {
	return this["mango.session"].(map[string]interface{})
}

// This is the core app the user has written
type App func(Env) (Status, Headers, Body)

// These are pieces of middleware,
// which 'wrap' around the core App
// (and each other)
type Middleware func(Env, App) (Status, Headers, Body)

// Bundle a given list of Middleware pieces into a App
func bundle(r ...Middleware) App {
	if len(r) <= 1 {
		// Terminate the innermost piece of Middleware
		// Basically stops it from recursing any further.
		return func(input Env) (Status, Headers, Body) {
			return r[0](input, func(Env) (Status, Headers, Body) {
				panic("Core Mango App should never call it's upstream function.")
			})
		}
	}
	return wrap(r[0], bundle(r[1:]...))
}

// Attach a piece of Middleware to the outside
// of a App. This wraps the inner App
// inside the outer Middleware.
func wrap(middleware Middleware, app App) App {
	return func(input Env) (Status, Headers, Body) {
		return middleware(input, app)
	}
}

// Convert a App into Middleware
// We convert the core app into a Middleware
// so we can pass it to Bundle as part of the
// stack. Because the App does not call its
// upstream method, the resulting Middleware
// will just ignore any upstream passed to it.
func middlewareify(app App) Middleware {
	return func(input Env, upstream App) (Status, Headers, Body) {
		return app(input)
	}
}

type Stack struct {
	Address    string
	middleware []Middleware
	app        App
}

func Version() []int {
	return []int{0, 3, 5}
}

func VersionString() string {
	v := Version()
	return fmt.Sprintf("%d.%02d.%02d", v[0], v[1], v[2])
}

func (this *Stack) Middleware(middleware ...Middleware) {
	this.middleware = middleware
}

func (this *Stack) Compile(app App) App {
	this.app = app
	return bundle(append(this.middleware, middlewareify(this.app))...)
}

func (this *Stack) HandlerFunc(app App) http.HandlerFunc {
	compiled_app := this.Compile(app)
	return func(w http.ResponseWriter, r *http.Request) {
		env := make(map[string]interface{})
		env["mango.request"] = &Request{r}
		env["mango.version"] = Version()

		status, headers, body := compiled_app(env)

		for key, values := range headers {
			for _, value := range values {
				w.Header().Add(key, value)
			}
		}
		w.WriteHeader(int(status))
		fmt.Fprintf(w, string(body))
	}
}

func (this *Stack) Run(app App) os.Error {
	if this.Address == "" {
		this.Address = "0.0.0.0:8000"
	}
	fmt.Println("Starting Mango Stack On:", this.Address)
	http.HandleFunc("/", this.HandlerFunc(app))
	return http.ListenAndServe(this.Address, nil)
}
<|endoftext|>


<commit_before>package swarm

import (
	"sync"
	"testing"
	"time"

	peer "github.com/jbenet/go-ipfs/p2p/peer"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
)

func TestSimultOpen(t *testing.T) {
	// t.Skip("skipping for another test")
	t.Parallel()

	ctx := context.Background()
	swarms := makeSwarms(ctx, t, 2)

	// connect everyone
	{
		var wg sync.WaitGroup
		connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) {
			// copy for other peer
			log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.local, dst, addr)
			s.peers.AddAddress(dst, addr)
			if _, err := s.Dial(ctx, dst); err != nil {
				t.Fatal("error swarm dialing to peer", err)
			}
			wg.Done()
		}

		log.Info("Connecting swarms simultaneously.")
		wg.Add(2)
		go connect(swarms[0], swarms[1].local, swarms[1].ListenAddresses()[0])
		go connect(swarms[1], swarms[0].local, swarms[0].ListenAddresses()[0])
		wg.Wait()
	}

	for _, s := range swarms {
		s.Close()
	}
}

func TestSimultOpenMany(t *testing.T) {
	// t.Skip("very very slow")
	t.Parallel()

	addrs := 20
	SubtestSwarm(t, addrs, 10)
}

func TestSimultOpenFewStress(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	// t.Skip("skipping for another test")
	t.Parallel()

	msgs := 40
	swarms := 2
	rounds := 10
	// rounds := 100

	for i := 0; i < rounds; i++ {
		SubtestSwarm(t, swarms, msgs)
		<-time.After(10 * time.Millisecond)
	}
}
<commit_msg>swarm: TestSimultOpenMany has problems in CI.<commit_after>package swarm

import (
	"sync"
	"testing"
	"time"

	peer "github.com/jbenet/go-ipfs/p2p/peer"
	ci "github.com/jbenet/go-ipfs/util/testutil/ci"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
)

func TestSimultOpen(t *testing.T) {
	// t.Skip("skipping for another test")
	t.Parallel()

	ctx := context.Background()
	swarms := makeSwarms(ctx, t, 2)

	// connect everyone
	{
		var wg sync.WaitGroup
		connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) {
			// copy for other peer
			log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.local, dst, addr)
			s.peers.AddAddress(dst, addr)
			if _, err := s.Dial(ctx, dst); err != nil {
				t.Fatal("error swarm dialing to peer", err)
			}
			wg.Done()
		}

		log.Info("Connecting swarms simultaneously.")
		wg.Add(2)
		go connect(swarms[0], swarms[1].local, swarms[1].ListenAddresses()[0])
		go connect(swarms[1], swarms[0].local, swarms[0].ListenAddresses()[0])
		wg.Wait()
	}

	for _, s := range swarms {
		s.Close()
	}
}

func TestSimultOpenMany(t *testing.T) {
	// t.Skip("very very slow")

	addrs := 20
	rounds := 10
	if ci.IsRunning() {
		addrs = 10
		rounds = 5
	}
	SubtestSwarm(t, addrs, rounds)
}

func TestSimultOpenFewStress(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	// t.Skip("skipping for another test")
	t.Parallel()

	msgs := 40
	swarms := 2
	rounds := 10
	// rounds := 100

	for i := 0; i < rounds; i++ {
		SubtestSwarm(t, swarms, msgs)
		<-time.After(10 * time.Millisecond)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"text/tabwriter"

	"github.com/ryanuber/go-license"
)

const root = "vendor/"

// Response contains license and file info
type response struct {
	License string
	File    string
}

type request struct {
	path        string
	licenseFile string
}

func catch(err error) {
	if err != nil {
		panic(err)
	}
}
func getLicense(input chan request, output chan response, size int) {
	for i := range input {
		file := strings.TrimPrefix(i.path, root)
		if i.licenseFile == "" {
			output <- response{File: file, License: "FILE HAS NO LICENSE"}
		} else {
			l, err := license.NewFromFile(i.licenseFile)
			if err != nil {
				f, err := os.Open(i.licenseFile)
				catch(err)
				dat, err := bufio.NewReader(f).Peek(size)
				catch(err)
				output <- response{File: file, License: "#" + string(dat)}
			} else {
				output <- response{File: file, License: l.Type}
			}
		}
	}
}

func isLicense(fileName string) bool {
	fileName = strings.ToUpper(fileName)
	return strings.Contains(fileName, "LICENSE") || strings.Contains(fileName, "COPYING")
}

func isPackage(filePath string) bool {
	files, err := ioutil.ReadDir(filePath)
	if err != nil {
		return false
	}
	for _, file := range files {
		if file.Name() == ".git" {
			return true
		}
	}
	return false
}

func getLicenseFile(filePath string) string {
	files, err := ioutil.ReadDir(filePath)
	if err != nil {
		return ""
	}
	for _, file := range files {
		if isLicense(file.Name()) {
			return path.Join(filePath, file.Name())
		}
	}
	return ""
}

func getPackageLicenses() map[string]string {
	var licenseInfo = map[string]string{}

	filepath.Walk(root, func(filePath string, f os.FileInfo, err error) error {
		if err != nil {
			fmt.Println("No vendor folder found. Aborting..")
			return err
		}

		if isPackage(filePath) {
			licenseInfo[filePath] = getLicenseFile(filePath)
		} else if isLicense(filePath) && f.Mode().IsRegular() {
			licenseInfo[path.Dir(filePath)] = filePath
		}

		return nil
	})
	return licenseInfo
}

func main() {
	input := make(chan request)
	output := make(chan response)

	packageFiles := getPackageLicenses()

	size := 100
	if len(os.Args) > 1 {
		size, _ = strconv.Atoi(os.Args[1])
	}
	for a := 0; a < 5; a++ {
		go getLicense(input, output, size)
	}

	if len(packageFiles) == 0 { // checks that a license file was gotten
		return
	}

	go func() {
		for k, v := range packageFiles {
			input <- request{
				path:        k,
				licenseFile: v,
			}
		}
	}()
	w := tabwriter.NewWriter(os.Stdout, 1, 4, 2, ' ', 0)
	for i := 0; i < len(packageFiles); i++ {
		out := <-output
		_, err := w.Write([]byte(out.File + "\t" + out.License + "\n"))
		if err != nil {
			panic(err)
		}
	}
	w.Flush()

}
<commit_msg>Change license formatting and sort licenses<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"sort"
	"strconv"

	"github.com/ryanuber/go-license"
)

const root = "vendor/"

// Response contains license and file info
type response struct {
	License string
	File    string
}

// ByName type is used to implement sort interface to sort response by name
type ByName []response

func (s ByName) Len() int           { return len(s) }
func (s ByName) Less(i, j int) bool { return s[i].File < s[j].File }
func (s ByName) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

func (r response) String() string {
	return r.File + " ------- > " + r.License + "\n"
}

type request struct {
	Path        string
	LicenseFile string
}

func formatLicense(file string, size int) string {
	f, err := ioutil.ReadFile(file)
	if err != nil {
		panic(err)
	}
	data := strings.Split(string(f), "\n")
	if size <= len(data) {
		data = data[0:size]
	}
	formatted := "UNKOWN LICENSE\n\n|\t" + strings.Join(data, "\n|\t") + "\n"
	return formatted
}

func getLicense(input chan request, output chan response, size int) {
	for i := range input {
		file := strings.TrimPrefix(i.Path, root)
		if i.LicenseFile == "" {
			output <- response{File: file, License: "FILE HAS NO LICENSE"}
		} else {
			l, err := license.NewFromFile(i.LicenseFile)
			if err != nil {
				data := formatLicense(i.LicenseFile, size)
				output <- response{File: file, License: data}
			} else {
				output <- response{File: file, License: l.Type}
			}
		}
	}
}

func isLicense(fileName string) bool {
	fileName = strings.ToUpper(fileName)
	return strings.Contains(fileName, "LICENSE") || strings.Contains(fileName, "COPYING")
}

func isPackage(filePath string) bool {
	files, err := ioutil.ReadDir(filePath)
	if err != nil {
		return false
	}
	for _, file := range files {
		if file.Name() == ".git" {
			return true
		}
	}
	return false
}

func getLicenseFile(filePath string) string {
	files, err := ioutil.ReadDir(filePath)
	if err != nil {
		return ""
	}
	for _, file := range files {
		if isLicense(file.Name()) {
			return path.Join(filePath, file.Name())
		}
	}
	return ""
}

func getPackageLicenses() map[string]string {
	var licenseInfo = map[string]string{}

	filepath.Walk(root, func(filePath string, f os.FileInfo, err error) error {
		if err != nil {
			fmt.Println("No vendor folder found. Aborting..")
			return err
		}

		if isPackage(filePath) {
			licenseInfo[filePath] = getLicenseFile(filePath)
		} else if isLicense(filePath) && f.Mode().IsRegular() {
			licenseInfo[path.Dir(filePath)] = filePath
		}

		return nil
	})
	return licenseInfo
}

func main() {
	input := make(chan request)
	output := make(chan response)

	packageFiles := getPackageLicenses()

	size := 5
	if len(os.Args) > 1 {
		size, _ = strconv.Atoi(os.Args[1])
	}
	for a := 0; a < 5; a++ {
		go getLicense(input, output, size)
	}

	if len(packageFiles) == 0 {
		return
	}

	go func() {
		for k, v := range packageFiles {
			input <- request{
				Path:        k,
				LicenseFile: v,
			}
		}
	}()
	var out []response
	for i := 0; i < len(packageFiles); i++ {
		out = append(out, <-output)
	}

	sort.Sort(ByName(out))
	for _, v := range out {
		fmt.Println(v)
	}

}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "github.com/arbovm/levenshtein"
    "bufio"
    "os"
    "strings"
)

func generateHash(path string) {
  inFile, _ := os.Open(path)
  defer inFile.Close()
  scanner := bufio.NewScanner(inFile)
	scanner.Split(bufio.ScanLines)

  for scanner.Scan() {
    s := strings.Replace(scanner.Text(),"/","",-1)
    partials, num := getPartials(s)
    if num>2 {
        for i := 0; i < num; i ++ {
            addToCache(partials[i],s)
        }
    }
  }
}

func addToCache(spartial string, s string) {
    f, err := os.OpenFile("cache/" + spartial, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
    if err != nil {
        fmt.Println("%v",spartial)
        panic(err)
    }

    defer f.Close()

    if _, err = f.WriteString(s+"\n"); err != nil {
        fmt.Println("%v",spartial)
        panic(err)
    }
}

func getPartials(s string) ([]string, int) {
    partials := make([]string,1000)
    num := 0
    s = strings.Replace(s,"/","",-1)
    slen := len(s)
    if slen <= 4 {
       partials[num] = s
       num = num + 1
    } else {
        for i := 0; i <= slen-3; i ++ {
            partials[num] = s[i:i+3]
            num = num + 1
        }
    }
    return partials,num
}

func getMatch(s string) (string) {
    partials, num := getPartials(s)
    matches := make([]string,1000)
    numm := 0
    if num>2 {
        for i := 0; i < num; i ++ {
            // allPossibleMatches
        }
    }
    match := "No match"
    return match
}

func main() {
    //generateHash("wordlist")
    match := getMatch("aardvark")
    fmt.Println("%v",match)
    s1 := "marcy playground"
    s2 := "mary playground"
    fmt.Printf("The distance between %v and %v is %v\n",
        s1, s2, levenshtein.Distance(s1, s2))
    // -> The distance between kitten and sitting is 3
}
<commit_msg>Workds<commit_after>package main

import (
    "fmt"
    "github.com/arbovm/levenshtein"
    "bufio"
    "os"
    "strings"
)

func generateHash(path string) {
  inFile, _ := os.Open(path)
  defer inFile.Close()
  scanner := bufio.NewScanner(inFile)
	scanner.Split(bufio.ScanLines)

  for scanner.Scan() {
    s := strings.Replace(scanner.Text(),"/","",-1)
    partials, num := getPartials(s)
    if num>2 {
        for i := 0; i < num; i ++ {
            addToCache(partials[i],s)
        }
    }
  }
}

func addToCache(spartial string, s string) {
    f, err := os.OpenFile("cache/" + spartial, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
    if err != nil {
        fmt.Println("%v",spartial)
        panic(err)
    }

    defer f.Close()

    if _, err = f.WriteString(s+"\n"); err != nil {
        fmt.Println("%v",spartial)
        panic(err)
    }
}

func stringInSlice(a string, list []string) bool {
    for _, b := range list {
        if b == a {
            return true
        }
    }
    return false
}

func getPartials(s string) ([]string, int) {
    partials := make([]string,1000)
    num := 0
    s = strings.Replace(s,"/","",-1)
    slen := len(s)
    if slen <= 4 {
       partials[num] = s
       num = num + 1
    } else {
        for i := 0; i <= slen-4; i ++ {
            partials[num] = s[i:i+4]
            num = num + 1
        }
    }
    return partials,num
}

func getMatch(s string) (string) {
    match := "No match"
    partials, num := getPartials(s)
    matches := make([]string,10000)
    numm := 0
    for i := 0; i < num; i ++ {

      inFile, _ := os.Open("cache/"+partials[i])
      defer inFile.Close()
      scanner := bufio.NewScanner(inFile)
        scanner.Split(bufio.ScanLines)

      for scanner.Scan() {
        if stringInSlice(scanner.Text(),matches) == false {
            matches[numm] = scanner.Text()
            numm = numm + 1
        }
      }

    }
    //fmt.Printf("%v",matches[0:numm])
    fmt.Printf("searching through %v matches\n",numm)
    bestLevenshtein := 1000
    for i := 0; i < numm; i ++ {
      d := levenshtein.Distance(s, matches[i])
      if (d < bestLevenshtein) {
        bestLevenshtein = d
        match = matches[i]
      }
    }
    return match
}

func main() {
    //generateHash("wordlist")
    match := getMatch("madgascar")
    fmt.Printf("Match: %v\n",match)
    s1 := "marcy playground"
    s2 := "mary playground"
    fmt.Printf("The distance between %v and %v is %v\n",
        s1, s2, levenshtein.Distance(s1, s2))
    // -> The distance between kitten and sitting is 3
}
<|endoftext|>


<commit_before>package main

import (
  "os"
  "flag"
  "fmt"
  "io"
  "log"
  "net/http"
  "net/http/cookiejar"
  "net/url"
  "strings"
  "path"
  "code.google.com/p/go.net/html"
)

func Split2(str, sep string) (string, string) {
  s := strings.Split(str, sep)
  return s[0], s[1]
}

func fetch(url string) {
  res, err := http.Get(url)
  if err != nil {
    log.Fatal(err)
  }
  if err != nil {
    log.Fatal(err)
  }
  // http://stackoverflow.com/questions/1821811/how-to-read-write-from-to-file
  fo, err := os.Create(path.Base(url))
  if err != nil {
    panic(err)
  }
  defer func() {
    if err := fo.Close(); err != nil {
      panic(err)
    }
  }()
  buf := make([]byte, 65536)
  for {
    n, err := res.Body.Read(buf)
    if err != nil && err != io.EOF {
      panic(err)
    }
    if n == 0 {
      break
    }

    if _, err := fo.Write(buf[:n]); err != nil {
      panic(err)
    }
  }
  res.Body.Close()
}

func Fuckoff(pres *http.Response) {
}

func FindLinks(body io.Reader) chan string {
  c := make(chan string)

  go func() {
    z := html.NewTokenizer(body)
    for {
      tt := z.Next()
      if tt == html.ErrorToken {
        break
      }
      if tt == html.StartTagToken {
        tn, _ := z.TagName()
        if len(tn) == 1 && tn[0] == 'a' {
          for {
            key, value, more := z.TagAttr()
            // http://stackoverflow.com/questions/14230145/what-is-the-best-way-to-convert-byte-array-to-string
            if string(key) == "href" {
              v := string(value)
              // http://codereview.stackexchange.com/questions/28386/fibonacci-generator-with-golang
              c <- v
            }
            if !more {
              break
            }
          }
        }
      }
    }
    c <- ""
  }()

  return c
}

const (
  _ = iota
  linkfound
  linkcrawled
)

func findBoardLinks(client *http.Client, start string, board_links map[string]int) (board_links_out map[string]int) {
  res, err := client.Get(start)
  if err != nil {
    log.Fatal(err)
  }
  //fmt.Printf("%s\n", start)

  c := FindLinks(res.Body)
  for {
    v := <- c
    if v == "" {
      break
    }
    fmt.Printf("%v\n", v)
    if strings.Contains(v, start) && strings.Contains(v, "?board=") {
      fmt.Printf("recursing\n")
      _, found := board_links[v]
      if !found {
        board_links[v] = linkfound
      }
    }
  }
  res.Body.Close()
  return board_links
}

func main() {
  //var start string
  //flag.StringVar(&start, "start", "", "starting url")
  //flag.Parse()

  flag.Parse()
  start := flag.Args()[0]
  username, password := Split2(flag.Args()[1], ":")

  // http://stackoverflow.com/questions/18414212/golang-how-to-follow-location-with-cookie
  options := cookiejar.Options{}
  cookie_jar, err := cookiejar.New(&options)
  client := &http.Client{
    Jar: cookie_jar,
  }
  pres, err := client.PostForm(start + "index.php?action=login2",
    url.Values{"user": {username}, "passwrd": {password}})
  if err != nil {
    log.Fatal(err)
  }
  Fuckoff(pres)

  board_links := map[string]int{}
  board_links = findBoardLinks(client, start, board_links)
}
<commit_msg>Crawling<commit_after>package main

import (
  "os"
  "flag"
  "fmt"
  "io"
  "log"
  "net/http"
  "net/http/cookiejar"
  "net/url"
  "strings"
  "path"
  "code.google.com/p/go.net/html"
)

func Split2(str, sep string) (string, string) {
  s := strings.Split(str, sep)
  return s[0], s[1]
}

func fetch(url string) {
  res, err := http.Get(url)
  if err != nil {
    log.Fatal(err)
  }
  if err != nil {
    log.Fatal(err)
  }
  // http://stackoverflow.com/questions/1821811/how-to-read-write-from-to-file
  fo, err := os.Create(path.Base(url))
  if err != nil {
    panic(err)
  }
  defer func() {
    if err := fo.Close(); err != nil {
      panic(err)
    }
  }()
  buf := make([]byte, 65536)
  for {
    n, err := res.Body.Read(buf)
    if err != nil && err != io.EOF {
      panic(err)
    }
    if n == 0 {
      break
    }

    if _, err := fo.Write(buf[:n]); err != nil {
      panic(err)
    }
  }
  res.Body.Close()
}

func Fuckoff(pres *http.Response) {
}

func FindLinks(body io.Reader) chan string {
  c := make(chan string)

  go func() {
    z := html.NewTokenizer(body)
    for {
      tt := z.Next()
      if tt == html.ErrorToken {
        break
      }
      if tt == html.StartTagToken {
        tn, _ := z.TagName()
        if len(tn) == 1 && tn[0] == 'a' {
          for {
            key, value, more := z.TagAttr()
            // http://stackoverflow.com/questions/14230145/what-is-the-best-way-to-convert-byte-array-to-string
            if string(key) == "href" {
              v := string(value)
              // http://codereview.stackexchange.com/questions/28386/fibonacci-generator-with-golang
              c <- v
            }
            if !more {
              break
            }
          }
        }
      }
    }
    c <- ""
  }()

  return c
}

const (
  _ = iota
  linkfound
  linkcrawled
)

func findBoardLinks(client *http.Client, start string, board_links map[string]int) (board_links_out map[string]int) {
  res, err := client.Get(start)
  if err != nil {
    log.Fatal(err)
  }
  //fmt.Printf("%s\n", start)

  c := FindLinks(res.Body)
  for {
    v := <- c
    if v == "" {
      break
    }
    //fmt.Printf("%v\n", v)
    if strings.Contains(v, start) && strings.Contains(v, "?board=") {
      // index.php?board=18.0;sort=starter -- drop ;.*
      if strings.Contains(v, ";") {
        pre, _ := Split2(v, ";")
        v = pre
      }
      //fmt.Printf("recursing\n")
      _, found := board_links[v]
      if !found {
        board_links[v] = linkfound
      }
    }
  }
  res.Body.Close()
  return board_links
}

func main() {
  //var start string
  //flag.StringVar(&start, "start", "", "starting url")
  //flag.Parse()

  flag.Parse()
  start := flag.Args()[0]
  username, password := Split2(flag.Args()[1], ":")

  // http://stackoverflow.com/questions/18414212/golang-how-to-follow-location-with-cookie
  options := cookiejar.Options{}
  cookie_jar, err := cookiejar.New(&options)
  client := &http.Client{
    Jar: cookie_jar,
  }
  pres, err := client.PostForm(start + "index.php?action=login2",
    url.Values{"user": {username}, "passwrd": {password}})
  if err != nil {
    log.Fatal(err)
  }
  Fuckoff(pres)

  board_links := map[string]int{}
  board_links[start] = linkfound
  for {
    copy := board_links
    any := false
    for key, value := range copy {
      if value == linkcrawled {
        continue
      }
      fmt.Printf("%v\n", key)
      board_links = findBoardLinks(client, key, board_links)
      board_links[key] = linkcrawled
      any = true
    }
    if !any {
      break
    }
  }
}
<|endoftext|>


<commit_before>package fscache

import (
	"bytes"
	"errors"
	"io"
	"sync"
	"time"
)

type memFS struct {
	mu    sync.RWMutex
	files map[string]*memFile
}

func NewMemFs() FileSystem {
	return &memFS{
		files: make(map[string]*memFile),
	}
}

func (fs *memFS) Reload(add func(key, name string)) error {
	return nil
}

func (fs *memFS) AccessTimes(name string) (rt, wt time.Time, err error) {
	fs.mu.RLock()
	defer fs.mu.RUnlock()
	f, ok := fs.files[name]
	if ok {
		return f.rt, f.wt, nil
	}
	return rt, wt, errors.New("file has not been read")
}

func (fs *memFS) Create(name string) (File, error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if _, ok := fs.files[name]; ok {
		return nil, errors.New("file exists")
	}
	f := &memFile{
		name: name,
		r:    bytes.NewBuffer(nil),
		wt:   time.Now(),
	}
	fs.files[name] = f
	return f, nil
}

func (fs *memFS) Open(name string) (io.ReadCloser, error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if f, ok := fs.files[name]; ok {
		f.rt = time.Now()
		return &memReader{memFile: f}, nil
	}
	return nil, errors.New("file does not exist")
}

func (fs *memFS) Remove(name string) error {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	delete(fs.files, name)
	return nil
}

func (fs *memFS) RemoveAll() error {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	fs.files = nil
	return nil
}

type memFile struct {
	mu     sync.RWMutex
	name   string
	r      *bytes.Buffer
	rt, wt time.Time
}

func (f *memFile) Name() string {
	return f.name
}

func (f *memFile) Write(p []byte) (int, error) {
	if len(p) > 0 {
		f.mu.Lock()
		defer f.mu.Unlock()
		return f.r.Write(p)
	}
	return len(p), nil
}

func (f *memFile) Bytes() []byte {
	f.mu.RLock()
	defer f.mu.RUnlock()
	return f.r.Bytes()
}

func (f *memFile) Close() error {
	return nil
}

type memReader struct {
	*memFile
	n int
}

func (r *memReader) Read(p []byte) (n int, err error) {
	n, err = bytes.NewReader(r.Bytes()[r.n:]).Read(p)
	r.n += n
	return n, err
}

func (r *memReader) Close() error {
	return nil
}
<commit_msg>added godoc<commit_after>package fscache

import (
	"bytes"
	"errors"
	"io"
	"sync"
	"time"
)

type memFS struct {
	mu    sync.RWMutex
	files map[string]*memFile
}

// NewMemFs creates an in-memory FileSystem.
// It does not support persistence (Reload is a nop).
func NewMemFs() FileSystem {
	return &memFS{
		files: make(map[string]*memFile),
	}
}

func (fs *memFS) Reload(add func(key, name string)) error {
	return nil
}

func (fs *memFS) AccessTimes(name string) (rt, wt time.Time, err error) {
	fs.mu.RLock()
	defer fs.mu.RUnlock()
	f, ok := fs.files[name]
	if ok {
		return f.rt, f.wt, nil
	}
	return rt, wt, errors.New("file has not been read")
}

func (fs *memFS) Create(name string) (File, error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if _, ok := fs.files[name]; ok {
		return nil, errors.New("file exists")
	}
	f := &memFile{
		name: name,
		r:    bytes.NewBuffer(nil),
		wt:   time.Now(),
	}
	fs.files[name] = f
	return f, nil
}

func (fs *memFS) Open(name string) (io.ReadCloser, error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if f, ok := fs.files[name]; ok {
		f.rt = time.Now()
		return &memReader{memFile: f}, nil
	}
	return nil, errors.New("file does not exist")
}

func (fs *memFS) Remove(name string) error {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	delete(fs.files, name)
	return nil
}

func (fs *memFS) RemoveAll() error {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	fs.files = nil
	return nil
}

type memFile struct {
	mu     sync.RWMutex
	name   string
	r      *bytes.Buffer
	rt, wt time.Time
}

func (f *memFile) Name() string {
	return f.name
}

func (f *memFile) Write(p []byte) (int, error) {
	if len(p) > 0 {
		f.mu.Lock()
		defer f.mu.Unlock()
		return f.r.Write(p)
	}
	return len(p), nil
}

func (f *memFile) Bytes() []byte {
	f.mu.RLock()
	defer f.mu.RUnlock()
	return f.r.Bytes()
}

func (f *memFile) Close() error {
	return nil
}

type memReader struct {
	*memFile
	n int
}

func (r *memReader) Read(p []byte) (n int, err error) {
	n, err = bytes.NewReader(r.Bytes()[r.n:]).Read(p)
	r.n += n
	return n, err
}

func (r *memReader) Close() error {
	return nil
}
<|endoftext|>