<commit_before>/*
Package table provides functionality for holding and describing tabular data.

TODO(2020-03-01) Add support for justification.
- The table should have a default justification for each column.
- Each column should be able to override the default table justification.
*/
package table

import (
	"bytes"
	"fmt"
	"strings"

	"github.com/kward/golib/math"
	kstrings "github.com/kward/golib/strings"
)

const MAX_COLS = 100

// Row describes a row in the table.
type Row struct {
	columns   []*Column // Columnar data of the row.
	sizes     []int     // Sizes of the columns.
	isComment bool
}

// NewRow instantiates a new row. If the row is a comment, there can be only one
// record.
func NewRow(records []string, isComment bool) (*Row, error) {
	if isComment && len(records) > 1 {
		return nil, fmt.Errorf("only one record allowed for a comment row")
	}
	return newRow(records, isComment), nil
}

func newRow(records []string, isComment bool) *Row {
	cols := []*Column{}
	sizes := []int{}
	for _, r := range records {
		cols = append(cols, &Column{cell: r})
		sizes = append(sizes, len(r))
	}
	return &Row{cols, sizes, isComment}
}

// Values returns the cell data for the row.
func (r *Row) Values() []string {
	vs := make([]string, r.NumColumns())
	for i, c := range r.Columns() {
		vs[i] = c.Value()
	}
	return vs
}

// Columns held in the row.
func (r *Row) Columns() []*Column { return r.columns }

// NumColumns returns the number of columns in the row.
func (r *Row) NumColumns() int { return len(r.columns) }

// Sizes of the columns.
func (r *Row) Sizes() []int { return r.sizes }

// IsComment returns true if the full line is a comment.
func (r *Row) IsComment() bool { return r.isComment }

// String implements fmt.Stringer.
func (r *Row) String() string {
	var buf bytes.Buffer
	for i, col := range r.columns {
		if i > 0 {
			buf.WriteString(" ")
		}
		buf.WriteString(fmt.Sprintf("%q", col.String()))
	}
	return buf.String()
}

// Column holds the data for each column.
type Column struct {
	cell string // The actual cell data.
}

// Value of the column.
func (c *Column) Value() string { return c.cell }

// Length of the cell.
func (c *Column) Length() int { return len(c.cell) }

// String implements fmt.Stringer.
func (c *Column) String() string { return c.cell }

type Table struct {
	opts *options

	rows     []*Row
	colSizes []int
}

func NewTable(opts ...func(*options) error) (*Table, error) {
	o := &options{}
	o.setCommentPrefix("#")
	o.setEnableComments(false)
	o.setSectionReset(false)
	for _, opt := range opts {
		if err := opt(o); err != nil {
			return nil, err
		}
	}
	return &Table{
		opts: o,
		rows: []*Row{},
	}, nil
}

// Append lines to the table.
func (t *Table) Append(records ...[]string) {
	for _, rs := range records {
		cs := make([]*Column, len(rs)) // Columns.
		for i, r := range rs {
			cs[i] = &Column{cell: r}
		}
		t.rows = append(t.rows, &Row{
			columns: cs,
		})
	}
}

// ColSizes returns the maximum size of each column.
func (t *Table) ColSizes() []int { return t.colSizes }

// Rows returns the table row data.
func (t *Table) Rows() []*Row { return t.rows }

// NumRows returns the number of rows in the table.
func (t *Table) NumRows() int { return len(t.rows) }

// String implements fmt.Stringer.
func (t *Table) String() string {
	var buf bytes.Buffer
	buf.WriteRune('[')
	for i, row := range t.rows {
		if i > 0 {
			buf.WriteString(", ")
		}
		buf.WriteString(row.String())
	}
	buf.WriteRune(']')
	return fmt.Sprintf("%v sizes: %v\n", buf.String(), t.colSizes)
}

// Split lines of text into a table.
//
// The count `n` determines the number of substrings to return:
//
//     n > 0: at most n columns; the last column will be the unsplit remainder.
//     n == 0: the result is nil (an empty table)
//     n < 0: all columns
func Split(lines []string, ifs string, n int, opts ...func(*options) error) (*Table, error) {
	if n > MAX_COLS {
		return nil, fmt.Errorf("column count %d exceeds supported maximum number of columns %d", n, MAX_COLS)
	}

	tbl, err := NewTable(opts...)
	if err != nil {
		return nil, fmt.Errorf("error instantiating a table; %s", err)
	}

	var sizes []int
	switch {
	case n > 0:
		sizes = make([]int, n)
	case n == 0:
		return tbl, nil
	case n < 0:
		sizes = make([]int, MAX_COLS)
	}

	colsSeen := 0
	rows := []*Row{}
	for _, line := range lines {
		row := splitLine(tbl.opts, line, ifs, n)
		for j, col := range row.Columns() {
			if j >= MAX_COLS-1 {
				j = MAX_COLS - 1
			}
			if !row.IsComment() {
				sizes[j] = math.Max(col.Length(), sizes[j])
			}
		}
		rows = append(rows, row)
		colsSeen = math.Max(row.NumColumns(), colsSeen)
	}

	tbl.rows = rows
	tbl.colSizes = sizes[:colsSeen]
	return tbl, nil
}

func splitLine(opts *options, line string, ifs string, columns int) *Row {
	isComment := false
	var recs []string
	if opts.enableComments && strings.HasPrefix(line, opts.commentPrefix) {
		recs = []string{line}
		isComment = true
	} else {
		recs = kstrings.SplitNMerged(line, ifs, columns)
	}
	return newRow(recs, isComment)
}
<commit_msg>Use newRow() instead of replicating the functionality badly.<commit_after>/*
Package table provides functionality for holding and describing tabular data.

TODO(2020-03-01) Add support for justification.
- The table should have a default justification for each column.
- Each column should be able to override the default table justification.
*/
package table

import (
	"bytes"
	"fmt"
	"strings"

	"github.com/kward/golib/math"
	kstrings "github.com/kward/golib/strings"
)

const MAX_COLS = 100

// Row describes a row in the table.
type Row struct {
	columns   []*Column // Columnar data of the row.
	sizes     []int     // Sizes of the columns.
	isComment bool
}

// NewRow instantiates a new row. If the row is a comment, there can be only one
// record.
func NewRow(records []string, isComment bool) (*Row, error) {
	if isComment && len(records) > 1 {
		return nil, fmt.Errorf("only one record allowed for a comment row")
	}
	return newRow(records, isComment), nil
}

func newRow(records []string, isComment bool) *Row {
	cols := []*Column{}
	sizes := []int{}
	for _, r := range records {
		cols = append(cols, &Column{cell: r})
		sizes = append(sizes, len(r))
	}
	return &Row{cols, sizes, isComment}
}

// Values returns the cell data for the row.
func (r *Row) Values() []string {
	vs := make([]string, r.NumColumns())
	for i, c := range r.Columns() {
		vs[i] = c.Value()
	}
	return vs
}

// Columns held in the row.
func (r *Row) Columns() []*Column { return r.columns }

// NumColumns returns the number of columns in the row.
func (r *Row) NumColumns() int { return len(r.columns) }

// Sizes of the columns.
func (r *Row) Sizes() []int { return r.sizes }

// IsComment returns true if the full line is a comment.
func (r *Row) IsComment() bool { return r.isComment }

// String implements fmt.Stringer.
func (r *Row) String() string {
	var buf bytes.Buffer
	for i, col := range r.columns {
		if i > 0 {
			buf.WriteString(" ")
		}
		buf.WriteString(fmt.Sprintf("%q", col.String()))
	}
	return buf.String()
}

// Column holds the data for each column.
type Column struct {
	cell string // The actual cell data.
}

// Value of the column.
func (c *Column) Value() string { return c.cell }

// Length of the cell.
func (c *Column) Length() int { return len(c.cell) }

// String implements fmt.Stringer.
func (c *Column) String() string { return c.cell }

type Table struct {
	opts *options

	rows     []*Row
	colSizes []int
}

func NewTable(opts ...func(*options) error) (*Table, error) {
	o := &options{}
	o.setCommentPrefix("#")
	o.setEnableComments(false)
	o.setSectionReset(false)
	for _, opt := range opts {
		if err := opt(o); err != nil {
			return nil, err
		}
	}
	return &Table{
		opts: o,
		rows: []*Row{},
	}, nil
}

// Append lines to the table.
func (t *Table) Append(records ...[]string) {
	for _, rs := range records {
		t.rows = append(t.rows, newRow(rs, false))
	}
}

// ColSizes returns the maximum size of each column.
func (t *Table) ColSizes() []int { return t.colSizes }

// Rows returns the table row data.
func (t *Table) Rows() []*Row { return t.rows }

// NumRows returns the number of rows in the table.
func (t *Table) NumRows() int { return len(t.rows) }

// String implements fmt.Stringer.
func (t *Table) String() string {
	var buf bytes.Buffer
	buf.WriteRune('[')
	for i, row := range t.rows {
		if i > 0 {
			buf.WriteString(", ")
		}
		buf.WriteString(row.String())
	}
	buf.WriteRune(']')
	return fmt.Sprintf("%v sizes: %v\n", buf.String(), t.colSizes)
}

// Split lines of text into a table.
//
// The count `n` determines the number of substrings to return:
//
//     n > 0: at most n columns; the last column will be the unsplit remainder.
//     n == 0: the result is nil (an empty table)
//     n < 0: all columns
func Split(lines []string, ifs string, n int, opts ...func(*options) error) (*Table, error) {
	if n > MAX_COLS {
		return nil, fmt.Errorf("column count %d exceeds supported maximum number of columns %d", n, MAX_COLS)
	}

	tbl, err := NewTable(opts...)
	if err != nil {
		return nil, fmt.Errorf("error instantiating a table; %s", err)
	}

	var sizes []int
	switch {
	case n > 0:
		sizes = make([]int, n)
	case n == 0:
		return tbl, nil
	case n < 0:
		sizes = make([]int, MAX_COLS)
	}

	colsSeen := 0
	rows := []*Row{}
	for _, line := range lines {
		row := splitLine(tbl.opts, line, ifs, n)
		for j, col := range row.Columns() {
			if j >= MAX_COLS-1 {
				j = MAX_COLS - 1
			}
			if !row.IsComment() {
				sizes[j] = math.Max(col.Length(), sizes[j])
			}
		}
		rows = append(rows, row)
		colsSeen = math.Max(row.NumColumns(), colsSeen)
	}

	tbl.rows = rows
	tbl.colSizes = sizes[:colsSeen]
	return tbl, nil
}

func splitLine(opts *options, line string, ifs string, columns int) *Row {
	isComment := false
	var recs []string
	if opts.enableComments && strings.HasPrefix(line, opts.commentPrefix) {
		recs = []string{line}
		isComment = true
	} else {
		recs = kstrings.SplitNMerged(line, ifs, columns)
	}
	return newRow(recs, isComment)
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Mute Communications Ltd.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package protoengine

import (
	"io"
	"io/ioutil"

	"github.com/mutecomm/mute/def"
	"github.com/mutecomm/mute/encode/base64"
	"github.com/mutecomm/mute/log"
	"github.com/mutecomm/mute/mix/client"
)

func (pe *ProtoEngine) create(
	w io.Writer,
	minDelay, maxDelay int32,
	tokenString, nymaddress string,
	r io.Reader,
) error {
	msg, err := ioutil.ReadAll(r)
	if err != nil {
		return log.Error(err)
	}
	message, err := base64.Decode(string(msg))
	if err != nil {
		return log.Error(err)
	}
	token, err := base64.Decode(tokenString)
	if err != nil {
		return log.Error(err)
	}
	na, err := base64.Decode(nymaddress)
	if err != nil {
		return log.Error(err)
	}
	mo := client.MessageInput{
		SenderMinDelay: minDelay,
		SenderMaxDelay: maxDelay,
		Token:          token,
		NymAddress:     na,
		Message:        message,
		SMTPPort:       2025,                          // TODO: allow to set SMTPPort
		SmartHost:      "mix.serviceguard.chavpn.net", // TODO: allow to set SmartHost
		CACert:         def.CACert,
	}.Create()
	if mo.Error != nil {
		return log.Error(mo.Error)
	}
	envelope := mo.Marshal()
	if _, err := io.WriteString(w, base64.Encode(envelope)); err != nil {
		return log.Error(err)
	}
	return nil
}
<commit_msg>disable SmartHost<commit_after>// Copyright (c) 2015 Mute Communications Ltd.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package protoengine

import (
	"io"
	"io/ioutil"

	"github.com/mutecomm/mute/def"
	"github.com/mutecomm/mute/encode/base64"
	"github.com/mutecomm/mute/log"
	"github.com/mutecomm/mute/mix/client"
)

func (pe *ProtoEngine) create(
	w io.Writer,
	minDelay, maxDelay int32,
	tokenString, nymaddress string,
	r io.Reader,
) error {
	msg, err := ioutil.ReadAll(r)
	if err != nil {
		return log.Error(err)
	}
	message, err := base64.Decode(string(msg))
	if err != nil {
		return log.Error(err)
	}
	token, err := base64.Decode(tokenString)
	if err != nil {
		return log.Error(err)
	}
	na, err := base64.Decode(nymaddress)
	if err != nil {
		return log.Error(err)
	}
	mo := client.MessageInput{
		SenderMinDelay: minDelay,
		SenderMaxDelay: maxDelay,
		Token:          token,
		NymAddress:     na,
		Message:        message,
		//SMTPPort:       2025,                          // TODO: allow to set SMTPPort
		//SmartHost:      "mix.serviceguard.chavpn.net", // TODO: allow to set SmartHost
		CACert: def.CACert,
	}.Create()
	if mo.Error != nil {
		return log.Error(mo.Error)
	}
	envelope := mo.Marshal()
	if _, err := io.WriteString(w, base64.Encode(envelope)); err != nil {
		return log.Error(err)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"github.com/butlermatt/dsdoc/parser"
	"strings"
)

var buf bytes.Buffer
var tree bytes.Buffer

func genText(doc *parser.Document) bytes.Buffer {
	walkTextDoc(doc, "")
	tree.WriteString("\n---\n\n")
	tree.WriteString(buf.String())
	return tree
}

func walkTextDoc(doc *parser.Document, sep string) {
	buf.WriteString(fmt.Sprintln("Name:", doc.Name))
	buf.WriteString(fmt.Sprint("\n", doc.Short, "\n\n"))
	buf.WriteString(fmt.Sprintln("Type:", doc.Type))
	if doc.Is != "" {
		buf.WriteString(fmt.Sprintln("$is:", doc.Is))
	}
	if doc.ParentName != "" {
		buf.WriteString(fmt.Sprintln("Parent:", doc.Parent.Name))
	}
	if doc.Long != "" {
		buf.WriteString(fmt.Sprint("Description:\n", doc.Long, "\n\n"))
	}

	if doc.Type == parser.ActionDoc {
		if len(doc.Params) > 0 {
			buf.WriteString("Params:\n")
			for _, p := range doc.Params {
				buf.WriteString(fmt.Sprintln("     Name:", p.Name))
				buf.WriteString(fmt.Sprintln("     Type:", p.Type))
				buf.WriteString(fmt.Sprintln("    ", p.Description))
				buf.WriteRune('\n')
			}
			buf.WriteRune('\n')
		}

		buf.WriteString(fmt.Sprintln("Return type:", doc.Return))
		if len(doc.Columns) > 0 {
			buf.WriteString("Columns:\n")
			for _, p := range doc.Columns {
				buf.WriteString(fmt.Sprintln("     Name:", p.Name))
				buf.WriteString(fmt.Sprintln("     Type:", p.Type))
				buf.WriteString(fmt.Sprintln("    ", p.Description))
				buf.WriteRune('\n')
			}
		}
	}

	if doc.ValueType != "" {
		buf.WriteString(fmt.Sprintln("Value Type:", doc.ValueType, "  "))
		buf.WriteString(fmt.Sprintln("Writable:", doc.Writable, "  "))
	}
	buf.WriteString("\n---\n\n")

	if (doc.Type == parser.ActionDoc) {
		var args string
		var params []string
		for _, a := range doc.Params {
			params = append(params, a.Name)
		}
		args = strings.Join(params, ", ")
		tree.WriteString(fmt.Sprintf("%s- @%s(%s)\n", sep, doc.Name, args))
	} else {
		tree.WriteString(fmt.Sprintf("%s- %s\n", sep, doc.Name))
	}
	if len(doc.Children) > 0 {
		for _, ch := range doc.Children {
			walkTextDoc(ch, sep+" |")
		}
	}
}

func genMarkdown(doc *parser.Document) bytes.Buffer {
	tree.WriteString("```\n")
	walkMdDoc(doc, "")
	tree.WriteString("```\n\n---\n\n")
	tree.WriteString(buf.String())
	return tree
}

func walkMdDoc(doc *parser.Document, sep string) {
	buf.WriteString(fmt.Sprint("### ", doc.Name, "  \n\n"))
	buf.WriteString(fmt.Sprint(doc.Short, "  \n\n"))
	buf.WriteString(fmt.Sprintln("Type:", doc.Type, "  "))
	if doc.Is != "" {
		buf.WriteString(fmt.Sprintln("$is:", doc.Is, "  "))
	}
	if doc.ParentName != "" {
		buf.WriteString(fmt.Sprintf("Parent: [%s](#%s)  \n", doc.Parent.Name, strings.ToLower(doc.Parent.Name)))
	}
	if doc.Long != "" {
		buf.WriteString(fmt.Sprint("\nDescription:  \n", doc.Long, "  \n\n"))
	}

	if doc.Type == parser.ActionDoc {
		if len(doc.Params) > 0 {
			buf.WriteString("Params:  \n\n")
			buf.WriteString("Name | Type | Description\n")
			buf.WriteString("--- | --- | ---\n")
			for _, p := range doc.Params {
				buf.WriteString(fmt.Sprintf("%s | `%s` | %s\n", p.Name, p.Type, p.Description))
			}
			buf.WriteString("\n")
		}

		buf.WriteString(fmt.Sprintln("Return type:", doc.Return, "  "))
		if len(doc.Columns) > 0 {
			buf.WriteString("Columns:  \n\n")
			buf.WriteString("Name | Type | Description\n")
			buf.WriteString("--- | --- | ---\n")
			for _, p := range doc.Columns {
				buf.WriteString(fmt.Sprintf("%s | `%s` | %s \n", p.Name, p.Type, p.Description))
			}
		}
	}

	if doc.ValueType != "" {
		buf.WriteString(fmt.Sprintf("Value Type: `%s`  \n", doc.ValueType))
		buf.WriteString(fmt.Sprintf("Writable: `%s`  \n", doc.Writable))
	}
	buf.WriteString("\n---\n\n")

	if (doc.Type == parser.ActionDoc) {
		var args string
		var params []string
		for _, a := range doc.Params {
			params = append(params, a.Name)
		}
		args = strings.Join(params, ", ")
		tree.WriteString(fmt.Sprintf("%s- @%s(%s)\n", sep, doc.Name, args))
	} else {
		tree.WriteString(fmt.Sprintf("%s- %s\n", sep, doc.Name))
	}
	if len(doc.Children) > 0 {
		for _, ch := range doc.Children {
			walkMdDoc(ch, sep+" |")
		}
	}
}
<commit_msg>Add links in tree to each description.<commit_after>package main

import (
	"bytes"
	"fmt"
	"github.com/butlermatt/dsdoc/parser"
	"strings"
)

var buf bytes.Buffer
var tree bytes.Buffer

func genText(doc *parser.Document) bytes.Buffer {
	walkTextDoc(doc, "")
	tree.WriteString("\n---\n\n")
	tree.WriteString(buf.String())
	return tree
}

func walkTextDoc(doc *parser.Document, sep string) {
	buf.WriteString(fmt.Sprintln("Name:", doc.Name))
	buf.WriteString(fmt.Sprint("\n", doc.Short, "\n\n"))
	buf.WriteString(fmt.Sprintln("Type:", doc.Type))
	if doc.Is != "" {
		buf.WriteString(fmt.Sprintln("$is:", doc.Is))
	}
	if doc.ParentName != "" {
		buf.WriteString(fmt.Sprintln("Parent:", doc.Parent.Name))
	}
	if doc.Long != "" {
		buf.WriteString(fmt.Sprint("Description:\n", doc.Long, "\n\n"))
	}

	if doc.Type == parser.ActionDoc {
		if len(doc.Params) > 0 {
			buf.WriteString("Params:\n")
			for _, p := range doc.Params {
				buf.WriteString(fmt.Sprintln("     Name:", p.Name))
				buf.WriteString(fmt.Sprintln("     Type:", p.Type))
				buf.WriteString(fmt.Sprintln("    ", p.Description))
				buf.WriteRune('\n')
			}
			buf.WriteRune('\n')
		}

		buf.WriteString(fmt.Sprintln("Return type:", doc.Return))
		if len(doc.Columns) > 0 {
			buf.WriteString("Columns:\n")
			for _, p := range doc.Columns {
				buf.WriteString(fmt.Sprintln("     Name:", p.Name))
				buf.WriteString(fmt.Sprintln("     Type:", p.Type))
				buf.WriteString(fmt.Sprintln("    ", p.Description))
				buf.WriteRune('\n')
			}
		}
	}

	if doc.ValueType != "" {
		buf.WriteString(fmt.Sprintln("Value Type:", doc.ValueType, "  "))
		buf.WriteString(fmt.Sprintln("Writable:", doc.Writable, "  "))
	}
	buf.WriteString("\n---\n\n")

	if (doc.Type == parser.ActionDoc) {
		var args string
		var params []string
		for _, a := range doc.Params {
			params = append(params, a.Name)
		}
		args = strings.Join(params, ", ")
		tree.WriteString(fmt.Sprintf("%s- @%s(%s)\n", sep, doc.Name, args))
	} else {
		var vType string
		if doc.ValueType != "" {
			vType = fmt.Sprintf(" *%s (%s)*", doc.ValueType, doc.Writable)
		}
		tree.WriteString(fmt.Sprintf("%s- %s%s\n", sep, doc.Name, vType))
	}
	if len(doc.Children) > 0 {
		for _, ch := range doc.Children {
			walkTextDoc(ch, sep+" |")
		}
	}
}

func genMarkdown(doc *parser.Document) bytes.Buffer {
	tree.WriteString(" <pre>\n")
	walkMdDoc(doc, "")
	tree.WriteString(" </pre>\n\n---\n\n")
	tree.WriteString(buf.String())
	return tree
}

func walkMdDoc(doc *parser.Document, sep string) {
	buf.WriteString(fmt.Sprint("### ", doc.Name, "  \n\n"))
	buf.WriteString(fmt.Sprint(doc.Short, "  \n\n"))
	buf.WriteString(fmt.Sprintln("Type:", doc.Type, "  "))
	if doc.Is != "" {
		buf.WriteString(fmt.Sprintln("$is:", doc.Is, "  "))
	}
	if doc.ParentName != "" {
		buf.WriteString(fmt.Sprintf("Parent: [%s](#%s)  \n", doc.Parent.Name, strings.ToLower(doc.Parent.Name)))
	}
	if doc.Long != "" {
		buf.WriteString(fmt.Sprint("\nDescription:  \n", doc.Long, "  \n\n"))
	}

	if doc.Type == parser.ActionDoc {
		if len(doc.Params) > 0 {
			buf.WriteString("Params:  \n\n")
			buf.WriteString("Name | Type | Description\n")
			buf.WriteString("--- | --- | ---\n")
			for _, p := range doc.Params {
				buf.WriteString(fmt.Sprintf("%s | `%s` | %s\n", p.Name, p.Type, p.Description))
			}
			buf.WriteString("\n")
		}

		buf.WriteString(fmt.Sprintln("Return type:", doc.Return, "  "))
		if len(doc.Columns) > 0 {
			buf.WriteString("Columns:  \n\n")
			buf.WriteString("Name | Type | Description\n")
			buf.WriteString("--- | --- | ---\n")
			for _, p := range doc.Columns {
				buf.WriteString(fmt.Sprintf("%s | `%s` | %s \n", p.Name, p.Type, p.Description))
			}
		}
	}

	if doc.ValueType != "" {
		buf.WriteString(fmt.Sprintf("Value Type: `%s`  \n", doc.ValueType))
		buf.WriteString(fmt.Sprintf("Writable: `%s`  \n", doc.Writable))
	}
	buf.WriteString("\n---\n\n")

	if (doc.Type == parser.ActionDoc) {
		var args string
		var params []string
		for _, a := range doc.Params {
			params = append(params, a.Name)
		}
		args = strings.Join(params, ", ")
		tree.WriteString(fmt.Sprintf("%s-[@%s(%s)](#%s)\n", sep, doc.Name, args, strings.ToLower(doc.Name)))
	} else {
		var vType string
		if doc.ValueType != "" {
			vType = fmt.Sprintf(" - %s", doc.ValueType)
		}
		tree.WriteString(fmt.Sprintf("%s-[%s](#%s)%s\n", sep, doc.Name, strings.ToLower(doc.Name), vType))
	}
	if len(doc.Children) > 0 {
		for _, ch := range doc.Children {
			walkMdDoc(ch, sep+" |")
		}
	}
}
<|endoftext|>


<commit_before>//go:generate templates -s templates -o templates/templates.go
package schematic

import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/format"
	"os"
	"strings"
	"text/template"

	bundle "github.com/interagent/schematic/templates"
)

var templates *template.Template

func init() {
	templates = template.New("package.tmpl").Funcs(helpers)
	templates = template.Must(bundle.Parse(templates))
}

// ResolvedSet stores a set of pointers to objects that have already been
// resolved to prevent infinite loops.
type ResolvedSet map[interface{}]bool

// Insert marks a pointer as resolved.
func (rs ResolvedSet) Insert(o interface{}) {
	rs[o] = true
}

// Has returns if a pointer has already been resolved.
func (rs ResolvedSet) Has(o interface{}) bool {
	return rs[o]
}

// Generate generates code according to the schema.
func (s *Schema) Generate() ([]byte, error) {
	var buf bytes.Buffer

	s = s.Resolve(nil, ResolvedSet{})

	name := strings.ToLower(strings.Split(s.Title, " ")[0])
	templates.ExecuteTemplate(&buf, "package.tmpl", name)

	// TODO: Check if we need time.
	templates.ExecuteTemplate(&buf, "imports.tmpl", []string{
		"encoding/json", "fmt", "io", "reflect", "net/http", "runtime",
		"time", "bytes", "context", "strings",
		"github.com/google/go-querystring/query",
	})
	templates.ExecuteTemplate(&buf, "service.tmpl", struct {
		Name    string
		URL     string
		Version string
	}{
		Name:    name,
		URL:     s.URL(),
		Version: s.Version,
	})

	for _, name := range sortedKeys(s.Properties) {
		schema := s.Properties[name]
		// Skipping definitions because there is no links, nor properties.
		if schema.Links == nil && schema.Properties == nil {
			continue
		}

		context := struct {
			Name       string
			Definition *Schema
		}{
			Name:       name,
			Definition: schema,
		}

		if !context.Definition.AreTitleLinksUnique() {
			return nil, fmt.Errorf("duplicate titles detected for %s", context.Name)
		}

		templates.ExecuteTemplate(&buf, "struct.tmpl", context)
		templates.ExecuteTemplate(&buf, "funcs.tmpl", context)
	}

	// Remove blank lines added by text/template
	bytes := newlines.ReplaceAll(buf.Bytes(), []byte(""))

	// Format sources
	clean, err := format.Source(bytes)
	if err != nil {
		return buf.Bytes(), err
	}
	return clean, nil
}

// Resolve resolves reference inside the schema.
func (s *Schema) Resolve(r *Schema, rs ResolvedSet) *Schema {
	if r == nil {
		r = s
	}

	for {
		if s.Ref != nil {
			s = s.Ref.Resolve(r)
		} else if len(s.OneOf) > 0 {
			s = s.OneOf[0].Ref.Resolve(r)
		} else if len(s.AnyOf) > 0 {
			s = s.AnyOf[0].Ref.Resolve(r)
		} else {
			break
		}
	}

	if rs.Has(s) {
		// Already resolved
		return s
	}
	rs.Insert(s)

	for n, d := range s.Definitions {
		s.Definitions[n] = d.Resolve(r, rs)
	}
	for n, p := range s.Properties {
		s.Properties[n] = p.Resolve(r, rs)
	}
	for n, p := range s.PatternProperties {
		s.PatternProperties[n] = p.Resolve(r, rs)
	}
	if s.Items != nil {
		s.Items = s.Items.Resolve(r, rs)
	}
	for _, l := range s.Links {
		l.Resolve(r, rs)
	}
	return s
}

// Types returns the array of types described by this schema.
func (s *Schema) Types() (types []string, err error) {
	if arr, ok := s.Type.([]interface{}); ok {
		for _, v := range arr {
			types = append(types, v.(string))
		}
	} else if str, ok := s.Type.(string); ok {
		types = append(types, str)
	} else {
		err = fmt.Errorf("unknown type %v", s.Type)
	}
	return types, err
}

// GoType returns the Go type for the given schema as string.
func (s *Schema) GoType() string {
	return s.goType(true, true)
}

// IsCustomType returns true if the schema declares a custom type.
func (s *Schema) IsCustomType() bool {
	return len(s.Properties) > 0
}

func (s *Schema) goType(required bool, force bool) (goType string) {
	// Resolve JSON reference/pointer
	types, err := s.Types()
	if err != nil {
		fail(s, err)
	}
	for _, kind := range types {
		switch kind {
		case "boolean":
			goType = "bool"
		case "string":
			switch s.Format {
			case "date-time":
				goType = "time.Time"
			default:
				goType = "string"
			}
		case "number":
			goType = "float64"
		case "integer":
			goType = "int"
		case "any":
			goType = "interface{}"
		case "array":
			if s.Items != nil {
				goType = "[]" + s.Items.goType(required, force)
			} else {
				goType = "[]interface{}"
			}
		case "object":
			// Check if patternProperties exists.
			if s.PatternProperties != nil {
				for _, prop := range s.PatternProperties {
					goType = fmt.Sprintf("map[string]%s", prop.GoType())
					break // We don't support more than one pattern for now.
				}
				continue
			}
			buf := bytes.NewBufferString("struct {")
			for _, name := range sortedKeys(s.Properties) {
				prop := s.Properties[name]
				req := contains(name, s.Required) || force
				templates.ExecuteTemplate(buf, "field.tmpl", struct {
					Definition *Schema
					Name       string
					Required   bool
					Type       string
				}{
					Definition: prop,
					Name:       name,
					Required:   req,
					Type:       prop.goType(req, force),
				})
			}
			buf.WriteString("}")
			goType = buf.String()
		case "null":
			continue
		default:
			fail(s, fmt.Errorf("unknown type %s", kind))
		}
	}
	if goType == "" {
		fail(s, fmt.Errorf("type not found : %s", types))
	}
	// Types allow null
	if contains("null", types) || !(required || force) {
		// Don't need a pointer for these types to be "nilable"
		if goType != "interface{}" && !strings.HasPrefix(goType, "[]") && !strings.HasPrefix(goType, "map[") {
			return "*" + goType
		}
	}
	return goType
}

// Values returns function return values types.
func (s *Schema) Values(name string, l *Link) []string {
	var values []string
	name = returnType(name, s, l)
	if s.EmptyResult(l) {
		values = append(values, "error")
	} else if s.ReturnsCustomType(l) {
		values = append(values, fmt.Sprintf("*%s", name), "error")
	} else {
		values = append(values, name, "error")
	}
	return values
}

// AreTitleLinksUnique ensures that all titles are unique for a schema.
//
// If more than one link in a given schema has the same title, we cannot
// accurately generate the client from the schema. Although it's not strictly a
// schema violation, it needs to be fixed before the client can be properly
// generated.
func (s *Schema) AreTitleLinksUnique() bool {
	titles := map[string]bool{}
	var uniqueLinks []*Link
	for _, link := range s.Links {
		title := strings.ToLower(link.Title)
		if _, ok := titles[title]; !ok {
			uniqueLinks = append(uniqueLinks, link)
		}
		titles[title] = true
	}

	return len(uniqueLinks) == len(s.Links)
}

// URL returns schema base URL.
func (s *Schema) URL() string {
	for _, l := range s.Links {
		if l.Rel == "self" {
			return l.HRef.String()
		}
	}
	return ""
}

// ReturnsCustomType returns true if the link returns a custom type.
func (s *Schema) ReturnsCustomType(l *Link) bool {
	if l.TargetSchema != nil {
		return len(l.TargetSchema.Properties) > 0
	}
	return len(s.Properties) > 0
}

// ReturnedGoType returns Go type returned by the given link as a string.
func (s *Schema) ReturnedGoType(name string, l *Link) string {
	if l.TargetSchema != nil {
		if l.TargetSchema.Items == s {
			return "[]" + initialCap(name)
		}
		return l.TargetSchema.goType(true, true)
	}
	return s.goType(true, true)
}

// EmptyResult retursn true if the link result should be empty.
func (s *Schema) EmptyResult(l *Link) bool {
	var (
		types []string
		err   error
	)
	if l.TargetSchema != nil {
		types, err = l.TargetSchema.Types()
	} else {
		types, err = s.Types()
	}
	if err != nil {
		return true
	}
	return len(types) == 1 && types[0] == "null"
}

// Parameters returns function parameters names and types.
func (l *Link) Parameters(name string) ([]string, map[string]string) {
	if l.HRef == nil {
		// No HRef property
		panic(fmt.Errorf("no href property declared for %s", l.Title))
	}
	var order []string
	params := make(map[string]string)
	for _, name := range l.HRef.Order {
		def := l.HRef.Schemas[name]
		order = append(order, name)
		params[name] = def.GoType()
	}
	if l.Schema != nil {
		order = append(order, "o")
		t, required := l.GoType()
		if l.AcceptsCustomType() {
			params["o"] = paramType(name, l)
		} else {
			params["o"] = t
		}
		if !required {
			params["o"] = "*" + params["o"]
		}
	}
	if l.Rel == "instances" && strings.ToUpper(l.Method) == "GET" {
		order = append(order, "lr")
		params["lr"] = "*ListRange"
	}
	return order, params
}

// AcceptsCustomType returns true if the link schema is not a primitive type
func (l *Link) AcceptsCustomType() bool {
	if l.Schema != nil && l.Schema.IsCustomType() {
		return true
	}
	return false
}

// Resolve resolve link schema and href.
func (l *Link) Resolve(r *Schema, rs ResolvedSet) {
	if l.Schema != nil {
		l.Schema = l.Schema.Resolve(r, rs)
	}
	if l.TargetSchema != nil {
		l.TargetSchema = l.TargetSchema.Resolve(r, rs)
	}
	l.HRef.Resolve(r, rs)
}

// GoType returns Go type for the given schema as string and a bool specifying whether it is required
func (l *Link) GoType() (string, bool) {
	t := l.Schema.goType(true, false)
	if t[0] == '*' {
		return t[1:], false
	}
	return t, true
}

func fail(v interface{}, err error) {
	el, _ := json.MarshalIndent(v, "    ", "  ")

	fmt.Printf(
		"Error processing schema element:\n    %s\n\nFailed with: %s\n",
		el,
		err,
	)
	os.Exit(1)
}
<commit_msg>Send error messages to STDERR<commit_after>//go:generate templates -s templates -o templates/templates.go
package schematic

import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/format"
	"os"
	"strings"
	"text/template"

	bundle "github.com/interagent/schematic/templates"
)

var templates *template.Template

func init() {
	templates = template.New("package.tmpl").Funcs(helpers)
	templates = template.Must(bundle.Parse(templates))
}

// ResolvedSet stores a set of pointers to objects that have already been
// resolved to prevent infinite loops.
type ResolvedSet map[interface{}]bool

// Insert marks a pointer as resolved.
func (rs ResolvedSet) Insert(o interface{}) {
	rs[o] = true
}

// Has returns if a pointer has already been resolved.
func (rs ResolvedSet) Has(o interface{}) bool {
	return rs[o]
}

// Generate generates code according to the schema.
func (s *Schema) Generate() ([]byte, error) {
	var buf bytes.Buffer

	s = s.Resolve(nil, ResolvedSet{})

	name := strings.ToLower(strings.Split(s.Title, " ")[0])
	templates.ExecuteTemplate(&buf, "package.tmpl", name)

	// TODO: Check if we need time.
	templates.ExecuteTemplate(&buf, "imports.tmpl", []string{
		"encoding/json", "fmt", "io", "reflect", "net/http", "runtime",
		"time", "bytes", "context", "strings",
		"github.com/google/go-querystring/query",
	})
	templates.ExecuteTemplate(&buf, "service.tmpl", struct {
		Name    string
		URL     string
		Version string
	}{
		Name:    name,
		URL:     s.URL(),
		Version: s.Version,
	})

	for _, name := range sortedKeys(s.Properties) {
		schema := s.Properties[name]
		// Skipping definitions because there is no links, nor properties.
		if schema.Links == nil && schema.Properties == nil {
			continue
		}

		context := struct {
			Name       string
			Definition *Schema
		}{
			Name:       name,
			Definition: schema,
		}

		if !context.Definition.AreTitleLinksUnique() {
			return nil, fmt.Errorf("duplicate titles detected for %s", context.Name)
		}

		templates.ExecuteTemplate(&buf, "struct.tmpl", context)
		templates.ExecuteTemplate(&buf, "funcs.tmpl", context)
	}

	// Remove blank lines added by text/template
	bytes := newlines.ReplaceAll(buf.Bytes(), []byte(""))

	// Format sources
	clean, err := format.Source(bytes)
	if err != nil {
		return buf.Bytes(), err
	}
	return clean, nil
}

// Resolve resolves reference inside the schema.
func (s *Schema) Resolve(r *Schema, rs ResolvedSet) *Schema {
	if r == nil {
		r = s
	}

	for {
		if s.Ref != nil {
			s = s.Ref.Resolve(r)
		} else if len(s.OneOf) > 0 {
			s = s.OneOf[0].Ref.Resolve(r)
		} else if len(s.AnyOf) > 0 {
			s = s.AnyOf[0].Ref.Resolve(r)
		} else {
			break
		}
	}

	if rs.Has(s) {
		// Already resolved
		return s
	}
	rs.Insert(s)

	for n, d := range s.Definitions {
		s.Definitions[n] = d.Resolve(r, rs)
	}
	for n, p := range s.Properties {
		s.Properties[n] = p.Resolve(r, rs)
	}
	for n, p := range s.PatternProperties {
		s.PatternProperties[n] = p.Resolve(r, rs)
	}
	if s.Items != nil {
		s.Items = s.Items.Resolve(r, rs)
	}
	for _, l := range s.Links {
		l.Resolve(r, rs)
	}
	return s
}

// Types returns the array of types described by this schema.
func (s *Schema) Types() (types []string, err error) {
	if arr, ok := s.Type.([]interface{}); ok {
		for _, v := range arr {
			types = append(types, v.(string))
		}
	} else if str, ok := s.Type.(string); ok {
		types = append(types, str)
	} else {
		err = fmt.Errorf("unknown type %v", s.Type)
	}
	return types, err
}

// GoType returns the Go type for the given schema as string.
func (s *Schema) GoType() string {
	return s.goType(true, true)
}

// IsCustomType returns true if the schema declares a custom type.
func (s *Schema) IsCustomType() bool {
	return len(s.Properties) > 0
}

func (s *Schema) goType(required bool, force bool) (goType string) {
	// Resolve JSON reference/pointer
	types, err := s.Types()
	if err != nil {
		fail(s, err)
	}
	for _, kind := range types {
		switch kind {
		case "boolean":
			goType = "bool"
		case "string":
			switch s.Format {
			case "date-time":
				goType = "time.Time"
			default:
				goType = "string"
			}
		case "number":
			goType = "float64"
		case "integer":
			goType = "int"
		case "any":
			goType = "interface{}"
		case "array":
			if s.Items != nil {
				goType = "[]" + s.Items.goType(required, force)
			} else {
				goType = "[]interface{}"
			}
		case "object":
			// Check if patternProperties exists.
			if s.PatternProperties != nil {
				for _, prop := range s.PatternProperties {
					goType = fmt.Sprintf("map[string]%s", prop.GoType())
					break // We don't support more than one pattern for now.
				}
				continue
			}
			buf := bytes.NewBufferString("struct {")
			for _, name := range sortedKeys(s.Properties) {
				prop := s.Properties[name]
				req := contains(name, s.Required) || force
				templates.ExecuteTemplate(buf, "field.tmpl", struct {
					Definition *Schema
					Name       string
					Required   bool
					Type       string
				}{
					Definition: prop,
					Name:       name,
					Required:   req,
					Type:       prop.goType(req, force),
				})
			}
			buf.WriteString("}")
			goType = buf.String()
		case "null":
			continue
		default:
			fail(s, fmt.Errorf("unknown type %s", kind))
		}
	}
	if goType == "" {
		fail(s, fmt.Errorf("type not found : %s", types))
	}
	// Types allow null
	if contains("null", types) || !(required || force) {
		// Don't need a pointer for these types to be "nilable"
		if goType != "interface{}" && !strings.HasPrefix(goType, "[]") && !strings.HasPrefix(goType, "map[") {
			return "*" + goType
		}
	}
	return goType
}

// Values returns function return values types.
func (s *Schema) Values(name string, l *Link) []string {
	var values []string
	name = returnType(name, s, l)
	if s.EmptyResult(l) {
		values = append(values, "error")
	} else if s.ReturnsCustomType(l) {
		values = append(values, fmt.Sprintf("*%s", name), "error")
	} else {
		values = append(values, name, "error")
	}
	return values
}

// AreTitleLinksUnique ensures that all titles are unique for a schema.
//
// If more than one link in a given schema has the same title, we cannot
// accurately generate the client from the schema. Although it's not strictly a
// schema violation, it needs to be fixed before the client can be properly
// generated.
func (s *Schema) AreTitleLinksUnique() bool {
	titles := map[string]bool{}
	var uniqueLinks []*Link
	for _, link := range s.Links {
		title := strings.ToLower(link.Title)
		if _, ok := titles[title]; !ok {
			uniqueLinks = append(uniqueLinks, link)
		}
		titles[title] = true
	}

	return len(uniqueLinks) == len(s.Links)
}

// URL returns schema base URL.
func (s *Schema) URL() string {
	for _, l := range s.Links {
		if l.Rel == "self" {
			return l.HRef.String()
		}
	}
	return ""
}

// ReturnsCustomType returns true if the link returns a custom type.
func (s *Schema) ReturnsCustomType(l *Link) bool {
	if l.TargetSchema != nil {
		return len(l.TargetSchema.Properties) > 0
	}
	return len(s.Properties) > 0
}

// ReturnedGoType returns Go type returned by the given link as a string.
func (s *Schema) ReturnedGoType(name string, l *Link) string {
	if l.TargetSchema != nil {
		if l.TargetSchema.Items == s {
			return "[]" + initialCap(name)
		}
		return l.TargetSchema.goType(true, true)
	}
	return s.goType(true, true)
}

// EmptyResult retursn true if the link result should be empty.
func (s *Schema) EmptyResult(l *Link) bool {
	var (
		types []string
		err   error
	)
	if l.TargetSchema != nil {
		types, err = l.TargetSchema.Types()
	} else {
		types, err = s.Types()
	}
	if err != nil {
		return true
	}
	return len(types) == 1 && types[0] == "null"
}

// Parameters returns function parameters names and types.
func (l *Link) Parameters(name string) ([]string, map[string]string) {
	if l.HRef == nil {
		// No HRef property
		panic(fmt.Errorf("no href property declared for %s", l.Title))
	}
	var order []string
	params := make(map[string]string)
	for _, name := range l.HRef.Order {
		def := l.HRef.Schemas[name]
		order = append(order, name)
		params[name] = def.GoType()
	}
	if l.Schema != nil {
		order = append(order, "o")
		t, required := l.GoType()
		if l.AcceptsCustomType() {
			params["o"] = paramType(name, l)
		} else {
			params["o"] = t
		}
		if !required {
			params["o"] = "*" + params["o"]
		}
	}
	if l.Rel == "instances" && strings.ToUpper(l.Method) == "GET" {
		order = append(order, "lr")
		params["lr"] = "*ListRange"
	}
	return order, params
}

// AcceptsCustomType returns true if the link schema is not a primitive type
func (l *Link) AcceptsCustomType() bool {
	if l.Schema != nil && l.Schema.IsCustomType() {
		return true
	}
	return false
}

// Resolve resolve link schema and href.
func (l *Link) Resolve(r *Schema, rs ResolvedSet) {
	if l.Schema != nil {
		l.Schema = l.Schema.Resolve(r, rs)
	}
	if l.TargetSchema != nil {
		l.TargetSchema = l.TargetSchema.Resolve(r, rs)
	}
	l.HRef.Resolve(r, rs)
}

// GoType returns Go type for the given schema as string and a bool specifying whether it is required
func (l *Link) GoType() (string, bool) {
	t := l.Schema.goType(true, false)
	if t[0] == '*' {
		return t[1:], false
	}
	return t, true
}

func fail(v interface{}, err error) {
	el, _ := json.MarshalIndent(v, "    ", "  ")

	fmt.Fprintf(
		os.Stderr,
		"Error processing schema element:\n    %s\n\nFailed with: %s\n",
		el,
		err,
	)
	os.Exit(1)
}
<|endoftext|>


<commit_before>// Copyright 2015 The go-python Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"go/ast"
	"go/doc"
	"go/parser"
	"go/token"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"golang.org/x/tools/go/packages"

	"github.com/go-python/gopy/bind"
)

// argStr returns the full command args as a string, without path to exe
func argStr() string {
	ma := make([]string, len(os.Args))
	copy(ma, os.Args)
	_, cmd := filepath.Split(ma[0])
	ma[0] = cmd
	for i := range ma {
		if strings.HasPrefix(ma[i], "-main=") {
			ma[i] = "-main=\"" + ma[i][6:] + "\""
		}
	}
	return strings.Join(ma, " ")
}

// genOutDir makes the output directory and returns its absolute path
func genOutDir(odir string) (string, error) {
	cwd, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}

	if odir == "" {
		odir = cwd
	} else {
		err = os.MkdirAll(odir, 0755)
		if err != nil {
			return odir, fmt.Errorf("gopy-gen: could not create output directory: %v", err)
		}
	}
	odir, err = filepath.Abs(odir)
	if err != nil {
		return odir, fmt.Errorf("gopy-gen: could not infer absolute path to output directory: %v", err)
	}
	return odir, nil
}

// genPkg generates output for all the current packages that have been parsed,
// in the given output directory
// mode = gen, build, pkg, exe
func genPkg(mode bind.BuildMode, cfg *BuildCfg) error {
	var err error
	cfg.OutputDir, err = genOutDir(cfg.OutputDir)
	if err != nil {
		return err
	}
	if !filepath.IsAbs(cfg.VM) {
		cfg.VM, err = exec.LookPath(cfg.VM)
		if err != nil {
			return errors.Wrapf(err, "could not locate absolute path to python VM")
		}
	}

	pyvers, err := getPythonVersion(cfg.VM)
	if err != nil {
		return err
	}
	err = bind.GenPyBind(mode, libExt, extraGccArgs, pyvers, cfg.DynamicLinking, &cfg.BindCfg)
	if err != nil {
		log.Println(err)
	}
	return err
}

func loadPackage(path string, buildFirst bool, buildTags string) (*packages.Package, error) {
	cwd, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	if buildFirst {
		args := []string{"build", "-v", "path"}
		if buildTags != "" {
			args = append(args, "-tags", buildTags)
		}
		fmt.Printf("go %v\n", strings.Join(args, " "))
		cmd := exec.Command("go", args...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Dir = cwd

		err = cmd.Run()
		if err != nil {
			log.Printf("Note: there was an error building [%s] -- will continue but it may fail later: %v\n",
				path,
				err,
			)
		}
	}

	// golang.org/x/tools/go/packages supports modules or GOPATH etc
	bpkgs, err := packages.Load(&packages.Config{Mode: packages.LoadTypes}, path)
	if err != nil {
		log.Printf("error resolving import path [%s]: %v\n",
			path,
			err,
		)
		return nil, err
	}

	bpkg := bpkgs[0] // only ever have one at a time
	return bpkg, nil
}

func parsePackage(bpkg *packages.Package) (*bind.Package, error) {
	if len(bpkg.GoFiles) == 0 {
		err := fmt.Errorf("gopy: no files in package %q", bpkg.PkgPath)
		fmt.Println(err)
		return nil, err
	}
	dir, _ := filepath.Split(bpkg.GoFiles[0])
	p := bpkg.Types

	if bpkg.Name == "main" {
		err := fmt.Errorf("gopy: skipping 'main' package %q", bpkg.PkgPath)
		fmt.Println(err)
		return nil, err
	}

	fset := token.NewFileSet()
	var pkgast *ast.Package
	pkgs, err := parser.ParseDir(fset, dir, nil, parser.ParseComments)
	if err != nil {
		return nil, err
	}
	pkgast = pkgs[p.Name()]
	if pkgast == nil {
		return nil, fmt.Errorf("gopy: could not find AST for package %q", p.Name())
	}

	pkgdoc := doc.New(pkgast, bpkg.PkgPath, 0)
	return bind.NewPackage(p, pkgdoc)
}
<commit_msg>Fix<commit_after>// Copyright 2015 The go-python Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"go/ast"
	"go/doc"
	"go/parser"
	"go/token"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"golang.org/x/tools/go/packages"

	"github.com/go-python/gopy/bind"
)

// argStr returns the full command args as a string, without path to exe
func argStr() string {
	ma := make([]string, len(os.Args))
	copy(ma, os.Args)
	_, cmd := filepath.Split(ma[0])
	ma[0] = cmd
	for i := range ma {
		if strings.HasPrefix(ma[i], "-main=") {
			ma[i] = "-main=\"" + ma[i][6:] + "\""
		}
	}
	return strings.Join(ma, " ")
}

// genOutDir makes the output directory and returns its absolute path
func genOutDir(odir string) (string, error) {
	cwd, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}

	if odir == "" {
		odir = cwd
	} else {
		err = os.MkdirAll(odir, 0755)
		if err != nil {
			return odir, fmt.Errorf("gopy-gen: could not create output directory: %v", err)
		}
	}
	odir, err = filepath.Abs(odir)
	if err != nil {
		return odir, fmt.Errorf("gopy-gen: could not infer absolute path to output directory: %v", err)
	}
	return odir, nil
}

// genPkg generates output for all the current packages that have been parsed,
// in the given output directory
// mode = gen, build, pkg, exe
func genPkg(mode bind.BuildMode, cfg *BuildCfg) error {
	var err error
	cfg.OutputDir, err = genOutDir(cfg.OutputDir)
	if err != nil {
		return err
	}
	if !filepath.IsAbs(cfg.VM) {
		cfg.VM, err = exec.LookPath(cfg.VM)
		if err != nil {
			return errors.Wrapf(err, "could not locate absolute path to python VM")
		}
	}

	pyvers, err := getPythonVersion(cfg.VM)
	if err != nil {
		return err
	}
	err = bind.GenPyBind(mode, libExt, extraGccArgs, pyvers, cfg.DynamicLinking, &cfg.BindCfg)
	if err != nil {
		log.Println(err)
	}
	return err
}

func loadPackage(path string, buildFirst bool, buildTags string) (*packages.Package, error) {
	cwd, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	if buildFirst {
		args := []string{"build", "-v", "path"}
		if buildTags != "" {
			buildTagStr := fmt.Sprintf("\"%s\"", strings.Join(strings.Split(buildTags, ","), ""))
			args = append(args, "-tags", buildTagStr)
		}
		fmt.Printf("go %v\n", strings.Join(args, " "))
		cmd := exec.Command("go", args...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Dir = cwd

		err = cmd.Run()
		if err != nil {
			log.Printf("Note: there was an error building [%s] -- will continue but it may fail later: %v\n",
				path,
				err,
			)
		}
	}

	// golang.org/x/tools/go/packages supports modules or GOPATH etc
	bpkgs, err := packages.Load(&packages.Config{Mode: packages.LoadTypes}, path)
	if err != nil {
		log.Printf("error resolving import path [%s]: %v\n",
			path,
			err,
		)
		return nil, err
	}

	bpkg := bpkgs[0] // only ever have one at a time
	return bpkg, nil
}

func parsePackage(bpkg *packages.Package) (*bind.Package, error) {
	if len(bpkg.GoFiles) == 0 {
		err := fmt.Errorf("gopy: no files in package %q", bpkg.PkgPath)
		fmt.Println(err)
		return nil, err
	}
	dir, _ := filepath.Split(bpkg.GoFiles[0])
	p := bpkg.Types

	if bpkg.Name == "main" {
		err := fmt.Errorf("gopy: skipping 'main' package %q", bpkg.PkgPath)
		fmt.Println(err)
		return nil, err
	}

	fset := token.NewFileSet()
	var pkgast *ast.Package
	pkgs, err := parser.ParseDir(fset, dir, nil, parser.ParseComments)
	if err != nil {
		return nil, err
	}
	pkgast = pkgs[p.Name()]
	if pkgast == nil {
		return nil, fmt.Errorf("gopy: could not find AST for package %q", p.Name())
	}

	pkgdoc := doc.New(pkgast, bpkg.PkgPath, 0)
	return bind.NewPackage(p, pkgdoc)
}
<|endoftext|>


<commit_before>package nds

import (
	"appengine"
	"appengine/datastore"
	"appengine/memcache"
	"reflect"
	"sync"
)

// getMultiLimit is the App Engine datastore limit for the maximum number
// of entities that can be got by datastore.GetMulti at once.
// nds.GetMulti increases this limit by performing as many
// datastore.GetMulti as required concurrently and collating the results.
const getMultiLimit = 1000

// GetMulti works just like datastore.GetMulti except for two important
// advantages:
//
// 1) It removes the API limit of 1000 entities per request by
// calling the datastore as many times as required to fetch all the keys. It
// does this efficiently and concurrently.
//
// 2) If you use an appengine.Context created from this packages NewContext the
// GetMulti function will automatically invoke a caching mechanism identical
// to the Python ndb package. It also has the same strong cache consistency
// guarantees as the Python ndb package. It will check local memory for an
// entity, then check memcache and then the datastore. This has the potential
// to greatly speed up your entity access and reduce Google App Engine costs.
// Note that if you use GetMulti with this packages NewContext, you must do all
// your other datastore accesses with other methods from this package to ensure
// cache consistency.
//
// Increase the datastore timeout if you get datastore_v3: TIMEOUT errors when
// getting thousands of entities. You can do this using
// http://godoc.org/code.google.com/p/appengine-go/appengine#Timeout.
func GetMulti(c appengine.Context,
	keys []*datastore.Key, dst interface{}) error {

	v := reflect.ValueOf(dst)
	if err := checkMultiArgs(keys, v); err != nil {
		return err
	}

	if len(keys) == 0 {
		return nil
	}

	callCount := (len(keys)-1)/getMultiLimit + 1
	errs := make([]error, callCount)

	wg := sync.WaitGroup{}
	wg.Add(callCount)
	for i := 0; i < callCount; i++ {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}

		index := i
		keySlice := keys[lo:hi]
		dstSlice := v.Slice(lo, hi)

		go func() {
			// Default to datastore.GetMulti if we do not get a nds.context.
			if cc, ok := c.(*context); ok {
				errs[index] = getMulti(cc, keySlice, dstSlice)
			} else {
				errs[index] = datastore.GetMulti(c,
					keySlice, dstSlice.Interface())
			}
			wg.Done()
		}()
	}
	wg.Wait()

	// Quick escape if all errors are nil.
	errsNil := true
	for _, err := range errs {
		if err != nil {
			errsNil = false
		}
	}
	if errsNil {
		return nil
	}

	groupedErrs := make(appengine.MultiError, len(keys))
	for i, err := range errs {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		if me, ok := err.(appengine.MultiError); ok {
			copy(groupedErrs[lo:hi], me)
		} else if err != nil {
			return err
		}
	}
	return groupedErrs
}

// Get is a wrapper around GetMulti. Its return values are identical to
// datastore.Get.
func Get(c appengine.Context, key *datastore.Key, dst interface{}) error {
	err := GetMulti(c, []*datastore.Key{key}, []interface{}{dst})
	if me, ok := err.(appengine.MultiError); ok {
		return me[0]
	}
	return err
}

type getMultiState struct {
	keys      []*datastore.Key
	vals      reflect.Value
	errs      appengine.MultiError
	errsExist bool

	keyIndex map[*datastore.Key]int

	missingMemoryKeys map[*datastore.Key]bool

	missingMemcacheKeys map[*datastore.Key]bool

	// These are keys someone else has locked.
	lockedMemcacheKeys map[*datastore.Key]bool

	// These are keys we have locked.
	lockedMemcacheItems map[string]*memcache.Item

	missingDatastoreKeys map[*datastore.Key]bool
}

func newGetMultiState(keys []*datastore.Key,
	vals reflect.Value) *getMultiState {
	gs := &getMultiState{
		keys: keys,
		vals: vals,
		errs: make(appengine.MultiError, vals.Len()),

		keyIndex: make(map[*datastore.Key]int),

		missingMemoryKeys: make(map[*datastore.Key]bool),

		missingMemcacheKeys: make(map[*datastore.Key]bool),
		lockedMemcacheKeys:  make(map[*datastore.Key]bool),

		missingDatastoreKeys: make(map[*datastore.Key]bool),
	}

	for i, key := range keys {
		gs.keyIndex[key] = i
	}
	return gs
}

// getMulti attempts to get entities from local cache, memcache, then the
// datastore. It also tries to replenish each cache in turn if an entity is
// available.
// The not so obvious part is replenishing memcache with the datastore to
// ensure we don't write stale values.
//
// Here's how it works assuming there is nothing in local cache. (Note this is
// taken form Python ndb):
// Firstly get as many entities from memcache as possible. The returned values
// can be in one of three states: No entity, locked value or the acutal entity.
//
// Actual entity case:
// If the value from memcache is an actual entity then replensish the local
// cache and return that entity to the caller.
//
// Locked entity case:
// If the value is locked then just ignore that entity and go to the datastore
// to see if it exists.
//
// No entity case:
// If no entity is returned from memcache then do the following things to ensure
// we don't accidentally update memcache with stale values.
// 1) Lock that entity in memcache by setting memcacheLock on that entities key.
//    Note that the lock timeout is 32 seconds to cater for a datastore edge
//    case which I currently can't quite remember.
// 2) Immediately get that entity back from memcache ensuring the compare and
//    swap ID is set.
// 3) Get the entity from the datastore.
// 4) Set the entity in memcache using compare and swap. If this succeeds then
//    we are guaranteed to have the latest value in memcache. If it fails due
//    to a CAS failure then there must have been a concurrent write to
//    memcache and now the memcache for that key is out of action for 32
//    seconds.
//
// Note that within a transaction, much of this functionality is lost to ensure
// datastore consistency.
//
// dst argument must be a slice.
func getMulti(cc *context, keys []*datastore.Key, dst reflect.Value) error {

	gs := newGetMultiState(keys, dst)

	if err := loadMemory(cc, gs); err != nil {
		return err
	}

	if !cc.inTransaction {
		if err := loadMemcache(cc, gs); err != nil {
			return err
		}

		// Lock memcache while we get new data from the datastore.
		if err := lockMemcache(cc, gs); err != nil {
			return err
		}
	}

	if err := loadDatastore(cc, gs); err != nil {
		return err
	}

	if !cc.inTransaction {
		if err := saveMemcache(cc, gs); err != nil {
			return err
		}
	}

	if err := saveMemory(cc, gs); err != nil {
		return err
	}

	if gs.errsExist {
		return gs.errs
	}
	return nil
}

func loadMemory(cc *context, gs *getMultiState) error {
	cc.RLock()
	defer cc.RUnlock()

	for index, key := range gs.keys {
		if pl, ok := cc.cache[key.Encode()]; ok {
			if len(pl) == 0 {
				gs.errs[index] = datastore.ErrNoSuchEntity
				gs.errsExist = true
			} else {
				if err := setValue(index, gs.vals, &pl); err != nil {
					return err
				}
			}
		} else {
			gs.missingMemoryKeys[key] = true
		}
	}
	return nil
}

func loadMemcache(cc *context, gs *getMultiState) error {

	memcacheKeys := make([]string, 0, len(gs.missingMemoryKeys))
	for key := range gs.missingMemoryKeys {
		memcacheKeys = append(memcacheKeys, createMemcacheKey(key))
	}

	items, err := memcache.GetMulti(cc, memcacheKeys)
	if err != nil {
		return err
	}
	for key := range gs.missingMemoryKeys {
		memcacheKey := createMemcacheKey(key)

		if item, ok := items[memcacheKey]; ok {
			if isItemLocked(item) {
				gs.lockedMemcacheKeys[key] = true
			} else {
				pl, err := decodePropertyList(item.Value)
				if err != nil {
					return err
				}
				index := gs.keyIndex[key]
				if err := setValue(index, gs.vals, &pl); err != nil {
					return err
				}
			}
		} else {
			gs.missingMemcacheKeys[key] = true
		}
	}
	return nil
}

func loadDatastore(c appengine.Context, gs *getMultiState) error {

	keys := make([]*datastore.Key, 0,
		len(gs.missingMemoryKeys)+len(gs.lockedMemcacheKeys))
	for key := range gs.missingMemoryKeys {
		keys = append(keys, key)
	}
	for key := range gs.lockedMemcacheKeys {
		keys = append(keys, key)
	}
	pls := make([]datastore.PropertyList,
		len(gs.missingMemoryKeys)+len(gs.lockedMemcacheKeys))

	if err := datastore.GetMulti(c, keys, pls); err == nil {
		for i, key := range keys {
			index := gs.keyIndex[key]
			if err := setValue(index, gs.vals, &pls[i]); err != nil {
				return err
			}
		}
	} else if me, ok := err.(appengine.MultiError); ok {
		for i, err := range me {
			if err == nil {
				index := gs.keyIndex[keys[i]]
				if err := setValue(index, gs.vals, &pls[i]); err != nil {
					return err
				}
			} else if err == datastore.ErrNoSuchEntity {
				index := gs.keyIndex[keys[i]]
				gs.errs[index] = datastore.ErrNoSuchEntity
				gs.errsExist = true
				gs.missingDatastoreKeys[keys[i]] = true
			} else {
				return err
			}
		}
	} else {
		return err
	}
	return nil
}

func saveMemcache(c appengine.Context, gs *getMultiState) error {

	items := []*memcache.Item{}
	for key := range gs.missingMemcacheKeys {
		memcacheKey := createMemcacheKey(key)
		if !gs.missingDatastoreKeys[key] {
			index := gs.keyIndex[key]
			s := addrValue(gs.vals.Index(index))
			pl := datastore.PropertyList{}
			if err := saveStruct(s.Interface(), &pl); err != nil {
				return err
			}

			data, err := encodePropertyList(pl)
			if err != nil {
				return err
			}
			if item, ok := gs.lockedMemcacheItems[memcacheKey]; ok {
				item.Value = data
				item.Flags = 0
				items = append(items, item)
			} else {
				item := &memcache.Item{
					Key:   memcacheKey,
					Value: data,
				}
				items = append(items, item)
			}
		}
	}
	if err := memcache.CompareAndSwapMulti(
		c, items); err == memcache.ErrCASConflict {
		return nil
	} else if err == memcache.ErrNotStored {
		return nil
	} else {
		return err
	}
}

func saveMemory(cc *context, gs *getMultiState) error {
	cc.Lock()
	defer cc.Unlock()
	for i, err := range gs.errs {
		if err == nil {
			s := addrValue(gs.vals.Index(i))
			pl := datastore.PropertyList{}
			if err := saveStruct(s.Interface(), &pl); err != nil {
				return err
			}
			cc.cache[gs.keys[i].Encode()] = pl
		}
	}
	return nil
}

func isItemLocked(item *memcache.Item) bool {
	return item.Flags == memcacheLock
}

func lockMemcache(c appengine.Context, gs *getMultiState) error {

	lockItems := make([]*memcache.Item, 0, len(gs.missingMemcacheKeys))
	memcacheKeys := make([]string, 0, len(gs.missingMemcacheKeys))
	for key := range gs.missingMemcacheKeys {
		memcacheKey := createMemcacheKey(key)
		memcacheKeys = append(memcacheKeys, memcacheKey)

		item := &memcache.Item{
			Key:        memcacheKey,
			Flags:      memcacheLock,
			Value:      []byte{},
			Expiration: memcacheLockTime,
		}
		lockItems = append(lockItems, item)
	}
	if err := memcache.SetMulti(c, lockItems); err != nil {
		return err
	}

	items, err := memcache.GetMulti(c, memcacheKeys)
	if err != nil {
		return err
	}
	gs.lockedMemcacheItems = items

	return nil
}
<commit_msg>Rearranged functions for better continuity.<commit_after>package nds

import (
	"appengine"
	"appengine/datastore"
	"appengine/memcache"
	"reflect"
	"sync"
)

// getMultiLimit is the App Engine datastore limit for the maximum number
// of entities that can be got by datastore.GetMulti at once.
// nds.GetMulti increases this limit by performing as many
// datastore.GetMulti as required concurrently and collating the results.
const getMultiLimit = 1000

// GetMulti works just like datastore.GetMulti except for two important
// advantages:
//
// 1) It removes the API limit of 1000 entities per request by
// calling the datastore as many times as required to fetch all the keys. It
// does this efficiently and concurrently.
//
// 2) If you use an appengine.Context created from this packages NewContext the
// GetMulti function will automatically invoke a caching mechanism identical
// to the Python ndb package. It also has the same strong cache consistency
// guarantees as the Python ndb package. It will check local memory for an
// entity, then check memcache and then the datastore. This has the potential
// to greatly speed up your entity access and reduce Google App Engine costs.
// Note that if you use GetMulti with this packages NewContext, you must do all
// your other datastore accesses with other methods from this package to ensure
// cache consistency.
//
// Increase the datastore timeout if you get datastore_v3: TIMEOUT errors when
// getting thousands of entities. You can do this using
// http://godoc.org/code.google.com/p/appengine-go/appengine#Timeout.
func GetMulti(c appengine.Context,
	keys []*datastore.Key, dst interface{}) error {

	v := reflect.ValueOf(dst)
	if err := checkMultiArgs(keys, v); err != nil {
		return err
	}

	if len(keys) == 0 {
		return nil
	}

	callCount := (len(keys)-1)/getMultiLimit + 1
	errs := make([]error, callCount)

	wg := sync.WaitGroup{}
	wg.Add(callCount)
	for i := 0; i < callCount; i++ {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}

		index := i
		keySlice := keys[lo:hi]
		dstSlice := v.Slice(lo, hi)

		go func() {
			// Default to datastore.GetMulti if we do not get a nds.context.
			if cc, ok := c.(*context); ok {
				errs[index] = getMulti(cc, keySlice, dstSlice)
			} else {
				errs[index] = datastore.GetMulti(c,
					keySlice, dstSlice.Interface())
			}
			wg.Done()
		}()
	}
	wg.Wait()

	// Quick escape if all errors are nil.
	errsNil := true
	for _, err := range errs {
		if err != nil {
			errsNil = false
		}
	}
	if errsNil {
		return nil
	}

	groupedErrs := make(appengine.MultiError, len(keys))
	for i, err := range errs {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		if me, ok := err.(appengine.MultiError); ok {
			copy(groupedErrs[lo:hi], me)
		} else if err != nil {
			return err
		}
	}
	return groupedErrs
}

// Get is a wrapper around GetMulti. Its return values are identical to
// datastore.Get.
func Get(c appengine.Context, key *datastore.Key, dst interface{}) error {
	err := GetMulti(c, []*datastore.Key{key}, []interface{}{dst})
	if me, ok := err.(appengine.MultiError); ok {
		return me[0]
	}
	return err
}

type getMultiState struct {
	keys      []*datastore.Key
	vals      reflect.Value
	errs      appengine.MultiError
	errsExist bool

	keyIndex map[*datastore.Key]int

	missingMemoryKeys map[*datastore.Key]bool

	missingMemcacheKeys map[*datastore.Key]bool

	// These are keys someone else has locked.
	lockedMemcacheKeys map[*datastore.Key]bool

	// These are keys we have locked.
	lockedMemcacheItems map[string]*memcache.Item

	missingDatastoreKeys map[*datastore.Key]bool
}

func newGetMultiState(keys []*datastore.Key,
	vals reflect.Value) *getMultiState {
	gs := &getMultiState{
		keys: keys,
		vals: vals,
		errs: make(appengine.MultiError, vals.Len()),

		keyIndex: make(map[*datastore.Key]int),

		missingMemoryKeys: make(map[*datastore.Key]bool),

		missingMemcacheKeys: make(map[*datastore.Key]bool),
		lockedMemcacheKeys:  make(map[*datastore.Key]bool),

		missingDatastoreKeys: make(map[*datastore.Key]bool),
	}

	for i, key := range keys {
		gs.keyIndex[key] = i
	}
	return gs
}

// getMulti attempts to get entities from local cache, memcache, then the
// datastore. It also tries to replenish each cache in turn if an entity is
// available.
// The not so obvious part is replenishing memcache with the datastore to
// ensure we don't write stale values.
//
// Here's how it works assuming there is nothing in local cache. (Note this is
// taken form Python ndb):
// Firstly get as many entities from memcache as possible. The returned values
// can be in one of three states: No entity, locked value or the acutal entity.
//
// Actual entity case:
// If the value from memcache is an actual entity then replensish the local
// cache and return that entity to the caller.
//
// Locked entity case:
// If the value is locked then just ignore that entity and go to the datastore
// to see if it exists.
//
// No entity case:
// If no entity is returned from memcache then do the following things to ensure
// we don't accidentally update memcache with stale values.
// 1) Lock that entity in memcache by setting memcacheLock on that entities key.
//    Note that the lock timeout is 32 seconds to cater for a datastore edge
//    case which I currently can't quite remember.
// 2) Immediately get that entity back from memcache ensuring the compare and
//    swap ID is set.
// 3) Get the entity from the datastore.
// 4) Set the entity in memcache using compare and swap. If this succeeds then
//    we are guaranteed to have the latest value in memcache. If it fails due
//    to a CAS failure then there must have been a concurrent write to
//    memcache and now the memcache for that key is out of action for 32
//    seconds.
//
// Note that within a transaction, much of this functionality is lost to ensure
// datastore consistency.
//
// dst argument must be a slice.
func getMulti(cc *context, keys []*datastore.Key, dst reflect.Value) error {

	gs := newGetMultiState(keys, dst)

	if err := loadMemory(cc, gs); err != nil {
		return err
	}

	if !cc.inTransaction {
		if err := loadMemcache(cc, gs); err != nil {
			return err
		}

		// Lock memcache while we get new data from the datastore.
		if err := lockMemcache(cc, gs); err != nil {
			return err
		}
	}

	if err := loadDatastore(cc, gs); err != nil {
		return err
	}

	if !cc.inTransaction {
		if err := saveMemcache(cc, gs); err != nil {
			return err
		}
	}

	if err := saveMemory(cc, gs); err != nil {
		return err
	}

	if gs.errsExist {
		return gs.errs
	}
	return nil
}

func loadMemory(cc *context, gs *getMultiState) error {
	cc.RLock()
	defer cc.RUnlock()

	for index, key := range gs.keys {
		if pl, ok := cc.cache[key.Encode()]; ok {
			if len(pl) == 0 {
				gs.errs[index] = datastore.ErrNoSuchEntity
				gs.errsExist = true
			} else {
				if err := setValue(index, gs.vals, &pl); err != nil {
					return err
				}
			}
		} else {
			gs.missingMemoryKeys[key] = true
		}
	}
	return nil
}

func loadMemcache(cc *context, gs *getMultiState) error {

	memcacheKeys := make([]string, 0, len(gs.missingMemoryKeys))
	for key := range gs.missingMemoryKeys {
		memcacheKeys = append(memcacheKeys, createMemcacheKey(key))
	}

	items, err := memcache.GetMulti(cc, memcacheKeys)
	if err != nil {
		return err
	}
	for key := range gs.missingMemoryKeys {
		memcacheKey := createMemcacheKey(key)

		if item, ok := items[memcacheKey]; ok {
			if item.Flags == memcacheLock {
				gs.lockedMemcacheKeys[key] = true
			} else {
				pl, err := decodePropertyList(item.Value)
				if err != nil {
					return err
				}
				index := gs.keyIndex[key]
				if err := setValue(index, gs.vals, &pl); err != nil {
					return err
				}
			}
		} else {
			gs.missingMemcacheKeys[key] = true
		}
	}
	return nil
}

func lockMemcache(c appengine.Context, gs *getMultiState) error {

	lockItems := make([]*memcache.Item, 0, len(gs.missingMemcacheKeys))
	memcacheKeys := make([]string, 0, len(gs.missingMemcacheKeys))
	for key := range gs.missingMemcacheKeys {
		memcacheKey := createMemcacheKey(key)
		memcacheKeys = append(memcacheKeys, memcacheKey)

		item := &memcache.Item{
			Key:        memcacheKey,
			Flags:      memcacheLock,
			Value:      []byte{},
			Expiration: memcacheLockTime,
		}
		lockItems = append(lockItems, item)
	}
	if err := memcache.SetMulti(c, lockItems); err != nil {
		return err
	}

	items, err := memcache.GetMulti(c, memcacheKeys)
	if err != nil {
		return err
	}
	gs.lockedMemcacheItems = items

	return nil
}
func loadDatastore(c appengine.Context, gs *getMultiState) error {

	keys := make([]*datastore.Key, 0,
		len(gs.missingMemoryKeys)+len(gs.lockedMemcacheKeys))
	for key := range gs.missingMemoryKeys {
		keys = append(keys, key)
	}
	for key := range gs.lockedMemcacheKeys {
		keys = append(keys, key)
	}
	pls := make([]datastore.PropertyList,
		len(gs.missingMemoryKeys)+len(gs.lockedMemcacheKeys))

	if err := datastore.GetMulti(c, keys, pls); err == nil {
		for i, key := range keys {
			index := gs.keyIndex[key]
			if err := setValue(index, gs.vals, &pls[i]); err != nil {
				return err
			}
		}
	} else if me, ok := err.(appengine.MultiError); ok {
		for i, err := range me {
			if err == nil {
				index := gs.keyIndex[keys[i]]
				if err := setValue(index, gs.vals, &pls[i]); err != nil {
					return err
				}
			} else if err == datastore.ErrNoSuchEntity {
				index := gs.keyIndex[keys[i]]
				gs.errs[index] = datastore.ErrNoSuchEntity
				gs.errsExist = true
				gs.missingDatastoreKeys[keys[i]] = true
			} else {
				return err
			}
		}
	} else {
		return err
	}
	return nil
}

func saveMemcache(c appengine.Context, gs *getMultiState) error {

	items := []*memcache.Item{}
	for key := range gs.missingMemcacheKeys {
		memcacheKey := createMemcacheKey(key)
		if !gs.missingDatastoreKeys[key] {
			index := gs.keyIndex[key]
			s := addrValue(gs.vals.Index(index))
			pl := datastore.PropertyList{}
			if err := saveStruct(s.Interface(), &pl); err != nil {
				return err
			}

			data, err := encodePropertyList(pl)
			if err != nil {
				return err
			}
			if item, ok := gs.lockedMemcacheItems[memcacheKey]; ok {
				item.Value = data
				item.Flags = 0
				items = append(items, item)
			} else {
				item := &memcache.Item{
					Key:   memcacheKey,
					Value: data,
				}
				items = append(items, item)
			}
		}
	}
	if err := memcache.CompareAndSwapMulti(
		c, items); err == memcache.ErrCASConflict {
		return nil
	} else if err == memcache.ErrNotStored {
		return nil
	} else {
		return err
	}
}

func saveMemory(cc *context, gs *getMultiState) error {
	cc.Lock()
	defer cc.Unlock()
	for i, err := range gs.errs {
		if err == nil {
			s := addrValue(gs.vals.Index(i))
			pl := datastore.PropertyList{}
			if err := saveStruct(s.Interface(), &pl); err != nil {
				return err
			}
			cc.cache[gs.keys[i].Encode()] = pl
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"os/exec"
	"strconv"
	"strings"
)

func getGitBranch() string {
	out, err := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").Output()
	if err != nil {
		return ""
	}
	return strings.Trim(string(out), "\n")
}

func getGitChanges() string {
	out, err := exec.Command("git", "status", "-s", "--porcelain").Output()
	if err != nil {
		return ""
	}
	counter := 0
	s := bufio.NewScanner(bytes.NewReader(out))
	for s.Scan() {
		counter++
	}
	return strconv.Itoa(counter)
}

func getGitPartial() string {
	return fmt.Sprintf("%s:%s", getGitChanges(), getGitBranch())
}
<commit_msg>Add changes<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"os/exec"
	"strings"
)

func getGitBranch() string {
	out, err := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").Output()
	if err != nil {
		return ""
	}
	return strings.Trim(string(out), "\n")
}

func getGitChanges() int {
	out, err := exec.Command("git", "status", "-s", "--porcelain").Output()
	if err != nil {
		return -1
	}
	changes := 0
	s := bufio.NewScanner(bytes.NewReader(out))
	for s.Scan() {
		changes++
	}
	return changes
}

func getGitPartial() string {
	partial := ""
	changes := getGitChanges()
	if changes > 0 {
		partial = fmt.Sprintf("%d:", changes)
	}
	partial = fmt.Sprintf("%s%s", partial, getGitBranch())
	return partial
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/xorpaul/uiprogress"
)

func resolveGitRepositories(uniqueGitModules map[string]GitModule) {
	if len(uniqueGitModules) <= 0 {
		Debugf("uniqueGitModules[] is empty, skipping...")
		return
	}
	var wgGit sync.WaitGroup
	bar := uiprogress.AddBar(len(uniqueGitModules)).AppendCompleted().PrependElapsed()
	bar.PrependFunc(func(b *uiprogress.Bar) string {
		return fmt.Sprintf("Resolving Git modules (%d/%d)", b.Current(), len(uniqueGitModules))
	})
	for url, gm := range uniqueGitModules {
		wgGit.Add(1)
		privateKey := gm.privateKey
		go func(url string, privateKey string, gm GitModule) {
			defer wgGit.Done()
			defer bar.Incr()
			if len(gm.privateKey) > 0 {
				Debugf("git repo url " + url + " with ssh key " + privateKey)
			} else {
				Debugf("git repo url " + url + " without ssh key")
			}

			//log.Println(config)
			// create save directory name from Git repo name
			repoDir := strings.Replace(strings.Replace(url, "/", "_", -1), ":", "-", -1)
			workDir := config.ModulesCacheDir + repoDir

			doMirrorOrUpdate(url, workDir, privateKey, gm.ignoreUnreachable)
			//	doCloneOrPull(source, workDir, targetDir, sa.Remote, branch, sa.PrivateKey)

		}(url, privateKey, gm)
	}
	wgGit.Wait()
}

func doMirrorOrUpdate(url string, workDir string, sshPrivateKey string, allowFail bool) bool {
	needSSHKey := true
	if strings.Contains(url, "github.com") || len(sshPrivateKey) == 0 {
		needSSHKey = false
	}

	er := ExecResult{}
	gitCmd := "git clone --mirror " + url + " " + workDir
	if fileExists(workDir) {
		gitCmd = "git --git-dir " + workDir + " remote update --prune"
	}

	if needSSHKey {
		er = executeCommand("ssh-agent bash -c 'ssh-add "+sshPrivateKey+"; "+gitCmd+"'", config.Timeout, allowFail)
	} else {
		er = executeCommand(gitCmd, config.Timeout, allowFail)
	}

	if er.returnCode != 0 {
		fmt.Println("WARN: git repository " + url + " does not exist or is unreachable at this moment!")
		return false
	}
	return true
}

func syncToModuleDir(srcDir string, targetDir string, tree string, allowFail bool, ignoreUnreachable bool) bool {
	mutex.Lock()
	syncGitCount++
	mutex.Unlock()
	logCmd := "git --git-dir " + srcDir + " log -n1 --pretty=format:%H " + tree
	er := executeCommand(logCmd, config.Timeout, allowFail)
	hashFile := targetDir + "/.latest_commit"
	needToSync := true
	if er.returnCode != 0 && allowFail {
		if ignoreUnreachable {
			Infof("Failed to populate module " + targetDir + " but ignore-unreachable is set. Continuing...")
			purgeDir(targetDir, "syncToModuleDir, because ignore-unreachable is set for this module")
		}
		return false
	}

	if len(er.output) > 0 {
		targetHash, _ := ioutil.ReadFile(hashFile)
		if string(targetHash) == er.output {
			needToSync = false
			//Debugf("Skipping, because no diff found between " + srcDir + "(" + er.output + ") and " + targetDir + "(" + string(targetHash) + ")")
		}

	}
	if needToSync && er.returnCode == 0 {
		Infof("Need to sync " + targetDir)
		mutex.Lock()
		needSyncGitCount++
		mutex.Unlock()
		if !dryRun {
			createOrPurgeDir(targetDir, "syncToModuleDir()")
			cmd := "git --git-dir " + srcDir + " archive " + tree + " | tar -x -C " + targetDir
			before := time.Now()
			out, err := exec.Command("bash", "-c", cmd).CombinedOutput()
			duration := time.Since(before).Seconds()
			mutex.Lock()
			ioGitTime += duration
			mutex.Unlock()
			Verbosef("syncToModuleDir(): Executing " + cmd + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")
			if err != nil {
				if !allowFail {
					Fatalf("syncToModuleDir(): Failed to execute command: " + cmd + " Output: " + string(out))
				} else {
					Infof("Failed to populate module " + targetDir + " but ignore-unreachable is set. Continuing...")
					return false
				}
			}

			er = executeCommand(logCmd, config.Timeout, false)
			if len(er.output) > 0 {
				Debugf("Writing hash " + er.output + " from command " + logCmd + " to " + hashFile)
				f, _ := os.Create(hashFile)
				defer f.Close()
				f.WriteString(er.output)
				f.Sync()
			}
		}
	}
	return true
}
<commit_msg>Fix remote execution<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/xorpaul/uiprogress"
)

func resolveGitRepositories(uniqueGitModules map[string]GitModule) {
	if len(uniqueGitModules) <= 0 {
		Debugf("uniqueGitModules[] is empty, skipping...")
		return
	}
	var wgGit sync.WaitGroup
	bar := uiprogress.AddBar(len(uniqueGitModules)).AppendCompleted().PrependElapsed()
	bar.PrependFunc(func(b *uiprogress.Bar) string {
		return fmt.Sprintf("Resolving Git modules (%d/%d)", b.Current(), len(uniqueGitModules))
	})
	for url, gm := range uniqueGitModules {
		wgGit.Add(1)
		privateKey := gm.privateKey
		go func(url string, privateKey string, gm GitModule) {
			defer wgGit.Done()
			defer bar.Incr()
			if len(gm.privateKey) > 0 {
				Debugf("git repo url " + url + " with ssh key " + privateKey)
			} else {
				Debugf("git repo url " + url + " without ssh key")
			}

			//log.Println(config)
			// create save directory name from Git repo name
			repoDir := strings.Replace(strings.Replace(url, "/", "_", -1), ":", "-", -1)
			workDir := config.ModulesCacheDir + repoDir

			doMirrorOrUpdate(url, workDir, privateKey, gm.ignoreUnreachable)
			//	doCloneOrPull(source, workDir, targetDir, sa.Remote, branch, sa.PrivateKey)

		}(url, privateKey, gm)
	}
	wgGit.Wait()
}

func doMirrorOrUpdate(url string, workDir string, sshPrivateKey string, allowFail bool) bool {
	needSSHKey := true
	if strings.Contains(url, "github.com") || len(sshPrivateKey) == 0 {
		needSSHKey = false
	}

	er := ExecResult{}
	gitCmd := "git clone --mirror " + url + " " + workDir
	if fileExists(workDir) {
		gitCmd = "git --git-dir " + workDir + " remote update --prune"
	}

	if needSSHKey {
		er = executeCommand("ssh-agent bash -c 'ssh-add "+sshPrivateKey+"; "+gitCmd+"'", config.Timeout, allowFail)
	} else {
		er = executeCommand(gitCmd, config.Timeout, allowFail)
	}

	if er.returnCode != 0 {
		fmt.Println("WARN: git repository " + url + " does not exist or is unreachable at this moment!")
		return false
	}
	return true
}

func syncToModuleDir(srcDir string, targetDir string, tree string, allowFail bool, ignoreUnreachable bool) bool {
	mutex.Lock()
	syncGitCount++
	mutex.Unlock()
	logCmd := "git --git-dir " + srcDir + " log -n1 --pretty=format:%H " + tree
	er := executeCommand(logCmd, config.Timeout, allowFail)
	hashFile := targetDir + "/.latest_commit"
	needToSync := true
	if er.returnCode != 0 && allowFail {
		if ignoreUnreachable {
			Infof("Failed to populate module " + targetDir + " but ignore-unreachable is set. Continuing...")
			purgeDir(targetDir, "syncToModuleDir, because ignore-unreachable is set for this module")
		}
		return false
	}

	if len(er.output) > 0 {
		targetHash, _ := ioutil.ReadFile(hashFile)
		if string(targetHash) == er.output {
			needToSync = false
			//Debugf("Skipping, because no diff found between " + srcDir + "(" + er.output + ") and " + targetDir + "(" + string(targetHash) + ")")
		}

	}
	if needToSync && er.returnCode == 0 {
		Infof("Need to sync " + targetDir)
		mutex.Lock()
		needSyncGitCount++
		mutex.Unlock()
		if !dryRun {
			createOrPurgeDir(targetDir, "syncToModuleDir()")
			cmd := "git --git-dir " + srcDir + " archive '" + tree + "' | tar -x -C " + targetDir
			before := time.Now()
			out, err := exec.Command("bash", "-c", cmd).CombinedOutput()
			duration := time.Since(before).Seconds()
			mutex.Lock()
			ioGitTime += duration
			mutex.Unlock()
			Verbosef("syncToModuleDir(): Executing " + cmd + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")
			if err != nil {
				if !allowFail {
					Fatalf("syncToModuleDir(): Failed to execute command: " + cmd + " Output: " + string(out))
				} else {
					Infof("Failed to populate module " + targetDir + " but ignore-unreachable is set. Continuing...")
					return false
				}
			}

			er = executeCommand(logCmd, config.Timeout, false)
			if len(er.output) > 0 {
				Debugf("Writing hash " + er.output + " from command " + logCmd + " to " + hashFile)
				f, _ := os.Create(hashFile)
				defer f.Close()
				f.WriteString(er.output)
				f.Sync()
			}
		}
	}
	return true
}
<|endoftext|>


<commit_before>package gsmake

import (
	"os"
	"os/exec"
	"path/filepath"

	"github.com/gsdocker/gserrors"
	"github.com/gsdocker/gslogger"
	"github.com/gsdocker/gsos"
	"github.com/gsdocker/gsos/uuid"
)

type gitSCM struct {
	gslogger.Log        // mixin log APIs
	homepath     string // gsmake home path
	cmd          string // command name
	name         string // command display name
}

func newGitSCM(homepath string) (*gitSCM, error) {
	_, err := SearchCmd("git")

	if err != nil {
		return nil, err
	}

	return &gitSCM{
		Log:      gslogger.Get("gsmake"),
		cmd:      "git",
		name:     "GIT",
		homepath: homepath,
	}, nil
}

func (git *gitSCM) String() string {
	return git.name
}

func (git *gitSCM) Cmd() string {
	return git.cmd
}

// Update implement SCM interface func
func (git *gitSCM) Update(url string, name string) error {
	repopath := RepoDir(git.homepath, name)

	if !gsos.IsDir(repopath) {
		return gserrors.Newf(ErrPackage, "package %s not cached", name)
	}

	currentDir := gsos.CurrentDir()

	if err := os.Chdir(repopath); err != nil {
		return gserrors.Newf(err, "git change current dir to work path error")
	}

	command := exec.Command(git.name, "pull")

	command.Stderr = os.Stderr
	command.Stdin = os.Stdin
	command.Stdout = os.Stdout

	err := command.Run()

	os.Chdir(currentDir)

	if err != nil {
		return gserrors.Newf(err, "exec error :git pull")
	}

	return nil
}

func (git *gitSCM) Create(url string, name string, version string) (string, error) {
	repopath := RepoDir(git.homepath, name)

	// if the local repo not exist, then clone it from host site
	if !gsos.IsDir(repopath) {

		// first clone package into cache dir
		cachedir := filepath.Join(os.TempDir(), uuid.New())

		if err := os.MkdirAll(cachedir, 0755); err != nil {

			return repopath, err
		}

		if err := os.MkdirAll(filepath.Dir(repopath), 0755); err != nil {

			return repopath, err
		}

		command := exec.Command(git.cmd, "clone", url, cachedir)

		command.Stderr = os.Stderr
		command.Stdin = os.Stdin
		command.Stdout = os.Stdout

		if err := command.Run(); err != nil {
			return repopath, err
		}

		if err := gsos.CopyDir(cachedir, repopath); err != nil {
			return repopath, err
		}
	}

	currentDir := gsos.CurrentDir()

	// fix windows git can't handle symlink repo directory bug
	realpath, err := os.Readlink(repopath)

	if err == nil {
		repopath = realpath
	}

	if err := os.Chdir(repopath); err != nil {
		return "", err
	}

	if version == "current" {
		version = "master"
	}

	command := exec.Command(git.name, "checkout", version)

	command.Stderr = os.Stderr
	command.Stdin = os.Stdin
	command.Stdout = os.Stdout

	err = command.Run()

	os.Chdir(currentDir)

	if err != nil {
		return "", err
	}

	return repopath, nil
}

// Get implement SCM interface func
func (git *gitSCM) Get(url string, name string, version string, targetpath string) error {

	git.D("get package :%s", name)

	repopath, err := git.Create(url, name, version)

	if err != nil {
		return err
	}

	if gsos.IsExist(targetpath) {

		git.D("remove exist linked package :%s", targetpath)

		err := gsos.RemoveAll(targetpath)
		if err != nil {
			return gserrors.Newf(err, "git scm remove target dir error")
		}
	}

	return gsos.CopyDir(repopath, targetpath)
}
<commit_msg>update<commit_after>package gsmake

import (
	"os"
	"os/exec"
	"path/filepath"

	"github.com/gsdocker/gserrors"
	"github.com/gsdocker/gslogger"
	"github.com/gsdocker/gsos"
	"github.com/gsdocker/gsos/uuid"
)

type gitSCM struct {
	gslogger.Log        // mixin log APIs
	homepath     string // gsmake home path
	cmd          string // command name
	name         string // command display name
}

func newGitSCM(homepath string) (*gitSCM, error) {
	_, err := SearchCmd("git")

	if err != nil {
		return nil, err
	}

	return &gitSCM{
		Log:      gslogger.Get("gsmake"),
		cmd:      "git",
		name:     "GIT",
		homepath: homepath,
	}, nil
}

func (git *gitSCM) String() string {
	return git.name
}

func (git *gitSCM) Cmd() string {
	return git.cmd
}

// Update implement SCM interface func
func (git *gitSCM) Update(url string, name string) error {
	repopath := RepoDir(git.homepath, name)

	if !gsos.IsDir(repopath) {
		return gserrors.Newf(ErrPackage, "package %s not cached", name)
	}

	currentDir := gsos.CurrentDir()

	if err := os.Chdir(repopath); err != nil {
		return gserrors.Newf(err, "git change current dir to work path error")
	}

	command := exec.Command(git.name, "pull")

	err := command.Run()

	os.Chdir(currentDir)

	if err != nil {
		return gserrors.Newf(err, "exec error :git pull")
	}

	return nil
}

func (git *gitSCM) Create(url string, name string, version string) (string, error) {
	repopath := RepoDir(git.homepath, name)

	// if the local repo not exist, then clone it from host site
	if !gsos.IsDir(repopath) {

		// first clone package into cache dir
		cachedir := filepath.Join(os.TempDir(), uuid.New())

		if err := os.MkdirAll(cachedir, 0755); err != nil {

			return repopath, err
		}

		if err := os.MkdirAll(filepath.Dir(repopath), 0755); err != nil {

			return repopath, err
		}

		command := exec.Command(git.cmd, "clone", url, cachedir)

		if err := command.Run(); err != nil {
			return repopath, err
		}

		if err := gsos.CopyDir(cachedir, repopath); err != nil {
			return repopath, err
		}
	}

	currentDir := gsos.CurrentDir()

	// fix windows git can't handle symlink repo directory bug
	realpath, err := os.Readlink(repopath)

	if err == nil {
		repopath = realpath
	}

	if err := os.Chdir(repopath); err != nil {
		return "", err
	}

	if version == "current" {
		version = "master"
	}

	command := exec.Command(git.name, "checkout", version)

	err = command.Run()

	os.Chdir(currentDir)

	if err != nil {
		return "", err
	}

	return repopath, nil
}

// Get implement SCM interface func
func (git *gitSCM) Get(url string, name string, version string, targetpath string) error {

	git.D("get package :%s", name)

	repopath, err := git.Create(url, name, version)

	if err != nil {
		return err
	}

	if gsos.IsExist(targetpath) {

		git.D("remove exist linked package :%s", targetpath)

		err := gsos.RemoveAll(targetpath)
		if err != nil {
			return gserrors.Newf(err, "git scm remove target dir error")
		}
	}

	return gsos.CopyDir(repopath, targetpath)
}
<|endoftext|>


<commit_before>// Package md5 computes MD5 checksum for large files
package md5

import (
	"bufio"
	"crypto/md5"
	"fmt"
	"io"
	"os"
)

const bufferSize = 65536

// MD5sum returns MD5 checksum of filename
func MD5sum(filename string) (string, error) {
	if info, err := os.Stat(filename); err != nil || info.IsDir() {
		return "", err
	}

	file, err := os.Open(filename)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hash := md5.New()
	buf := make([]byte, bufferSize)
	reader := bufio.NewReader(file)
	for {
		if n, err := reader.Read(buf); err == nil {
			hash.Write(buf[:n])
		} else if err == io.EOF {
			break
		} else {
			return "", err
		}
	}
	return fmt.Sprintf("%x", hash.Sum(nil)), nil
}
<commit_msg>code clean up<commit_after>// Package md5 computes MD5 checksum for large files
package md5

import (
	"bufio"
	"crypto/md5"
	"fmt"
	"io"
	"os"
)

const bufferSize = 65536

// MD5sum returns MD5 checksum of filename
func MD5sum(filename string) (string, error) {
	if info, err := os.Stat(filename); err != nil || info.IsDir() {
		return "", err
	}

	file, err := os.Open(filename)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hash := md5.New()
	buf := make([]byte, bufferSize)
	reader := bufio.NewReader(file)
here:
	for {
		n, err := reader.Read(buf)
		switch err {
		case nil:
			hash.Write(buf[:n])
		case io.EOF:
			break here
		default:
			return "", err
		}
	}
	return fmt.Sprintf("%x", hash.Sum(nil)), nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/scott-linder/irc"
	"log"
	"os"
)

const (
	configFilename = "hal.json"
)

// config is the main bot config struct.
var config = struct {
	Host string
	Chan string
}{
	Host: "irc.freenode.net:6667",
	Chan: "#bots",
}

// loadConfig attempts to load config from configFilename.
func loadConfig() error {
	file, err := os.Open(configFilename)
	if err != nil {
		return err
	}
	defer file.Close()

	decoder := json.NewDecoder(file)
	if err := decoder.Decode(&config); err != nil {
		return err
	}
	return nil
}

// Pong plays your game.
type Pong struct{}

func (pong Pong) Accept(msg *irc.Msg) bool { return msg.Cmd == "PING" }
func (pong Pong) Handle(msg *irc.Msg, send chan<- *irc.Msg) {
	send <- &irc.Msg{Cmd: "PONG", Params: msg.Params}
}

// Echo talks back.
type Echo struct{}

func (echo Echo) Accept(msg *irc.Msg) bool { return msg.Cmd == "PRIVMSG" }
func (echo Echo) Handle(msg *irc.Msg, send chan<- *irc.Msg) {
	send <- &irc.Msg{Cmd: "PRIVMSG", Params: msg.Params}
}

func main() {
	fmt.Println("I am a HAL 9001 computer.")
	if err := loadConfig(); err != nil {
		log.Printf("error loading config file %v: %v", configFilename, err)
	}
	client, err := irc.Dial(config.Host)
	if err != nil {
		log.Fatal(err)
	}
	client.Register(Pong{})
	client.Register(Echo{})
	client.Nick("hal")
	client.Join(config.Chan)
	client.Listen()
}
<commit_msg>Added more sensible Host default.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/scott-linder/irc"
	"log"
	"os"
)

const (
	configFilename = "hal.json"
)

// config is the main bot config struct.
var config = struct {
	Host string
	Chan string
}{
	Host: "127.0.0.1:6667",
	Chan: "#bots",
}

// loadConfig attempts to load config from configFilename.
func loadConfig() error {
	file, err := os.Open(configFilename)
	if err != nil {
		return err
	}
	defer file.Close()

	decoder := json.NewDecoder(file)
	if err := decoder.Decode(&config); err != nil {
		return err
	}
	return nil
}

// Pong plays your game.
type Pong struct{}

func (pong Pong) Accept(msg *irc.Msg) bool { return msg.Cmd == "PING" }
func (pong Pong) Handle(msg *irc.Msg, send chan<- *irc.Msg) {
	send <- &irc.Msg{Cmd: "PONG", Params: msg.Params}
}

// Echo talks back.
type Echo struct{}

func (echo Echo) Accept(msg *irc.Msg) bool { return msg.Cmd == "PRIVMSG" }
func (echo Echo) Handle(msg *irc.Msg, send chan<- *irc.Msg) {
	send <- &irc.Msg{Cmd: "PRIVMSG", Params: msg.Params}
}

func main() {
	fmt.Println("I am a HAL 9001 computer.")
	if err := loadConfig(); err != nil {
		log.Printf("error loading config file %v: %v", configFilename, err)
	}
	client, err := irc.Dial(config.Host)
	if err != nil {
		log.Fatal(err)
	}
	client.Register(Pong{})
	client.Register(Echo{})
	client.Nick("hal")
	client.Join(config.Chan)
	client.Listen()
}
<|endoftext|>


<commit_before>package hot

import (
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"

	"gopkg.in/fsnotify.v1"
)

type Config struct {
	Watch          bool
	BaseName       string
	Dir            string
	Funcs          template.FuncMap
	LeftDelim      string
	RightDelim     string
	FilesExtension []string
	Log            io.Writer
}

type Template struct {
	tpl *template.Template
	cfg *Config
	Out io.Writer
}

func New(cfg *Config) (*Template, error) {
	leftDelim := "{{"
	rightDelim := "}}"
	if cfg.LeftDelim != "" {
		leftDelim = cfg.LeftDelim
	}
	if cfg.RightDelim != "" {
		rightDelim = cfg.RightDelim
	}
	tmpl := template.New(cfg.BaseName).Delims(leftDelim, rightDelim)
	if cfg.Funcs != nil {
		tmpl = template.New(cfg.BaseName).Funcs(cfg.Funcs).Delims(leftDelim, rightDelim)
	}
	tpl := &Template{
		tpl: tmpl,
		cfg: cfg,
		Out: os.Stdout,
	}
	if cfg.Log != nil {
		tpl.Out = cfg.Log
	}
	tpl.Init()
	err := tpl.Load(cfg.Dir)
	if err != nil {
		return nil, err
	}
	return tpl, nil
}

func (t *Template) Init() {
	if t.cfg.Watch {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGTERM, syscall.SIGINT)
		watcher, err := fsnotify.NewWatcher()
		if err != nil {
			fmt.Fprintln(t.Out, err)
			return
		}
		err = filepath.Walk(t.cfg.Dir, func(fPath string, info os.FileInfo, ferr error) error {
			if ferr != nil {
				return ferr
			}
			if info.IsDir() {
				fmt.Fprintln(t.Out, "start watching ", fPath)
				return watcher.Add(fPath)
			}
			return nil
		})
		if err != nil {
			watcher.Close()
			fmt.Fprintln(t.Out, err)
			return
		}
		go func() {
			for {
				select {
				case <-c:
					watcher.Close()
					fmt.Println("shutting down hot templates... done")
					os.Exit(0)
				case evt := <-watcher.Events:
					fmt.Fprintf(t.Out, "%s:  reloading... \n", evt.String())
					t.Reload()
				}
			}
		}()
	}
}

func (t *Template) Reload() {
	tpl := *t.tpl
	t.tpl = template.New(t.cfg.BaseName)
	if t.cfg.Funcs != nil {
		t.tpl = template.New(t.cfg.BaseName).Funcs(t.cfg.Funcs)
	}
	err := t.Load(t.cfg.Dir)
	if err != nil {
		fmt.Fprintln(t.Out, err.Error())
		t.tpl = &tpl
	}
}

func (t *Template) Load(dir string) error {
	fmt.Fprintln(t.Out, "loading...", dir)
	return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}

		extension := filepath.Ext(path)
		found := false
		for _, ext := range t.cfg.FilesExtension {
			if ext == extension {
				found = true
				break
			}
		}
		if !found {
			return nil
		}

		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}

		// We remove the directory name from the path
		// this means if we have directory foo, with file bar.tpl
		// full path for bar file foo/bar.tpl
		// we trim the foo part and remain with /bar.tpl
		name := path[len(dir):]

		name = filepath.ToSlash(name)

		name = strings.TrimPrefix(name, "/") // case  we missed the opening slash

		tpl := t.tpl.New(name)
		_, err = tpl.Parse(string(data))
		if err != nil {
			return err
		}
		return nil
	})
}

func (t *Template) Execute(w io.Writer, name string, ctx interface{}) error {
	return t.tpl.ExecuteTemplate(w, name, ctx)
}
<commit_msg>Shutting down log changed to t.Out<commit_after>package hot

import (
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"

	"gopkg.in/fsnotify.v1"
)

type Config struct {
	Watch          bool
	BaseName       string
	Dir            string
	Funcs          template.FuncMap
	LeftDelim      string
	RightDelim     string
	FilesExtension []string
	Log            io.Writer
}

type Template struct {
	tpl *template.Template
	cfg *Config
	Out io.Writer
}

func New(cfg *Config) (*Template, error) {
	leftDelim := "{{"
	rightDelim := "}}"
	if cfg.LeftDelim != "" {
		leftDelim = cfg.LeftDelim
	}
	if cfg.RightDelim != "" {
		rightDelim = cfg.RightDelim
	}
	tmpl := template.New(cfg.BaseName).Delims(leftDelim, rightDelim)
	if cfg.Funcs != nil {
		tmpl = template.New(cfg.BaseName).Funcs(cfg.Funcs).Delims(leftDelim, rightDelim)
	}
	tpl := &Template{
		tpl: tmpl,
		cfg: cfg,
		Out: os.Stdout,
	}
	if cfg.Log != nil {
		tpl.Out = cfg.Log
	}
	tpl.Init()
	err := tpl.Load(cfg.Dir)
	if err != nil {
		return nil, err
	}
	return tpl, nil
}

func (t *Template) Init() {
	if t.cfg.Watch {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGTERM, syscall.SIGINT)
		watcher, err := fsnotify.NewWatcher()
		if err != nil {
			fmt.Fprintln(t.Out, err)
			return
		}
		err = filepath.Walk(t.cfg.Dir, func(fPath string, info os.FileInfo, ferr error) error {
			if ferr != nil {
				return ferr
			}
			if info.IsDir() {
				fmt.Fprintln(t.Out, "start watching ", fPath)
				return watcher.Add(fPath)
			}
			return nil
		})
		if err != nil {
			watcher.Close()
			fmt.Fprintln(t.Out, err)
			return
		}
		go func() {
			for {
				select {
				case <-c:
					watcher.Close()
					fmt.Fprintf(t.Out,"shutting down hot templates... done")
					os.Exit(0)
				case evt := <-watcher.Events:
					fmt.Fprintf(t.Out, "%s:  reloading... \n", evt.String())
					t.Reload()
				}
			}
		}()
	}
}

func (t *Template) Reload() {
	tpl := *t.tpl
	t.tpl = template.New(t.cfg.BaseName)
	if t.cfg.Funcs != nil {
		t.tpl = template.New(t.cfg.BaseName).Funcs(t.cfg.Funcs)
	}
	err := t.Load(t.cfg.Dir)
	if err != nil {
		fmt.Fprintln(t.Out, err.Error())
		t.tpl = &tpl
	}
}

func (t *Template) Load(dir string) error {
	fmt.Fprintln(t.Out, "loading...", dir)
	return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}

		extension := filepath.Ext(path)
		found := false
		for _, ext := range t.cfg.FilesExtension {
			if ext == extension {
				found = true
				break
			}
		}
		if !found {
			return nil
		}

		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}

		// We remove the directory name from the path
		// this means if we have directory foo, with file bar.tpl
		// full path for bar file foo/bar.tpl
		// we trim the foo part and remain with /bar.tpl
		name := path[len(dir):]

		name = filepath.ToSlash(name)

		name = strings.TrimPrefix(name, "/") // case  we missed the opening slash

		tpl := t.tpl.New(name)
		_, err = tpl.Parse(string(data))
		if err != nil {
			return err
		}
		return nil
	})
}

func (t *Template) Execute(w io.Writer, name string, ctx interface{}) error {
	return t.tpl.ExecuteTemplate(w, name, ctx)
}
<|endoftext|>