<commit_before>// Copyright © 2016 Abcum Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package orbit

import (
	"time"

	"github.com/robertkrimen/otto"
)

// Orbit is a node.go context.
type Orbit struct {
	// Underlying Otto instance.
	*otto.Otto
	// External runtime variables.
	Vars map[string]interface{}
	// quit
	quit chan error
	// Loop runs pending timers
	loop chan *task
	// Timeout timer
	timer *time.Timer
	// Runtime timers
	timers map[*task]*task
	// Timeout sets a timeout
	timeout time.Duration
	// Module outputs are cached for future use.
	modules map[string]otto.Value
}

type (
	// Module is a javascript module
	module func(*Orbit) (otto.Value, error)
)

var (
	// Event groups
	inits []func(*Orbit)
	exits []func(*Orbit)
	fails []func(*Orbit, error)
	// Finder loads files
	finder func(*Orbit, []string) (interface{}, string, error)
	// Modules stores registered packages
	modules = make(map[string]module)
)

// Add adds a module to the runtime
func Add(name string, item interface{}) {
	switch what := item.(type) {
	case string:
		addSource(name, what)
	case []byte:
		addSource(name, what)
	case func(*Orbit) (otto.Value, error):
		addModule(name, what)
	}
}

func Find(ctx *Orbit, files []string) (interface{}, string, error) {
	return finder(ctx, files)
}

// OnInit registers a callback for when the program starts up
func OnInit(call func(*Orbit)) {
	inits = append(inits, call)
}

// OnExit registers a callback for when the program shuts down
func OnExit(call func(*Orbit)) {
	exits = append(exits, call)
}

// OnFail registers a callback for when the program encounters and error
func OnFail(call func(*Orbit, error)) {
	fails = append(fails, call)
}

// OnFile registers a callback for finding required files
func OnFile(call func(*Orbit, []string) (interface{}, string, error)) {
	finder = call
}

// New creates a new Orbit runtime
func New(timeout time.Duration) *Orbit {

	orbit := &Orbit{
		Otto:    otto.New(),
		Vars:    make(map[string]interface{}),
		quit:    make(chan error),
		loop:    make(chan *task),
		timers:  make(map[*task]*task),
		modules: make(map[string]otto.Value),
		timeout: timeout * time.Millisecond,
	}

	orbit.Interrupt = make(chan func(), 1)

	return orbit

}

// Def sets a global variable in the runtime
func (ctx *Orbit) Def(name string, item interface{}) {
	obj, _ := ctx.Get("global")
	obj.Object().Set(name, item)
	ctx.Set(name, item)
}

// Run executes some code. Code may be a string or a byte slice.
func (ctx *Orbit) Run(name string, code interface{}) (val otto.Value, err error) {

	defer func() {

		var ok bool

		if err, ok = recover().(error); ok {
			ctx.fail(err)
		}

		ctx.exit()

		if ctx.timer != nil {
			ctx.timer.Stop()
		}

		for timer := range ctx.timers {
			timer.timer.Stop()
			delete(ctx.timers, timer)
		}

	}()

	ctx.SetStackDepthLimit(20000)

	ctx.tick()

	ctx.init()

	// Run main code
	val, err = main(code, name)(ctx)
	if err != nil {
		panic(err)
	}

	// Wait for timers
	err = ctx.wait()
	if err != nil {
		panic(err)
	}

	return

}

func (ctx *Orbit) Quit(err error) {
	panic(err)
}

func (ctx *Orbit) init() {
	for _, e := range inits {
		e(ctx)
	}
}

func (ctx *Orbit) exit() {
	for _, e := range exits {
		e(ctx)
	}
}

func (ctx *Orbit) fail(err error) {
	for _, e := range fails {
		e(ctx, err)
	}
}
<commit_msg>Don't return the returned value from otto<commit_after>// Copyright © 2016 Abcum Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package orbit

import (
	"time"

	"github.com/robertkrimen/otto"
)

// Orbit is a node.go context.
type Orbit struct {
	// Underlying Otto instance.
	*otto.Otto
	// External runtime variables.
	Vars map[string]interface{}
	// quit
	quit chan error
	// Loop runs pending timers
	loop chan *task
	// Timeout timer
	timer *time.Timer
	// Runtime timers
	timers map[*task]*task
	// Timeout sets a timeout
	timeout time.Duration
	// Module outputs are cached for future use.
	modules map[string]otto.Value
}

type (
	// Module is a javascript module
	module func(*Orbit) (otto.Value, error)
)

var (
	// Event groups
	inits []func(*Orbit)
	exits []func(*Orbit)
	fails []func(*Orbit, error)
	// Finder loads files
	finder func(*Orbit, []string) (interface{}, string, error)
	// Modules stores registered packages
	modules = make(map[string]module)
)

// Add adds a module to the runtime
func Add(name string, item interface{}) {
	switch what := item.(type) {
	case string:
		addSource(name, what)
	case []byte:
		addSource(name, what)
	case func(*Orbit) (otto.Value, error):
		addModule(name, what)
	}
}

func Find(ctx *Orbit, files []string) (interface{}, string, error) {
	return finder(ctx, files)
}

// OnInit registers a callback for when the program starts up
func OnInit(call func(*Orbit)) {
	inits = append(inits, call)
}

// OnExit registers a callback for when the program shuts down
func OnExit(call func(*Orbit)) {
	exits = append(exits, call)
}

// OnFail registers a callback for when the program encounters and error
func OnFail(call func(*Orbit, error)) {
	fails = append(fails, call)
}

// OnFile registers a callback for finding required files
func OnFile(call func(*Orbit, []string) (interface{}, string, error)) {
	finder = call
}

// New creates a new Orbit runtime
func New(timeout time.Duration) *Orbit {

	orbit := &Orbit{
		Otto:    otto.New(),
		Vars:    make(map[string]interface{}),
		quit:    make(chan error),
		loop:    make(chan *task),
		timers:  make(map[*task]*task),
		modules: make(map[string]otto.Value),
		timeout: timeout * time.Millisecond,
	}

	orbit.Interrupt = make(chan func(), 1)

	return orbit

}

// Def sets a global variable in the runtime
func (ctx *Orbit) Def(name string, item interface{}) {
	obj, _ := ctx.Get("global")
	obj.Object().Set(name, item)
	ctx.Set(name, item)
}

// Run executes some code. Code may be a string or a byte slice.
func (ctx *Orbit) Run(name string, code interface{}) (err error) {

	defer func() {

		var ok bool

		if err, ok = recover().(error); ok {
			ctx.fail(err)
		}

		ctx.exit()

		if ctx.timer != nil {
			ctx.timer.Stop()
		}

		for timer := range ctx.timers {
			timer.timer.Stop()
			delete(ctx.timers, timer)
		}

	}()

	ctx.SetStackDepthLimit(20000)

	ctx.tick()

	ctx.init()

	// Run main code
	_, err = main(code, name)(ctx)
	if err != nil {
		panic(err)
	}

	// Wait for timers
	err = ctx.wait()
	if err != nil {
		panic(err)
	}

	return

}

func (ctx *Orbit) Quit(err error) {
	panic(err)
}

func (ctx *Orbit) init() {
	for _, e := range inits {
		e(ctx)
	}
}

func (ctx *Orbit) exit() {
	for _, e := range exits {
		e(ctx)
	}
}

func (ctx *Orbit) fail(err error) {
	for _, e := range fails {
		e(ctx, err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"sync"
	"sync/atomic"

	"github.com/caelifer/dups/balancer"
	"github.com/caelifer/dups/fstree"
	"github.com/caelifer/dups/mapreduce"
)

// Dup type describes found duplicat file
type Dup struct {
	Count int    // Number of identical copies for the hash
	Size  int64  // File size
	Hash  string // Crypto signature
	Path  string // Paths with matching signatures
}

// Value implements mapreduce.Value interface
func (d Dup) Value() interface{} {
	return d
}

func (d Dup) String() string {
	return fmt.Sprintf("%s:%d:%d:%q", d.Hash, d.Count, d.Size, d.Path)
}

// Global stats for activity report
var stats struct {
	TotlalNodes      uint64
	TotalCopies      uint64
	TotalWastedSpace uint64
}

// Global pool manager interfaced via WorkQueue
var WorkQueue = balancer.NewWorkQueue()

// Flags
var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
var memprofile = flag.String("memprofile", "", "write memory profile to this file")

func main() {
	// First parse flags
	flag.Parse()

	// Use all available CPU cores
	runtime.GOMAXPROCS(runtime.NumCPU())

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal(err)
		}
		defer func() {
			pprof.WriteHeapProfile(f)
			f.Close()
		}()
	}

	// Process command line params
	paths := flag.Args()

	if len(paths) == 0 {
		// Default is current directory
		paths = []string{"."}
	}

	log.Printf("INFO finding dups in %+v", paths)

	// Channel interfaces between Map() and Reduce() functions
	var keyValChan <-chan mapreduce.KeyValue
	var valChan <-chan mapreduce.Value

	// Start map-reduce and remove duplicate path nodes
	keyValChan = mapreduce.Map(makeNodeMapFnWithPaths(paths))
	valChan = mapreduce.Reduce(keyValChan, reduceByFileName)

	// Map by filesize
	keyValChan = mapreduce.Map(makeFileSizeMapFnFrom(valChan))
	valChan = mapreduce.Reduce(keyValChan, reduceByFileSize)

	// Map by fast SHA1 hash (first 1024 bytes)
	keyValChan = mapreduce.Map(makeFileHashMapFnFrom(valChan, true))
	valChan = mapreduce.Reduce(keyValChan, reduceByHash)

	// Map by SHA1 hash (full file hashing)
	keyValChan = mapreduce.Map(makeFileHashMapFnFrom(valChan, false))
	valChan = mapreduce.Reduce(keyValChan, reduceByHash)

	// Final reduce before reporting
	dups := make(chan Dup)
	go func(out chan<- Dup) {
		byHash := make(map[string][]*Node)

		for x := range valChan {
			n := x.Value().(*Node) // Type assert

			// Aggregate
			if v, ok := byHash[n.Hash]; ok {
				// Found node with the same file size
				byHash[n.Hash] = append(v, n)
			} else {
				byHash[n.Hash] = []*Node{n}
			}
		}

		// Reduce
		for hash, nodes := range byHash {
			count := len(nodes)
			if count > 1 {
				// Update free size stats
				atomic.AddUint64(&stats.TotalWastedSpace, uint64(nodes[0].Size*int64(count-1)))

				for _, node := range nodes {
					// Update dups number stats
					atomic.AddUint64(&stats.TotalCopies, uint64(1))
					out <- Dup{Count: count, Size: node.Size, Hash: hash, Path: node.Path}
				}
			}
		}
		close(out)
	}(dups)

	// Report
	for d := range dups {
		fmt.Println(d)
	}
	// Stats report
	log.Printf("Examined %d files, found %d dups, total wasted space %.2fGB\n",
		stats.TotlalNodes, stats.TotalCopies, float64(stats.TotalWastedSpace)/(1024*1024*1024))
}

// makeNodeMapFnWithPaths
func makeNodeMapFnWithPaths(paths []string) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		// Process all command line paths
		for _, path_ := range paths {
			// err := filepath.Walk(path_, func(path string, info os.FileInfo, err error) error {
			err := fstree.Walk(WorkQueue, path_, func(path string, info os.FileInfo, err error) error {
				// Handle passthrough error
				if err != nil {
					log.Println("WARN", err)
					return nil
				}

				// Only process simple files
				if IsFile(info) {
					size := info.Size()
					out <- mapreduce.NewKVType(
						mapreduce.KeyTypeFromString(path),
						&Node{Path: path, Size: size},
					)
				}
				return nil
			})

			if err != nil {
				log.Fatal(err)
			}
		}
	}
}

// reduceByFileName custom function remove nodes with duplicate paths
func reduceByFileName(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	byName := make(map[mapreduce.KeyType]mapreduce.Value)

	for x := range in {
		path := x.Key()           // Get key
		node := x.Value().(*Node) // Assert type

		// Add values to the map for aggregation, skip nodes with the same path
		if _, ok := byName[path]; !ok {
			byName[path] = node
		}
	}

	// Send potential duplicates one-by-one to the downstream processing
	for _, node := range byName {
		// Increase seen files counter
		atomic.AddUint64(&stats.TotlalNodes, 1)

		out <- node
	}
}

// Very simple function to map nodes by size
func makeFileSizeMapFnFrom(in <-chan mapreduce.Value) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		for x := range in {
			node := x.Value().(*Node) // Assert type

			out <- mapreduce.NewKVType(
				mapreduce.KeyTypeFromInt64(node.Size),
				node,
			)
		}
	}
}

// reduceByFileSize custom function to filter files by size
func reduceByFileSize(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	bySize := make(map[mapreduce.KeyType][]mapreduce.Value)

	for x := range in {
		size := x.Key()           // Get key
		node := x.Value().(*Node) // Assert type

		// Add values to the map for aggregation
		if v, ok := bySize[size]; ok {
			// Found node with the same file size
			bySize[size] = append(v, node)
		} else {
			bySize[size] = []mapreduce.Value{node}
		}
	}

	// Send potential duplicates one-by-one to the downstream processing
	for _, nodes := range bySize {
		// Found cluster
		if len(nodes) > 1 {
			for _, n := range nodes {
				out <- n
			}
		}
	}
}

func makeFileHashMapFnFrom(in <-chan mapreduce.Value, fast bool) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		var wg sync.WaitGroup
		for x := range in {
			node := x.Value().(*Node) // Assert type

			// Add to wait group
			wg.Add(1)

			// Calculate hash using balancer
			go func(n *Node) {
				WorkQueue <- func() {
					defer wg.Done() // Signal done

					// Default hash value for files < 1024 if fast is true
					hash := "0h"

					// Little optimization to avoid calculating fast hash on small files
					if !fast || node.Size > blockSize {
						// Always calculate hash if fast == false or file size > blockSize
						hash = n.calculateHash(fast) // Fast hash calculation - SHA1 of first 1024 bytes
					}

					// Don't process files for which we failed to calculate SHA1 hash
					if hash == "" {
						// log.Printf("WARN Unable calculate SHA1 hash for %q\n", node.Path)
						return
					}

					out <- mapreduce.NewKVType(
						mapreduce.KeyTypeFromString(hash),
						n,
					)
				}
			}(node)
		}
		// Wait for all results be submitted
		wg.Wait()
	}
}

func reduceByHash(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	byHash := make(map[mapreduce.KeyType][]*Node)

	for x := range in {
		hash := x.Key()
		node := x.Value().(*Node) // Assert type

		if v, ok := byHash[hash]; ok {
			// Found node with the same file size
			byHash[hash] = append(v, node)
		} else {
			byHash[hash] = []*Node{node}
		}
	}

	// Send out aggregeted results
	for hash, nodes := range byHash {
		if len(nodes) > 1 {
			// Send output for potential duplicates
			for _, node := range nodes {
				node.Hash = hash.String()
				out <- mapreduce.Value(node)
			}
		}
	}
}

func IsFile(fi os.FileInfo) bool {
	return fi.Mode()&os.ModeType == 0
}
<commit_msg>- Removed diagnostic message<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"sync"
	"sync/atomic"

	"github.com/caelifer/dups/balancer"
	"github.com/caelifer/dups/fstree"
	"github.com/caelifer/dups/mapreduce"
)

// Dup type describes found duplicat file
type Dup struct {
	Count int    // Number of identical copies for the hash
	Size  int64  // File size
	Hash  string // Crypto signature
	Path  string // Paths with matching signatures
}

// Value implements mapreduce.Value interface
func (d Dup) Value() interface{} {
	return d
}

func (d Dup) String() string {
	return fmt.Sprintf("%s:%d:%d:%q", d.Hash, d.Count, d.Size, d.Path)
}

// Global stats for activity report
var stats struct {
	TotlalNodes      uint64
	TotalCopies      uint64
	TotalWastedSpace uint64
}

// Global pool manager interfaced via WorkQueue
var WorkQueue = balancer.NewWorkQueue()

// Flags
var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
var memprofile = flag.String("memprofile", "", "write memory profile to this file")

func main() {
	// First parse flags
	flag.Parse()

	// Use all available CPU cores
	runtime.GOMAXPROCS(runtime.NumCPU())

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal(err)
		}
		defer func() {
			pprof.WriteHeapProfile(f)
			f.Close()
		}()
	}

	// Process command line params
	paths := flag.Args()

	if len(paths) == 0 {
		// Default is current directory
		paths = []string{"."}
	}

	// Channel interfaces between Map() and Reduce() functions
	var keyValChan <-chan mapreduce.KeyValue
	var valChan <-chan mapreduce.Value

	// Start map-reduce and remove duplicate path nodes
	keyValChan = mapreduce.Map(makeNodeMapFnWithPaths(paths))
	valChan = mapreduce.Reduce(keyValChan, reduceByFileName)

	// Map by filesize
	keyValChan = mapreduce.Map(makeFileSizeMapFnFrom(valChan))
	valChan = mapreduce.Reduce(keyValChan, reduceByFileSize)

	// Map by fast SHA1 hash (first 1024 bytes)
	keyValChan = mapreduce.Map(makeFileHashMapFnFrom(valChan, true))
	valChan = mapreduce.Reduce(keyValChan, reduceByHash)

	// Map by SHA1 hash (full file hashing)
	keyValChan = mapreduce.Map(makeFileHashMapFnFrom(valChan, false))
	valChan = mapreduce.Reduce(keyValChan, reduceByHash)

	// Final reduce before reporting
	dups := make(chan Dup)
	go func(out chan<- Dup) {
		byHash := make(map[string][]*Node)

		for x := range valChan {
			n := x.Value().(*Node) // Type assert

			// Aggregate
			if v, ok := byHash[n.Hash]; ok {
				// Found node with the same file size
				byHash[n.Hash] = append(v, n)
			} else {
				byHash[n.Hash] = []*Node{n}
			}
		}

		// Reduce
		for hash, nodes := range byHash {
			count := len(nodes)
			if count > 1 {
				// Update free size stats
				atomic.AddUint64(&stats.TotalWastedSpace, uint64(nodes[0].Size*int64(count-1)))

				for _, node := range nodes {
					// Update dups number stats
					atomic.AddUint64(&stats.TotalCopies, uint64(1))
					out <- Dup{Count: count, Size: node.Size, Hash: hash, Path: node.Path}
				}
			}
		}
		close(out)
	}(dups)

	// Report
	for d := range dups {
		fmt.Println(d)
	}
	// Stats report
	log.Printf("Examined %d files, found %d dups, total wasted space %.2fGB\n",
		stats.TotlalNodes, stats.TotalCopies, float64(stats.TotalWastedSpace)/(1024*1024*1024))
}

// makeNodeMapFnWithPaths
func makeNodeMapFnWithPaths(paths []string) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		// Process all command line paths
		for _, path_ := range paths {
			// err := filepath.Walk(path_, func(path string, info os.FileInfo, err error) error {
			err := fstree.Walk(WorkQueue, path_, func(path string, info os.FileInfo, err error) error {
				// Handle passthrough error
				if err != nil {
					log.Println("WARN", err)
					return nil
				}

				// Only process simple files
				if IsFile(info) {
					size := info.Size()
					out <- mapreduce.NewKVType(
						mapreduce.KeyTypeFromString(path),
						&Node{Path: path, Size: size},
					)
				}
				return nil
			})

			if err != nil {
				log.Fatal(err)
			}
		}
	}
}

// reduceByFileName custom function remove nodes with duplicate paths
func reduceByFileName(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	byName := make(map[mapreduce.KeyType]mapreduce.Value)

	for x := range in {
		path := x.Key()           // Get key
		node := x.Value().(*Node) // Assert type

		// Add values to the map for aggregation, skip nodes with the same path
		if _, ok := byName[path]; !ok {
			byName[path] = node
		}
	}

	// Send potential duplicates one-by-one to the downstream processing
	for _, node := range byName {
		// Increase seen files counter
		atomic.AddUint64(&stats.TotlalNodes, 1)

		out <- node
	}
}

// Very simple function to map nodes by size
func makeFileSizeMapFnFrom(in <-chan mapreduce.Value) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		for x := range in {
			node := x.Value().(*Node) // Assert type

			out <- mapreduce.NewKVType(
				mapreduce.KeyTypeFromInt64(node.Size),
				node,
			)
		}
	}
}

// reduceByFileSize custom function to filter files by size
func reduceByFileSize(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	bySize := make(map[mapreduce.KeyType][]mapreduce.Value)

	for x := range in {
		size := x.Key()           // Get key
		node := x.Value().(*Node) // Assert type

		// Add values to the map for aggregation
		if v, ok := bySize[size]; ok {
			// Found node with the same file size
			bySize[size] = append(v, node)
		} else {
			bySize[size] = []mapreduce.Value{node}
		}
	}

	// Send potential duplicates one-by-one to the downstream processing
	for _, nodes := range bySize {
		// Found cluster
		if len(nodes) > 1 {
			for _, n := range nodes {
				out <- n
			}
		}
	}
}

func makeFileHashMapFnFrom(in <-chan mapreduce.Value, fast bool) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		var wg sync.WaitGroup
		for x := range in {
			node := x.Value().(*Node) // Assert type

			// Add to wait group
			wg.Add(1)

			// Calculate hash using balancer
			go func(n *Node) {
				WorkQueue <- func() {
					defer wg.Done() // Signal done

					// Default hash value for files < 1024 if fast is true
					hash := "0h"

					// Little optimization to avoid calculating fast hash on small files
					if !fast || node.Size > blockSize {
						// Always calculate hash if fast == false or file size > blockSize
						hash = n.calculateHash(fast) // Fast hash calculation - SHA1 of first 1024 bytes
					}

					// Don't process files for which we failed to calculate SHA1 hash
					if hash == "" {
						// log.Printf("WARN Unable calculate SHA1 hash for %q\n", node.Path)
						return
					}

					out <- mapreduce.NewKVType(
						mapreduce.KeyTypeFromString(hash),
						n,
					)
				}
			}(node)
		}
		// Wait for all results be submitted
		wg.Wait()
	}
}

func reduceByHash(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	byHash := make(map[mapreduce.KeyType][]*Node)

	for x := range in {
		hash := x.Key()
		node := x.Value().(*Node) // Assert type

		if v, ok := byHash[hash]; ok {
			// Found node with the same file size
			byHash[hash] = append(v, node)
		} else {
			byHash[hash] = []*Node{node}
		}
	}

	// Send out aggregeted results
	for hash, nodes := range byHash {
		if len(nodes) > 1 {
			// Send output for potential duplicates
			for _, node := range nodes {
				node.Hash = hash.String()
				out <- mapreduce.Value(node)
			}
		}
	}
}

func IsFile(fi os.FileInfo) bool {
	return fi.Mode()&os.ModeType == 0
}
<|endoftext|>


<commit_before>//-----------------------------------------------------------------------------
// Package membership:
//-----------------------------------------------------------------------------

package main

//-----------------------------------------------------------------------------
// Package factored import statement:
//-----------------------------------------------------------------------------

import (

	// Standard library:
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"text/template"

	// Community:
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/packethost/packngo"
	"gopkg.in/alecthomas/kingpin.v2"
)

//----------------------------------------------------------------------------
// Typedefs:
//----------------------------------------------------------------------------

type Udata struct {
	Hostname  string
	Hostid    string
	Domain    string
	Role      string
	Ns1apikey string
	Fleettags string
	CAcert    string
	EtcdTkn   string
}

//-----------------------------------------------------------------------------
// Package variable declarations factored into a block:
//-----------------------------------------------------------------------------

var (

	//-----------------------------
	// coreseed: top level command
	//-----------------------------

	app = kingpin.New("coreseed", "Coreseed defines and deploys CoreOS clusters.")

	//-----------------------
	// udata: nested command
	//-----------------------

	cmdUdata = app.Command("udata", "Generate CoreOS cloud-config user-data.")

	flHostName = cmdUdata.Flag("hostname", "Short host name as in (hostname -s).").
			Required().PlaceHolder("CS_HOSTNAME").
			OverrideDefaultFromEnvar("CS_HOSTNAME").
			Short('h').String()

	flDomain = cmdUdata.Flag("domain", "Domain name as in (hostname -d).").
			Required().PlaceHolder("CS_DOMAIN").
			OverrideDefaultFromEnvar("CS_DOMAIN").
			Short('d').String()

	flHostRole = cmdUdata.Flag("role", "Choose one of [ master | node | edge ].").
			Required().PlaceHolder("CS_ROLE").
			OverrideDefaultFromEnvar("CS_ROLE").
			Short('r').String()

	flNs1Apikey = cmdUdata.Flag("ns1-api-key", "NS1 private API key.").
			Required().PlaceHolder("CS_NS1_KEY").
			OverrideDefaultFromEnvar("CS_NS1_KEY").
			Short('k').String()

	flFleetTags = cmdUdata.Flag("tags", "Comma separated list of fleet tags.").
			PlaceHolder("CS_TAGS").
			OverrideDefaultFromEnvar("CS_TAGS").
			Short('t').String()

	flCAcert = cmdUdata.Flag("ca-cert", "Path to CA certificate.").
			PlaceHolder("CS_CA_CERT").
			OverrideDefaultFromEnvar("CS_CA_CERT").
			Short('c').String()

	flEtcdTkn = cmdUdata.Flag("etcd-token", "Provide an etcd discovery token.").
			PlaceHolder("CS_ETCD_TKN").
			OverrideDefaultFromEnvar("CS_ETCD_TKN").
			Short('e').String()

	//----------------------------
	// run-packet: nested command
	//----------------------------

	cmdRunPacket = app.Command("run-packet", "Starts a CoreOS instance on Packet.net")

	flPktApiKey = cmdRunPacket.Flag("api-key", "Packet API key.").
			Required().PlaceHolder("PKT_APIKEY").
			OverrideDefaultFromEnvar("PKT_APIKEY").
			Short('k').String()

	flPktHostName = cmdRunPacket.Flag("hostname", "For the Packet.net dashboard.").
			Required().PlaceHolder("PKT_HOSTNAME").
			OverrideDefaultFromEnvar("PKT_HOSTNAME").
			Short('h').String()

	flPktProjId = cmdRunPacket.Flag("project-id", "Format: aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee").
			Required().PlaceHolder("PKT_PROJID").
			OverrideDefaultFromEnvar("PKT_PROJID").
			Short('i').String()

	flPktPlan = cmdRunPacket.Flag("plan", "One of [ baremetal_0 | baremetal_1 | baremetal_2 | baremetal_3 ]").
			Required().PlaceHolder("PKT_PLAN").
			OverrideDefaultFromEnvar("PKT_PLAN").
			Short('p').String()

	flPktOsys = cmdRunPacket.Flag("os", "One of [ coreos_stable | coreos_beta | coreos_alpha ]").
			Required().PlaceHolder("PKT_OS").
			OverrideDefaultFromEnvar("PKT_OS").
			Short('o').String()

	flPktFacility = cmdRunPacket.Flag("facility", "One of [ ewr1 | ams1 ]").
			Required().PlaceHolder("PKT_FACILITY").
			OverrideDefaultFromEnvar("PKT_FACILITY").
			Short('f').String()

	flPktBilling = cmdRunPacket.Flag("billing", "One of [ hourly | monthly ]").
			Required().PlaceHolder("PKT_BILLING").
			OverrideDefaultFromEnvar("PKT_BILLING").
			Short('b').String()

	//-------------------------
	// run-ec2: nested command
	//-------------------------

	cmdRunEc2 = app.Command("run-ec2", "Starts a CoreOS instance on Amazon EC2")

	flEc2Region = cmdRunEc2.Flag("region", "EC2 region.").
			Required().PlaceHolder("EC2_REGION").
			OverrideDefaultFromEnvar("EC2_REGION").
			Short('r').String()

	flEc2ImageId = cmdRunEc2.Flag("image-id", "EC2 image id.").
			Required().PlaceHolder("EC2_IMAGE_ID").
			OverrideDefaultFromEnvar("EC2_IMAGE_ID").
			Short('i').String()

	flEc2InsType = cmdRunEc2.Flag("instance-type", "EC2 instance type.").
			Required().PlaceHolder("EC2_INSTANCE_TYPE").
			OverrideDefaultFromEnvar("EC2_INSTANCE_TYPE").
			Short('t').String()

	flEc2KeyPair = cmdRunEc2.Flag("key-pair", "EC2 key pair.").
			Required().PlaceHolder("EC2_KEY_PAIR").
			OverrideDefaultFromEnvar("EC2_KEY_PAIR").
			Short('k').String()

	flEc2VpcId = cmdRunEc2.Flag("vpc-id", "EC2 VPC id.").
			Required().PlaceHolder("EC2_VPC_ID").
			OverrideDefaultFromEnvar("EC2_VPC_ID").
			Short('v').String()

	flEc2SubnetId = cmdRunEc2.Flag("subnet-id", "EC2 subnet id.").
			Required().PlaceHolder("EC2_SUBNET_ID").
			OverrideDefaultFromEnvar("EC2_SUBNET_ID").
			Short('s').String()

	flEc2SecGroupId = cmdRunEc2.Flag("sec-group-ids", "EC2 security group ids.").
			Required().PlaceHolder("EC2_SEC_GROUP_IDS").
			OverrideDefaultFromEnvar("EC2_SEC_GROUP_IDS").
			Short('g').String()
)

//----------------------------------------------------------------------------
// Entry point:
//----------------------------------------------------------------------------

func main() {

	// Sub-command selector:
	switch kingpin.MustParse(app.Parse(os.Args[1:])) {

	// coreseed udata ...
	case cmdUdata.FullCommand():
		cmd_udata()

	// coreseed run-packet ...
	case cmdRunPacket.FullCommand():
		cmd_run_packet()

	// coreseed run-ec2 ...
	case cmdRunEc2.FullCommand():
		cmd_run_ec2()
	}
}

//--------------------------------------------------------------------------
// func: cmd_udata
//--------------------------------------------------------------------------

func cmd_udata() {

	// Template udata structure:
	udata := Udata{
		Hostname:  *flHostName,
		Hostid:    string((*flHostName)[strings.LastIndex(*flHostName, "-")+1:]),
		Domain:    *flDomain,
		Role:      *flHostRole,
		Ns1apikey: *flNs1Apikey,
		Fleettags: *flFleetTags,
		EtcdTkn:   *flEtcdTkn,
	}

	// Read the CA certificate:
	if *flCAcert != "" {
		dat, err := ioutil.ReadFile(*flCAcert)
		checkError(err)
		udata.CAcert = strings.TrimSpace(strings.Replace(string(dat), "\n", "\n    ", -1))
	}

	// Render the template for the selected role:
	switch *flHostRole {
	case "master":
		t := template.New("master_udata")
		t, err := t.Parse(templ_master)
		err = t.Execute(os.Stdout, udata)
		checkError(err)
	case "node":
		t := template.New("node_udata")
		t, err := t.Parse(templ_node)
		err = t.Execute(os.Stdout, udata)
		checkError(err)
	case "edge":
		t := template.New("edge_udata")
		t, err := t.Parse(templ_edge)
		err = t.Execute(os.Stdout, udata)
		checkError(err)
	}
}

//--------------------------------------------------------------------------
// func: cmd_run_packet
//--------------------------------------------------------------------------

func cmd_run_packet() {

	// Read user-data from stdin:
	udata, err := ioutil.ReadAll(os.Stdin)
	checkError(err)

	// Connect and authenticate to the API endpoint:
	client := packngo.NewClient("", *flPktApiKey, nil)

	// Forge the request:
	createRequest := &packngo.DeviceCreateRequest{
		HostName:     *flPktHostName,
		Plan:         *flPktPlan,
		Facility:     *flPktFacility,
		OS:           *flPktOsys,
		BillingCycle: *flPktBilling,
		ProjectID:    *flPktProjId,
		UserData:     string(udata),
	}

	// Send the request:
	newDevice, _, err := client.Devices.Create(createRequest)
	checkError(err)

	// Pretty-print the response data:
	fmt.Println(newDevice)
}

//--------------------------------------------------------------------------
// func: cmd_run_ec2
//--------------------------------------------------------------------------

func cmd_run_ec2() {

	// Read user-data from stdin:
	udata, err := ioutil.ReadAll(os.Stdin)
	checkError(err)

	// Connect and authenticate to the API endpoint:
	svc := ec2.New(session.New(&aws.Config{Region: aws.String(*flEc2Region)}))

	// Send the request:
	runResult, err := svc.RunInstances(&ec2.RunInstancesInput{
		ImageId:      aws.String(*flEc2ImageId),
		InstanceType: aws.String(*flEc2InsType),
		MinCount:     aws.Int64(1),
		MaxCount:     aws.Int64(1),
		KeyName:      aws.String(*flEc2KeyPair),
		SubnetId:     aws.String(*flEc2SubnetId),
		UserData:     aws.String(base64.StdEncoding.EncodeToString([]byte(udata))),
	})

	checkError(err)

	// Pretty-print the response data:
	fmt.Println("Created instance", *runResult.Instances[0].InstanceId)
}

//---------------------------------------------------------------------------
// func: checkError
//---------------------------------------------------------------------------

func checkError(err error) {
	if err != nil {
		fmt.Println("Fatal error ", err.Error())
		os.Exit(1)
	}
}
<commit_msg>--sec-group-ids implementation<commit_after>//-----------------------------------------------------------------------------
// Package membership:
//-----------------------------------------------------------------------------

package main

//-----------------------------------------------------------------------------
// Package factored import statement:
//-----------------------------------------------------------------------------

import (

	// Standard library:
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"text/template"

	// Community:
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/packethost/packngo"
	"gopkg.in/alecthomas/kingpin.v2"
)

//----------------------------------------------------------------------------
// Typedefs:
//----------------------------------------------------------------------------

type Udata struct {
	Hostname  string
	Hostid    string
	Domain    string
	Role      string
	Ns1apikey string
	Fleettags string
	CAcert    string
	EtcdTkn   string
}

//-----------------------------------------------------------------------------
// Package variable declarations factored into a block:
//-----------------------------------------------------------------------------

var (

	//-----------------------------
	// coreseed: top level command
	//-----------------------------

	app = kingpin.New("coreseed", "Coreseed defines and deploys CoreOS clusters.")

	//-----------------------
	// udata: nested command
	//-----------------------

	cmdUdata = app.Command("udata", "Generate CoreOS cloud-config user-data.")

	flHostName = cmdUdata.Flag("hostname", "Short host name as in (hostname -s).").
			Required().PlaceHolder("CS_HOSTNAME").
			OverrideDefaultFromEnvar("CS_HOSTNAME").
			Short('h').String()

	flDomain = cmdUdata.Flag("domain", "Domain name as in (hostname -d).").
			Required().PlaceHolder("CS_DOMAIN").
			OverrideDefaultFromEnvar("CS_DOMAIN").
			Short('d').String()

	flHostRole = cmdUdata.Flag("role", "Choose one of [ master | node | edge ].").
			Required().PlaceHolder("CS_ROLE").
			OverrideDefaultFromEnvar("CS_ROLE").
			Short('r').String()

	flNs1Apikey = cmdUdata.Flag("ns1-api-key", "NS1 private API key.").
			Required().PlaceHolder("CS_NS1_KEY").
			OverrideDefaultFromEnvar("CS_NS1_KEY").
			Short('k').String()

	flFleetTags = cmdUdata.Flag("tags", "Comma separated list of fleet tags.").
			PlaceHolder("CS_TAGS").
			OverrideDefaultFromEnvar("CS_TAGS").
			Short('t').String()

	flCAcert = cmdUdata.Flag("ca-cert", "Path to CA certificate.").
			PlaceHolder("CS_CA_CERT").
			OverrideDefaultFromEnvar("CS_CA_CERT").
			Short('c').String()

	flEtcdTkn = cmdUdata.Flag("etcd-token", "Provide an etcd discovery token.").
			PlaceHolder("CS_ETCD_TKN").
			OverrideDefaultFromEnvar("CS_ETCD_TKN").
			Short('e').String()

	//----------------------------
	// run-packet: nested command
	//----------------------------

	cmdRunPacket = app.Command("run-packet", "Starts a CoreOS instance on Packet.net")

	flPktApiKey = cmdRunPacket.Flag("api-key", "Packet API key.").
			Required().PlaceHolder("PKT_APIKEY").
			OverrideDefaultFromEnvar("PKT_APIKEY").
			Short('k').String()

	flPktHostName = cmdRunPacket.Flag("hostname", "For the Packet.net dashboard.").
			Required().PlaceHolder("PKT_HOSTNAME").
			OverrideDefaultFromEnvar("PKT_HOSTNAME").
			Short('h').String()

	flPktProjId = cmdRunPacket.Flag("project-id", "Format: aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee").
			Required().PlaceHolder("PKT_PROJID").
			OverrideDefaultFromEnvar("PKT_PROJID").
			Short('i').String()

	flPktPlan = cmdRunPacket.Flag("plan", "One of [ baremetal_0 | baremetal_1 | baremetal_2 | baremetal_3 ]").
			Required().PlaceHolder("PKT_PLAN").
			OverrideDefaultFromEnvar("PKT_PLAN").
			Short('p').String()

	flPktOsys = cmdRunPacket.Flag("os", "One of [ coreos_stable | coreos_beta | coreos_alpha ]").
			Required().PlaceHolder("PKT_OS").
			OverrideDefaultFromEnvar("PKT_OS").
			Short('o').String()

	flPktFacility = cmdRunPacket.Flag("facility", "One of [ ewr1 | ams1 ]").
			Required().PlaceHolder("PKT_FACILITY").
			OverrideDefaultFromEnvar("PKT_FACILITY").
			Short('f').String()

	flPktBilling = cmdRunPacket.Flag("billing", "One of [ hourly | monthly ]").
			Required().PlaceHolder("PKT_BILLING").
			OverrideDefaultFromEnvar("PKT_BILLING").
			Short('b').String()

	//-------------------------
	// run-ec2: nested command
	//-------------------------

	cmdRunEc2 = app.Command("run-ec2", "Starts a CoreOS instance on Amazon EC2")

	flEc2Region = cmdRunEc2.Flag("region", "EC2 region.").
			Required().PlaceHolder("EC2_REGION").
			OverrideDefaultFromEnvar("EC2_REGION").
			Short('r').String()

	flEc2ImageId = cmdRunEc2.Flag("image-id", "EC2 image id.").
			Required().PlaceHolder("EC2_IMAGE_ID").
			OverrideDefaultFromEnvar("EC2_IMAGE_ID").
			Short('i').String()

	flEc2InsType = cmdRunEc2.Flag("instance-type", "EC2 instance type.").
			Required().PlaceHolder("EC2_INSTANCE_TYPE").
			OverrideDefaultFromEnvar("EC2_INSTANCE_TYPE").
			Short('t').String()

	flEc2KeyPair = cmdRunEc2.Flag("key-pair", "EC2 key pair.").
			Required().PlaceHolder("EC2_KEY_PAIR").
			OverrideDefaultFromEnvar("EC2_KEY_PAIR").
			Short('k').String()

	flEc2VpcId = cmdRunEc2.Flag("vpc-id", "EC2 VPC id.").
			Required().PlaceHolder("EC2_VPC_ID").
			OverrideDefaultFromEnvar("EC2_VPC_ID").
			Short('v').String()

	flEc2SubnetId = cmdRunEc2.Flag("subnet-id", "EC2 subnet id.").
			Required().PlaceHolder("EC2_SUBNET_ID").
			OverrideDefaultFromEnvar("EC2_SUBNET_ID").
			Short('s').String()

	flEc2SecGrpIds = cmdRunEc2.Flag("sec-group-ids", "EC2 security group ids.").
			Required().PlaceHolder("EC2_SEC_GROUP_IDS").
			OverrideDefaultFromEnvar("EC2_SEC_GROUP_IDS").
			Short('g').String()
)

//----------------------------------------------------------------------------
// Entry point:
//----------------------------------------------------------------------------

func main() {

	// Sub-command selector:
	switch kingpin.MustParse(app.Parse(os.Args[1:])) {

	// coreseed udata ...
	case cmdUdata.FullCommand():
		cmd_udata()

	// coreseed run-packet ...
	case cmdRunPacket.FullCommand():
		cmd_run_packet()

	// coreseed run-ec2 ...
	case cmdRunEc2.FullCommand():
		cmd_run_ec2()
	}
}

//--------------------------------------------------------------------------
// func: cmd_udata
//--------------------------------------------------------------------------

func cmd_udata() {

	// Template udata structure:
	udata := Udata{
		Hostname:  *flHostName,
		Hostid:    string((*flHostName)[strings.LastIndex(*flHostName, "-")+1:]),
		Domain:    *flDomain,
		Role:      *flHostRole,
		Ns1apikey: *flNs1Apikey,
		Fleettags: *flFleetTags,
		EtcdTkn:   *flEtcdTkn,
	}

	// Read the CA certificate:
	if *flCAcert != "" {
		dat, err := ioutil.ReadFile(*flCAcert)
		checkError(err)
		udata.CAcert = strings.TrimSpace(strings.Replace(string(dat), "\n", "\n    ", -1))
	}

	// Render the template for the selected role:
	switch *flHostRole {
	case "master":
		t := template.New("master_udata")
		t, err := t.Parse(templ_master)
		err = t.Execute(os.Stdout, udata)
		checkError(err)
	case "node":
		t := template.New("node_udata")
		t, err := t.Parse(templ_node)
		err = t.Execute(os.Stdout, udata)
		checkError(err)
	case "edge":
		t := template.New("edge_udata")
		t, err := t.Parse(templ_edge)
		err = t.Execute(os.Stdout, udata)
		checkError(err)
	}
}

//--------------------------------------------------------------------------
// func: cmd_run_packet
//--------------------------------------------------------------------------

func cmd_run_packet() {

	// Read user-data from stdin:
	udata, err := ioutil.ReadAll(os.Stdin)
	checkError(err)

	// Connect and authenticate to the API endpoint:
	client := packngo.NewClient("", *flPktApiKey, nil)

	// Forge the request:
	createRequest := &packngo.DeviceCreateRequest{
		HostName:     *flPktHostName,
		Plan:         *flPktPlan,
		Facility:     *flPktFacility,
		OS:           *flPktOsys,
		BillingCycle: *flPktBilling,
		ProjectID:    *flPktProjId,
		UserData:     string(udata),
	}

	// Send the request:
	newDevice, _, err := client.Devices.Create(createRequest)
	checkError(err)

	// Pretty-print the response data:
	fmt.Println(newDevice)
}

//--------------------------------------------------------------------------
// func: cmd_run_ec2
//--------------------------------------------------------------------------

func cmd_run_ec2() {

	// Read user-data from stdin:
	udata, err := ioutil.ReadAll(os.Stdin)
	checkError(err)

	// Connect and authenticate to the API endpoint:
	svc := ec2.New(session.New(&aws.Config{Region: aws.String(*flEc2Region)}))

	// Forge the SecurityGroupIds:
	var secGrps []*string
	for _, gid := range strings.Split(*flEc2SecGrpIds, ",") {
		secGrps = append(secGrps, aws.String(gid))
	}

	// Send the request:
	runResult, err := svc.RunInstances(&ec2.RunInstancesInput{
		ImageId:          aws.String(*flEc2ImageId),
		InstanceType:     aws.String(*flEc2InsType),
		MinCount:         aws.Int64(1),
		MaxCount:         aws.Int64(1),
		KeyName:          aws.String(*flEc2KeyPair),
		SubnetId:         aws.String(*flEc2SubnetId),
		SecurityGroupIds: secGrps,
		UserData:         aws.String(base64.StdEncoding.EncodeToString([]byte(udata))),
	})

	checkError(err)

	// Pretty-print the response data:
	fmt.Println("Created instance", *runResult.Instances[0].InstanceId)
}

//---------------------------------------------------------------------------
// func: checkError
//---------------------------------------------------------------------------

func checkError(err error) {
	if err != nil {
		fmt.Println("Fatal error ", err.Error())
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"

	"os"

	"strconv"

	"io/ioutil"

	"./db"
)

type Config struct {
	// Host for the app to listen on. May be empty to listen on all interfaces
	ListenHost string
	// Port for the app to listen on. Default 8888
	ListenPort             int
	ClientID               string
	ClientSecret           string
	SlackVerificationToken string
}

var config Config

func loadConfig() {
	port, _ := strconv.Atoi(os.Getenv("UNFURLER_PORT"))

	config = Config{
		ListenHost:             os.Getenv("UNFURLER_HOST"),
		ListenPort:             port,
		ClientID:               os.Getenv("UNFURLER_CLIENT_ID"),
		ClientSecret:           os.Getenv("UNFURLER_CLIENT_SECRET"),
		SlackVerificationToken: os.Getenv("UNFURLER_VERIFICATION_TOKEN"),
	}

	// set defaults and validate
	if config.ListenPort == 0 {
		config.ListenPort = 8888
	}

	if config.ClientID == "" {
		log.Fatal("UNFURLER_CLIENT_ID is not set")
	}
	if config.ClientSecret == "" {
		log.Fatal("UNFURLER_CLIENT_SECRET is not set")
	}
	if config.SlackVerificationToken == "" {
		log.Fatal("UNFURLER_VERIFICATION_TOKEN is not set")
	}
}

func serveFile(w http.ResponseWriter, path string) {
	content, err := ioutil.ReadFile(path)
	if err != nil {
		log.Printf("Could not serve file %s: %s", path, err.Error())
		http.Error(w, err.Error(), 500)
		return
	}
	w.Write(content)
}

func main() {
	loadConfig()
	db.Init()
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		serveFile(w, "static/index.html")
	})
	http.HandleFunc("/slack", slackEventHandler)
	http.HandleFunc("/oauth", oauthCallbackHandler)
	http.HandleFunc("/slash", slashCommandHandler)
	log.Printf("Unfurler listening on %s:%d...", config.ListenHost, config.ListenPort)
	log.Fatal(http.ListenAndServe(fmt.Sprintf("%s:%d", config.ListenHost, config.ListenPort), nil))
}
<commit_msg>make configdata private<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"

	"./db"

	"os"

	"strconv"

	"io/ioutil"
)

type configData struct {
	// Host for the app to listen on. May be empty to listen on all interfaces
	ListenHost string
	// Port for the app to listen on. Default 8888
	ListenPort             int
	ClientID               string
	ClientSecret           string
	SlackVerificationToken string
}

var config configData

func loadConfig() {
	port, _ := strconv.Atoi(os.Getenv("UNFURLER_PORT"))

	config = configData{
		ListenHost:             os.Getenv("UNFURLER_HOST"),
		ListenPort:             port,
		ClientID:               os.Getenv("UNFURLER_CLIENT_ID"),
		ClientSecret:           os.Getenv("UNFURLER_CLIENT_SECRET"),
		SlackVerificationToken: os.Getenv("UNFURLER_VERIFICATION_TOKEN"),
	}

	// set defaults and validate
	if config.ListenPort == 0 {
		config.ListenPort = 8888
	}

	if config.ClientID == "" {
		log.Fatal("UNFURLER_CLIENT_ID is not set")
	}
	if config.ClientSecret == "" {
		log.Fatal("UNFURLER_CLIENT_SECRET is not set")
	}
	if config.SlackVerificationToken == "" {
		log.Fatal("UNFURLER_VERIFICATION_TOKEN is not set")
	}
}

func serveFile(w http.ResponseWriter, path string) {
	content, err := ioutil.ReadFile(path)
	if err != nil {
		log.Printf("Could not serve file %s: %s", path, err.Error())
		http.Error(w, err.Error(), 500)
		return
	}
	w.Write(content)
}

func main() {
	loadConfig()
	db.Init()
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		serveFile(w, "static/index.html")
	})
	http.HandleFunc("/slack", slackEventHandler)
	http.HandleFunc("/oauth", oauthCallbackHandler)
	http.HandleFunc("/slash", slashCommandHandler)
	log.Printf("Unfurler listening on %s:%d...", config.ListenHost, config.ListenPort)
	log.Fatal(http.ListenAndServe(fmt.Sprintf("%s:%d", config.ListenHost, config.ListenPort), nil))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"time"

	"golang.org/x/oauth2"

	"github.com/google/go-github/github"
	"github.com/kelseyhightower/envconfig"
)

type commits []struct {
	Commit struct {
		Author struct {
			Name  string    `json:"name"`
			Email string    `json:"email"`
			Date  time.Time `json:"date"`
		} `json:"author"`
		Committer struct {
			Name  string    `json:"name"`
			Email string    `json:"email"`
			Date  time.Time `json:"date"`
		} `json:"committer"`
	} `json:"commit"`
}

type contribs []struct {
	Login         string `json:"login"`
	Contributions int    `json:"contributions"`
}

type stat struct {
	email   []string
	commits int
}

type Config struct {
	Token string
}

var org = flag.String("o", "", "github orginization")
var repo = flag.String("r", "", "github repository")
var check = flag.String("c", "top100", "Type of check to run (top100 (default), allcommits, apirates")

func main() {
	flag.Parse()

	if *org == "" {
		log.Fatal("need to specify an org")
	}
	if *repo == "" {
		log.Fatal("need to specify an repo")
	}

	config := &Config{}
	err := envconfig.Process("scrape", config)
	if err != nil {
		log.Fatal(err)
	}
	if config.Token == "" {
		log.Fatal("needs an access token")
	}

	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: config.Token},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)

	client := github.NewClient(tc)

	switch *check {
	case "top100":
		top100(client)
	case "allcommits":
		getAllCommits(client)
	case "apirates":
		rateLimit(client)
	default:
		log.Fatal("not a valid check")
	}
}

func rateLimit(client *github.Client) {
	r, _, err := client.RateLimit()
	if err != nil {
		log.Printf("error getting rate: %v", err)
		return
	}
	fmt.Printf("%d/%d\n", r.Remaining, r.Limit)
}

func checkAndAddEmail(e string, emails []string) bool {
	for _, s := range emails {
		if e == s {
			return true
		}
	}
	return false
}

func getAllCommits(client *github.Client) {
	opt := &github.CommitsListOptions{
		ListOptions: github.ListOptions{
			PerPage: 100,
		},
	}

	m := make(map[string]*stat)
	for {
		commits, resp, err := client.Repositories.ListCommits(*org, *repo, opt)
		if _, ok := err.(*github.RateLimitError); ok {
			log.Println("hit rate limit")
			return
		}
		if err != nil {
			log.Fatal(err)
		}
		for _, c := range commits {
			a := "username missing"
			e := "fake@fake.com"
			if c.Author != nil {
				a = *c.Author.Login
			}
			if c.Commit.Author != nil {
				e = *c.Commit.Author.Email
			}
			_, ok := m[a]
			if !ok {
				m[a] = &stat{email: []string{e}, commits: 1}
				continue
			}
			tmp := m[a]
			tmp.commits += 1
			if !checkAndAddEmail(e, tmp.email) {
				tmp.email = append(tmp.email, e)
			}
		}
		if resp.NextPage == 0 {
			break
		}
		opt.ListOptions.Page = resp.NextPage
	}
	total := 0
	atotal := 0
	for k, v := range m {
		total += v.commits
		atotal += 1
		fmt.Printf("Author: %s Commits: %v\n", k, v)
	}
	fmt.Printf("TOTAL: %d\n", total)
	fmt.Printf("ATOTAL: %d\n", atotal)
}

func top100(client *github.Client) {
	stats, _, err := client.Repositories.ListContributorsStats(*org, *repo)
	if _, ok := err.(*github.RateLimitError); ok {
		log.Println("hit rate limit")
	}
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(len(stats))
	for _, i := range stats {
		fmt.Printf(
			"Author: %v commits: %v\n",
			*i.Author.Login,
			*i.Total,
		)
	}
}
<commit_msg>added checks for open and closed PRs<commit_after>package main

import (
	"flag"
	"fmt"
	"log"

	"golang.org/x/oauth2"

	"github.com/google/go-github/github"
	"github.com/kelseyhightower/envconfig"
)

type stat struct {
	email []string
	count int
}

type Config struct {
	Token string
}

var org = flag.String("o", "", "github orginization")
var repo = flag.String("r", "", "github repository")
var check = flag.String("c", "top100", "Type of check to run (top100 (default), allcommits, apirates")

func main() {
	flag.Parse()

	if *org == "" {
		log.Fatal("need to specify an org")
	}
	if *repo == "" {
		log.Fatal("need to specify an repo")
	}

	config := &Config{}
	err := envconfig.Process("scrape", config)
	if err != nil {
		log.Fatal(err)
	}
	if config.Token == "" {
		log.Fatal("needs an access token")
	}

	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: config.Token},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)

	client := github.NewClient(tc)

	switch *check {
	case "top100":
		top100(client)
	case "allcommits":
		getAllCommits(client)
	case "apirates":
		rateLimit(client)
	case "openprs":
		getAllOpenPRs(client)
	case "closedprs":
		getAllClosedPRs(client)
	default:
		log.Fatal("not a valid check")
	}
}

func rateLimit(client *github.Client) {
	r, _, err := client.RateLimit()
	if err != nil {
		log.Printf("error getting rate: %v", err)
		return
	}
	fmt.Printf("%d/%d\n", r.Remaining, r.Limit)
}

func checkAndAddEmail(e string, emails []string) bool {
	for _, s := range emails {
		if e == s {
			return true
		}
	}
	return false
}

func getAllOpenPRs(client *github.Client) {
	opt := &github.PullRequestListOptions{
		ListOptions: github.ListOptions{
			PerPage: 100,
		},
	}

	m := make(map[string]*stat)
	for {
		prs, resp, err := client.PullRequests.List(*org, *repo, opt)
		if _, ok := err.(*github.RateLimitError); ok {
			log.Println("hit rate limit")
			return
		}
		if err != nil {
			log.Fatal(err)
		}
		for _, pr := range prs {
			a := "username missing"
			if pr.User != nil {
				a = *pr.User.Login
			}
			_, ok := m[a]
			if !ok {
				m[a] = &stat{email: []string{}, count: 1}
				continue
			}
			tmp := m[a]
			tmp.count += 1
		}
		if resp.NextPage == 0 {
			break
		}
		opt.ListOptions.Page = resp.NextPage
	}
	total := 0
	atotal := 0
	for k, v := range m {
		total += v.count
		atotal += 1
		fmt.Printf("Author: %s count: %d\n", k, v.count)
	}
	fmt.Printf("TOTAL: %d\n", total)
	fmt.Printf("ATOTAL: %d\n", atotal)
}

func getAllClosedPRs(client *github.Client) {
	opt := &github.PullRequestListOptions{
		State: "closed",
		ListOptions: github.ListOptions{
			PerPage: 100,
		},
	}

	m := make(map[string]*stat)
	for {
		prs, resp, err := client.PullRequests.List(*org, *repo, opt)
		if _, ok := err.(*github.RateLimitError); ok {
			log.Println("hit rate limit")
			return
		}
		if err != nil {
			log.Fatal(err)
		}
		for _, pr := range prs {
			a := "username missing"
			if pr.User != nil {
				a = *pr.User.Login
			}
			_, ok := m[a]
			if !ok {
				m[a] = &stat{email: []string{}, count: 1}
				continue
			}
			tmp := m[a]
			tmp.count += 1
		}
		if resp.NextPage == 0 {
			break
		}
		opt.ListOptions.Page = resp.NextPage
	}
	total := 0
	atotal := 0
	for k, v := range m {
		total += v.count
		atotal += 1
		fmt.Printf("Author: %s count: %d\n", k, v.count)
	}
	fmt.Printf("TOTAL: %d\n", total)
	fmt.Printf("ATOTAL: %d\n", atotal)
}

func getAllCommits(client *github.Client) {
	opt := &github.CommitsListOptions{
		ListOptions: github.ListOptions{
			PerPage: 100,
		},
	}

	m := make(map[string]*stat)
	for {
		commits, resp, err := client.Repositories.ListCommits(*org, *repo, opt)
		if _, ok := err.(*github.RateLimitError); ok {
			log.Println("hit rate limit")
			return
		}
		if err != nil {
			log.Fatal(err)
		}
		for _, c := range commits {
			a := "username missing"
			e := "fake@fake.com"
			if c.Author != nil {
				a = *c.Author.Login
			}
			if c.Commit.Author != nil {
				e = *c.Commit.Author.Email
			}
			_, ok := m[a]
			if !ok {
				m[a] = &stat{email: []string{e}, count: 1}
				continue
			}
			tmp := m[a]
			tmp.count += 1
			if !checkAndAddEmail(e, tmp.email) {
				tmp.email = append(tmp.email, e)
			}
		}
		if resp.NextPage == 0 {
			break
		}
		opt.ListOptions.Page = resp.NextPage
	}
	total := 0
	atotal := 0
	for k, v := range m {
		total += v.count
		atotal += 1
		fmt.Printf("Author: %s count: %v\n", k, v)
	}
	fmt.Printf("TOTAL: %d\n", total)
	fmt.Printf("ATOTAL: %d\n", atotal)
}

func top100(client *github.Client) {
	stats, _, err := client.Repositories.ListContributorsStats(*org, *repo)
	if _, ok := err.(*github.RateLimitError); ok {
		log.Println("hit rate limit")
	}
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(len(stats))
	for _, i := range stats {
		fmt.Printf(
			"Author: %v commits: %v\n",
			*i.Author.Login,
			*i.Total,
		)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/christoph-k/go-fsevents"
)

func main() {
	var pids []int
	var pidslock sync.Mutex
	var command []string
	var exclude []string

	for i, arg := range os.Args {
		if arg == "--" {
			command = os.Args[i+1:]
		}
		if arg == "-exclude" {
			excludestr := os.Args[i+1]
			exclude = strings.Split(excludestr, ",")
			for idx, e := range exclude {
				exclude[idx] = strings.Trim(e, " ")
			}
		}
	}
	if len(command) == 0 {
		usage()
		os.Exit(1)
	}

	// intercept SIGINT and kill childprocesses
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		kill(&pids, &pidslock)
		os.Exit(0)
	}()

	w, err := fsevents.NewWatcher(".", 200)
	if err != nil {
		panic(err)
	}
	go run(&pids, &pidslock, command)
	for {
		e := <-w.EventChan
		excluded := false
		for _, excl := range exclude {
			if strings.Contains(e.Path, excl) {
				excluded = true
			}
		}
		if e.FileInfo == nil || e.FileInfo.IsDir() || excluded {
			continue
		}
		fmt.Printf("%v has changed\n", e.Path)
		kill(&pids, &pidslock)
		w.Stop()
		w, err = fsevents.NewWatcher(".", 200)
		if err != nil {
			panic(err)
		}
		go run(&pids, &pidslock, command)
	}
}

func run(pids *[]int, pidslock *sync.Mutex, command []string) {
	fmt.Printf("running %v\n", command)
	cmd := exec.Command(command[0], command[1:]...)
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Start()

	p := cmd.Process
	if p != nil {
		pidslock.Lock()
		*pids = append(*pids, p.Pid)
		pidslock.Unlock()
	}
	cmd.Wait()
}

func kill(pids *[]int, pidslock *sync.Mutex) {
	pidslock.Lock()
	defer pidslock.Unlock()
	for _, pid := range *pids {
		fmt.Printf("killing %v\n", pid)
		syscall.Kill(-pid, syscall.SIGTERM)
		time.Sleep(time.Millisecond * 100)
		syscall.Kill(-pid, syscall.SIGKILL)
	}
	*pids = make([]int, 0)
}

func usage() {
	fmt.Printf("usage: wr [-exclude \"file1, file2\"] -- <command with args>\n")
}
<commit_msg>don't check too often so save some cpu<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/christoph-k/go-fsevents"
)

func main() {
	var pids []int
	var pidslock sync.Mutex
	var command []string
	var exclude []string

	for i, arg := range os.Args {
		if arg == "--" {
			command = os.Args[i+1:]
		}
		if arg == "-exclude" {
			excludestr := os.Args[i+1]
			exclude = strings.Split(excludestr, ",")
			for idx, e := range exclude {
				exclude[idx] = strings.Trim(e, " ")
			}
		}
	}
	if len(command) == 0 {
		usage()
		os.Exit(1)
	}

	// intercept SIGINT and kill childprocesses
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		kill(&pids, &pidslock)
		os.Exit(0)
	}()

	w, err := fsevents.NewWatcher(".", time.Millisecond*200)
	if err != nil {
		panic(err)
	}
	go run(&pids, &pidslock, command)
	for {
		e := <-w.EventChan
		excluded := false
		for _, excl := range exclude {
			if strings.Contains(e.Path, excl) {
				excluded = true
			}
		}
		if e.FileInfo == nil || e.FileInfo.IsDir() || excluded {
			continue
		}
		fmt.Printf("%v has changed\n", e.Path)
		kill(&pids, &pidslock)
		w.Stop()
		w, err = fsevents.NewWatcher(".", time.Millisecond*200)
		if err != nil {
			panic(err)
		}
		go run(&pids, &pidslock, command)
	}
}

func run(pids *[]int, pidslock *sync.Mutex, command []string) {
	fmt.Printf("running %v\n", command)
	cmd := exec.Command(command[0], command[1:]...)
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Start()

	p := cmd.Process
	if p != nil {
		pidslock.Lock()
		*pids = append(*pids, p.Pid)
		pidslock.Unlock()
	}
	cmd.Wait()
}

func kill(pids *[]int, pidslock *sync.Mutex) {
	pidslock.Lock()
	defer pidslock.Unlock()
	for _, pid := range *pids {
		fmt.Printf("killing %v\n", pid)
		syscall.Kill(-pid, syscall.SIGTERM)
		time.Sleep(time.Millisecond * 100)
		syscall.Kill(-pid, syscall.SIGKILL)
	}
	*pids = make([]int, 0)
}

func usage() {
	fmt.Printf("usage: wr [-exclude \"file1, file2\"] -- <command with args>\n")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"strings"
	"text/template"
)

type staredRepos struct {
	Name        string `json:"name"`
	URL         string `json:"html_url"`
	Description string `json:"description"`
}

var port string

func init() {
	flag.StringVar(&port, "port", ":8001", "Port to listen on")
}

func main() {
	flag.Parse()

	http.HandleFunc("/", showStar)
	fs := http.FileServer(http.Dir("assets"))
	http.Handle("/assets/", http.StripPrefix("/assets/", fs))
	http.HandleFunc("/favicon.ico", handleFavicon)
	http.ListenAndServe(port, nil)
}

func handleFavicon(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte(""))
}

func showStar(w http.ResponseWriter, r *http.Request) {

	username := strings.SplitN(r.URL.Path, "/", 3)[1]
	fmt.Printf("%v\n", username)

	resp, err := http.Get("https://api.github.com/users/" + username + "/starred")
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	fmt.Printf("%v\n\n", resp.Header.Get("Link"))

	var repos []staredRepos

	if err := json.NewDecoder(resp.Body).Decode(&repos); err != nil {
		fmt.Printf("%v\n", err)
		w.Write([]byte("Wrong username"))
		return
	}

	t, _ := template.ParseFiles("html/index.html")

	t.Execute(w, repos)
}
<commit_msg>Add function to get pages count<commit_after>package main

import (
	"encoding/json"
	"flag"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"text/template"
)

type staredRepos struct {
	Name        string `json:"name"`
	URL         string `json:"html_url"`
	Description string `json:"description"`
}

var port string

func init() {
	flag.StringVar(&port, "port", ":8001", "Port to listen on")
}

func main() {
	flag.Parse()

	http.HandleFunc("/", showStar)
	fs := http.FileServer(http.Dir("assets"))
	http.Handle("/assets/", http.StripPrefix("/assets/", fs))
	http.HandleFunc("/favicon.ico", handleFavicon)
	http.ListenAndServe(port, nil)
}

func handleFavicon(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte(""))
}

func showStar(w http.ResponseWriter, r *http.Request) {

	username := strings.SplitN(r.URL.Path, "/", 3)[1]
	log.Printf("%v\n", username)

	resp, err := http.Get("https://api.github.com/users/" + username + "/starred")
	if err != nil {
		log.Print(err)
		w.Write([]byte("Server error\n"))
		return
	}
	defer resp.Body.Close()

	pages, err := getPagesCount(resp.Header.Get("Link"))
	log.Printf("%v\n", pages)

	var repos []staredRepos

	if err := json.NewDecoder(resp.Body).Decode(&repos); err != nil {
		log.Printf("%v\n", err)
		w.Write([]byte("Wrong username"))
		return
	}

	t, _ := template.ParseFiles("html/index.html")

	t.Execute(w, repos)
}

func getPagesCount(s string) (i int64, err error) {
	re, err := regexp.Compile("page=\\d+")
	if err != nil {
		return 0, err
	}
	found := re.FindAllString(s, -1)

	re2, err := regexp.Compile("\\d+")
	if err != nil {
		return 0, err
	}

	i, err = strconv.ParseInt(re2.FindString(found[1]), 10, 64)
	if err != nil {
		return 0, err
	}

	return i, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/enaml-ops/omg-cli/aws-cli"
	"github.com/enaml-ops/omg-cli/azure-cli"
	"github.com/enaml-ops/omg-cli/bosh"
	gcpcli "github.com/enaml-ops/omg-cli/gcp-cli"
	"github.com/enaml-ops/omg-cli/photon-cli"
	"github.com/enaml-ops/omg-cli/utils"
	"github.com/enaml-ops/omg-cli/vsphere-cli"
	"github.com/enaml-ops/pluginlib/registry"
	"github.com/olekukonko/tablewriter"
	"github.com/pivotalservices/gtils/osutils"
	"github.com/xchapter7x/lo"
)

//Version of product
var Version string

// CloudConfigPluginsDir - location of cloud config plugins directory
var CloudConfigPluginsDir = "./.plugins/cloudconfig"

//ProductPluginsDir location of products plugins
var ProductPluginsDir = "./.plugins/product"

func main() {
	app := cli.NewApp()
	app.Version = Version
	app.Commands = []cli.Command{
		{
			Name:   "azure",
			Usage:  "azure [--flags] - deploy a bosh to azure",
			Action: azurecli.GetAction(BoshInitDeploy),
			Flags:  azurecli.GetFlags(),
		},
		{
			Name:   "aws",
			Usage:  "aws [--flags] - deploy a bosh to aws",
			Action: awscli.GetAction(BoshInitDeploy),
			Flags:  awscli.GetFlags(),
		},
		{
			Name:   "gcp",
			Usage:  "gcp [--flags] - deploy a bosh to GCP",
			Action: gcpcli.GetAction(BoshInitDeploy),
			Flags:  gcpcli.GetFlags(),
		},
		{
			Name:   "photon",
			Usage:  "photon [--flags] - deploy a bosh to photon",
			Action: photoncli.GetAction(BoshInitDeploy),
			Flags:  photoncli.GetFlags(),
		},
		{
			Name:   "vsphere",
			Usage:  "vsphere [--flags] - deploy a bosh to vsphere",
			Action: vspherecli.GetAction(BoshInitDeploy),
			Flags:  vspherecli.GetFlags(),
		},
		{
			Name: "list-cloudconfigs",
			Action: func(c *cli.Context) error {
				fmt.Println("Cloud Configs:")
				table := tablewriter.NewWriter(os.Stdin)
				table.SetHeader([]string{"Name", "Command", "Properties"})
				data := make([][]string, 0)
				formatProperties := func(p map[string]interface{}) string {
					var res string
					for n, v := range p {
						res += fmt.Sprintf("%v: %v\n", n, v)
					}
					return res
				}

				for _, plgn := range registry.ListCloudConfigs() {
					row := []string{
						plgn.Name,
						path.Base(plgn.Path),
						formatProperties(plgn.Properties),
					}
					data = append(data, row)
				}
				table.AppendBulk(data)
				table.Render()
				return nil
			},
		},
		{
			Name: "list-products",
			Action: func(c *cli.Context) error {
				ListProducts(os.Stdout, registry.ListProducts())
				return nil
			},
		},
		{
			Name:  "register-plugin",
			Usage: "register-plugin -type [cloudconfig, product] -pluginpath <plugin-binary>",
			Action: func(c *cli.Context) (err error) {

				if c.String("type") != "" && c.String("pluginpath") != "" {
					err = registerPlugin(c.String("type"), c.String("pluginpath"))
				}
				return
			},
			Flags: []cli.Flag{
				cli.StringFlag{Name: "type", Value: "product", Usage: "define if the plugin to be registered is a cloudconfig or a product"},
				cli.StringFlag{Name: "pluginpath", Value: "", Usage: "the path to the plugin you wish to register"},
			},
		},
		{
			Name:        "deploy-cloudconfig",
			Usage:       "deploy-cloudconfig <cloudconfig-name> [--flags] - deploy a cloudconfig to bosh",
			Flags:       bosh.GetAuthFlags(),
			Subcommands: utils.GetCloudConfigCommands(CloudConfigPluginsDir),
		},
		{
			Name:        "deploy-product",
			Usage:       "deploy-product <prod-name> [--flags] - deploy a product via bosh",
			Flags:       bosh.GetAuthFlags(),
			Subcommands: utils.GetProductCommands(ProductPluginsDir),
		},
	}
	err := app.Run(os.Args)
	if err != nil {
		fmt.Println("ERROR:", err)
		os.Exit(1)
	}
}

func init() {

	if strings.ToLower(os.Getenv("LOG_LEVEL")) != "debug" {
		log.SetOutput(ioutil.Discard)
	}
}

func registerPlugin(typename, pluginpath string) (err error) {
	var srcPlugin *os.File

	if srcPlugin, err = os.Open(pluginpath); err == nil {
		defer srcPlugin.Close()

		switch typename {
		case "cloudconfig":
			dstFilepath := path.Join(CloudConfigPluginsDir, path.Base(pluginpath))
			err = copyPlugin(srcPlugin, dstFilepath)

		case "product":
			dstFilepath := path.Join(ProductPluginsDir, path.Base(pluginpath))
			err = copyPlugin(srcPlugin, dstFilepath)

		default:
			err = errors.New("invalid type selected")
			lo.G.Error("error: ", err)
		}
	}
	return
}

func copyPlugin(src io.Reader, dst string) (err error) {
	var dstPlugin *os.File
	if dstPlugin, err = osutils.SafeCreate(dst); err == nil {
		defer dstPlugin.Close()
		_, err = io.Copy(dstPlugin, src)
		os.Chmod(dst, 755)
	}
	return
}

func formatProps(props map[string]interface{}) string {
	var keys []string
	for k, _ := range props {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	buf := &bytes.Buffer{}
	for _, key := range keys {
		fmt.Fprintf(buf, "%v: %v\n", key, props[key])
	}
	return buf.String()
}

func ListProducts(w io.Writer, products map[string]registry.Record) {
	table := tablewriter.NewWriter(w)
	table.SetHeader([]string{"Name", "Command", "Properties"})

	var keys []string
	for k, _ := range products {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, key := range keys {
		row := []string{
			products[key].Name,
			path.Base(products[key].Path),
			formatProps(products[key].Properties),
		}
		table.Append(row)
	}
	table.Render()
}
<commit_msg>[#129955195] add `product-meta` command to display metadata for a single product<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/enaml-ops/omg-cli/aws-cli"
	"github.com/enaml-ops/omg-cli/azure-cli"
	"github.com/enaml-ops/omg-cli/bosh"
	gcpcli "github.com/enaml-ops/omg-cli/gcp-cli"
	"github.com/enaml-ops/omg-cli/photon-cli"
	"github.com/enaml-ops/omg-cli/utils"
	"github.com/enaml-ops/omg-cli/vsphere-cli"
	"github.com/enaml-ops/pluginlib/registry"
	"github.com/olekukonko/tablewriter"
	"github.com/pivotalservices/gtils/osutils"
	"github.com/xchapter7x/lo"
)

// Version is the version of omg-cli.
var Version string

// CloudConfigPluginsDir is the directory where registered cloud config plugins are stored.
var CloudConfigPluginsDir = "./.plugins/cloudconfig"

// ProductPluginsDir is the directory where registered product plugins are stored.
var ProductPluginsDir = "./.plugins/product"

func main() {
	app := cli.NewApp()
	app.Version = Version
	app.Commands = []cli.Command{
		{
			Name:   "azure",
			Usage:  "azure [--flags] - deploy a bosh to azure",
			Action: azurecli.GetAction(BoshInitDeploy),
			Flags:  azurecli.GetFlags(),
		},
		{
			Name:   "aws",
			Usage:  "aws [--flags] - deploy a bosh to aws",
			Action: awscli.GetAction(BoshInitDeploy),
			Flags:  awscli.GetFlags(),
		},
		{
			Name:   "gcp",
			Usage:  "gcp [--flags] - deploy a bosh to GCP",
			Action: gcpcli.GetAction(BoshInitDeploy),
			Flags:  gcpcli.GetFlags(),
		},
		{
			Name:   "photon",
			Usage:  "photon [--flags] - deploy a bosh to photon",
			Action: photoncli.GetAction(BoshInitDeploy),
			Flags:  photoncli.GetFlags(),
		},
		{
			Name:   "vsphere",
			Usage:  "vsphere [--flags] - deploy a bosh to vsphere",
			Action: vspherecli.GetAction(BoshInitDeploy),
			Flags:  vspherecli.GetFlags(),
		},
		{
			Name: "list-cloudconfigs",
			Action: func(c *cli.Context) error {
				fmt.Println("Cloud Configs:")
				table := tablewriter.NewWriter(os.Stdin)
				table.SetHeader([]string{"Name", "Command", "Properties"})
				var data [][]string
				formatProperties := func(p map[string]interface{}) string {
					var res string
					for n, v := range p {
						res += fmt.Sprintf("%v: %v\n", n, v)
					}
					return res
				}

				for _, plgn := range registry.ListCloudConfigs() {
					row := []string{
						plgn.Name,
						path.Base(plgn.Path),
						formatProperties(plgn.Properties),
					}
					data = append(data, row)
				}
				table.AppendBulk(data)
				table.Render()
				return nil
			},
		},
		{
			Name: "list-products",
			Action: func(c *cli.Context) error {
				ListProducts(os.Stdout, registry.ListProducts())
				return nil
			},
		},
		{
			Name:  "product-meta",
			Usage: "product-meta <prod-name> - show product metadata",
			Action: func(c *cli.Context) error {
				if len(c.Args()) > 0 {
					return productMeta(c.Args().First())
				}
				return nil
			},
		},
		{
			Name:  "register-plugin",
			Usage: "register-plugin -type [cloudconfig, product] -pluginpath <plugin-binary>",
			Action: func(c *cli.Context) (err error) {
				if c.String("type") != "" && c.String("pluginpath") != "" {
					err = registerPlugin(c.String("type"), c.String("pluginpath"))
				}
				return
			},
			Flags: []cli.Flag{
				cli.StringFlag{Name: "type", Value: "product", Usage: "define if the plugin to be registered is a cloudconfig or a product"},
				cli.StringFlag{Name: "pluginpath", Value: "", Usage: "the path to the plugin you wish to register"},
			},
		},
		{
			Name:        "deploy-cloudconfig",
			Usage:       "deploy-cloudconfig <cloudconfig-name> [--flags] - deploy a cloudconfig to bosh",
			Flags:       bosh.GetAuthFlags(),
			Subcommands: utils.GetCloudConfigCommands(CloudConfigPluginsDir),
		},
		{
			Name:        "deploy-product",
			Usage:       "deploy-product <prod-name> [--flags] - deploy a product via bosh",
			Flags:       bosh.GetAuthFlags(),
			Subcommands: utils.GetProductCommands(ProductPluginsDir),
		},
	}
	err := app.Run(os.Args)
	if err != nil {
		fmt.Println("ERROR:", err)
		os.Exit(1)
	}
}

func init() {
	if strings.ToLower(os.Getenv("LOG_LEVEL")) != "debug" {
		log.SetOutput(ioutil.Discard)
	}
}

func registerPlugin(typename, pluginpath string) (err error) {
	var srcPlugin *os.File

	if srcPlugin, err = os.Open(pluginpath); err == nil {
		defer srcPlugin.Close()

		switch typename {
		case "cloudconfig":
			dstFilepath := path.Join(CloudConfigPluginsDir, path.Base(pluginpath))
			err = copyPlugin(srcPlugin, dstFilepath)

		case "product":
			dstFilepath := path.Join(ProductPluginsDir, path.Base(pluginpath))
			err = copyPlugin(srcPlugin, dstFilepath)

		default:
			err = errors.New("invalid type selected")
			lo.G.Error("error: ", err)
		}
	}
	return
}

func copyPlugin(src io.Reader, dst string) (err error) {
	var dstPlugin *os.File
	if dstPlugin, err = osutils.SafeCreate(dst); err == nil {
		defer dstPlugin.Close()
		_, err = io.Copy(dstPlugin, src)
		os.Chmod(dst, 755)
	}
	return
}

func formatProps(props map[string]interface{}) string {
	var keys []string
	for k := range props {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	buf := &bytes.Buffer{}

	// output version first, then the remainder of the properties (sorted)
	ver, ok := props["version"]
	if ok {
		fmt.Fprintln(buf, "version:", ver)
	}

	for _, key := range keys {
		// don't output version twice
		if ok && key == "version" {
			continue
		}
		fmt.Fprintf(buf, "%v: %v\n", key, props[key])
	}
	return buf.String()
}

// ListProducts writes a formatted list of products to w.
func ListProducts(w io.Writer, products map[string]registry.Record) {
	table := tablewriter.NewWriter(w)
	table.SetHeader([]string{"Name", "Command", "Properties"})

	var keys []string
	for k := range products {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, key := range keys {
		row := []string{
			products[key].Name,
			path.Base(products[key].Path),
			formatProps(products[key].Properties),
		}
		table.Append(row)
	}
	table.Render()
}

func productMeta(product string) error {
	record, ok := registry.ListProducts()[product]
	if !ok {
		return fmt.Errorf("product '%s' not found", product)
	}
	fmt.Println(formatProps(record.Properties))
	return nil
}
<|endoftext|>


<commit_before>package main

//go:generate esc -o templates.go ./templates

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"text/tabwriter"

	"github.com/bukalapak/snowboard/adapter/drafter"
	"github.com/bukalapak/snowboard/adapter/drafterc"
	"github.com/bukalapak/snowboard/mock"
	snowboard "github.com/bukalapak/snowboard/parser"
	"github.com/bukalapak/snowboard/render"
	"github.com/fsnotify/fsnotify"
	xerrors "github.com/pkg/errors"
	"github.com/urfave/cli"
)

var versionStr string
var (
	engine  snowboard.Parser
	engineC snowboard.Parser
)

func main() {
	engine = drafter.Engine{}
	engineC = drafterc.Engine{}

	cli.VersionPrinter = func(c *cli.Context) {
		fmt.Fprintf(c.App.Writer, "Snowboard version: %s\n", c.App.Version)
		fmt.Fprintf(c.App.Writer, "Drafter version: %s\n", engine.Version())
	}

	if versionStr == "" {
		versionStr = "HEAD"
	}

	app := cli.NewApp()
	app.Name = "snowboard"
	app.Usage = "API blueprint toolkit"
	app.Version = versionStr
	app.Before = func(c *cli.Context) error {
		if c.Args().Get(1) == "" {
			cli.ShowCommandHelp(c, c.Args().Get(0))
		}

		return nil
	}
	app.Commands = []cli.Command{
		{
			Name:  "lint",
			Usage: "Validate API blueprint",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "c",
					Usage: "Use character index instead of line and row number",
				},
			},
			Action: func(c *cli.Context) error {
				if c.Args().Get(0) == "" {
					return nil
				}

				if err := validate(c, c.Args().Get(0), c.Bool("c")); err != nil {
					if strings.Contains(err.Error(), "read failed") {
						return xerrors.Cause(err)
					}

					io.WriteString(c.App.Writer, err.Error())
				}

				return nil
			},
		},
		{
			Name:  "html",
			Usage: "Render HTML documentation",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "o",
					Usage: "HTML file",
				},
				cli.StringFlag{
					Name:  "t",
					Value: "alpha",
					Usage: "Template for HTML documentation",
				},
				cli.BoolFlag{
					Name:  "s",
					Usage: "Serve HTML via HTTP server",
				},
				cli.StringFlag{
					Name:  "b",
					Value: "127.0.0.1:8088",
					Usage: "HTTP server listen address",
				},
			},
			Action: func(c *cli.Context) error {
				if c.Args().Get(0) == "" {
					return nil
				}

				if c.Bool("s") {
					return watchHTML(c, c.Args().Get(0), c.String("o"), c.String("t"), c.String("b"))
				}

				return renderHTML(c, c.Args().Get(0), c.String("o"), c.String("t"))
			},
		},
		{
			Name:  "apib",
			Usage: "Render API blueprint",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "o",
					Usage: "API blueprint output file",
				},
			},
			Action: func(c *cli.Context) error {
				if c.Args().Get(0) == "" {
					return nil
				}

				return renderAPIB(c, c.Args().Get(0), c.String("o"))
			},
		},
		{
			Name:  "json",
			Usage: "Render API element json",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "o",
					Usage: "API element output file",
				},
			},
			Action: func(c *cli.Context) error {
				if c.Args().Get(0) == "" {
					return nil
				}

				return renderJSON(c, c.Args().Get(0), c.String("o"))
			},
		},
		{
			Name:  "mock",
			Usage: "Run Mock server",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "b",
					Value: "127.0.0.1:8087",
					Usage: "HTTP server listen address",
				},
			},
			Action: func(c *cli.Context) error {
				if c.Args().Get(0) == "" {
					return nil
				}

				return serveMock(c, c.String("b"), c.Args().Get(0))
			},
		},
	}
	app.Run(os.Args)
}

func readFile(fn string) ([]byte, error) {
	info, err := os.Stat(fn)
	if err != nil {
		return nil, errors.New("File is not exist")
	}

	if info.IsDir() {
		return nil, errors.New("File is a directory")
	}

	return ioutil.ReadFile(fn)
}

func readTemplate(fn string) ([]byte, error) {
	tf, err := readFile(fn)
	if err == nil {
		return tf, nil
	}

	fs := FS(false)
	ff, err := fs.Open("/templates/" + fn + ".html")
	if err != nil {
		return nil, err
	}

	defer ff.Close()
	return ioutil.ReadAll(ff)
}

func renderHTML(c *cli.Context, input, output, tplFile string) error {
	bp, err := snowboard.Load(input, engine)
	if err != nil {
		return err
	}

	tf, err := readTemplate(tplFile)
	if err != nil {
		return err
	}

	if output == "" {
		var bf bytes.Buffer

		if err := render.HTML(string(tf), &bf, bp); err != nil {
			return err
		}

		fmt.Fprintln(c.App.Writer, bf.String())
		return nil
	}

	of, err := os.Create(output)
	if err != nil {
		return err
	}
	defer of.Close()

	err = render.HTML(string(tf), of, bp)
	if err != nil {
		return err
	}

	fmt.Fprintln(c.App.Writer, "HTML has been generated!")
	return nil
}

func renderAPIB(c *cli.Context, input, output string) error {
	b, err := snowboard.Read(input)
	if err != nil {
		return err
	}

	if output == "" {
		fmt.Fprintln(c.App.Writer, string(b))
		return nil
	}

	of, err := os.Create(output)
	if err != nil {
		return err
	}
	defer of.Close()

	_, err = io.Copy(of, bytes.NewReader(b))
	if err != nil {
		return err
	}

	fmt.Fprintln(c.App.Writer, "API blueprint has been generated!")
	return nil
}

func renderJSON(c *cli.Context, input, output string) error {
	b, err := snowboard.LoadAsJSON(input, engine)
	if err != nil {
		return err
	}

	if output == "" {
		fmt.Fprintln(c.App.Writer, string(b))
		return nil
	}

	of, err := os.Create(output)
	if err != nil {
		return err
	}
	defer of.Close()

	_, err = io.Copy(of, bytes.NewReader(b))
	if err != nil {
		return err
	}

	fmt.Fprintln(c.App.Writer, "API element JSON has been generated!")
	return nil
}

func validate(c *cli.Context, input string, charIdx bool) error {
	b, err := readFile(input)
	if err != nil {
		return xerrors.Wrap(err, "read failed")
	}

	bf := bytes.NewReader(b)

	if charIdx {
		out, err := snowboard.Validate(bf, engine)
		if err != nil {
			return err
		}

		if out == nil {
			fmt.Fprintln(c.App.Writer, "OK")
			return nil
		}

		var buf bytes.Buffer

		s := "--------"
		w := tabwriter.NewWriter(&buf, 8, 0, 0, ' ', tabwriter.Debug)
		fmt.Fprintln(w, "Char Index\tDescription")
		fmt.Fprintf(w, "%s\t%s\n", s, strings.Repeat(s, 8))

		for _, n := range out.Annotations {
			for _, m := range n.SourceMaps {
				fmt.Fprintf(w, "%d:%d\t%s\n", m.Row, m.Col, n.Description)
			}
		}

		w.Flush()

		if len(out.Annotations) > 0 {
			return errors.New(buf.String())
		}
	} else {
		b, err := engineC.Validate(bf)
		if err != nil {
			return err
		}

		var buf bytes.Buffer

		w := tabwriter.NewWriter(&buf, 8, 0, 0, ' ', tabwriter.Debug)
		fmt.Fprintln(w, "Location\tSeverity\tDescription")

		fmt.Fprintf(w, "%s\t%s\t%s\n", dash(42), dash(16), dash(80))

		if string(b) == "" {
			fmt.Fprintln(c.App.Writer, "OK")
			return nil
		}

		ns := strings.Split(string(b), "\n")

		for _, n := range ns {
			nn := strings.SplitN(n, "; ", 2)
			mm := strings.SplitN(nn[0], "  ", 2)
			fmt.Fprintf(w, "%s\t%s\t%s\n", nn[1], mm[0], mm[1])
		}

		w.Flush()
		return errors.New(buf.String())
	}

	return nil
}

func dash(n int) string {
	return strings.Repeat("-", n)
}

func watchHTML(c *cli.Context, input, output, tplFile, bind string) error {
	if output == "" {
		output = "index.html"
	}

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}
	defer watcher.Close()

	done := make(chan bool)
	go func() {
		for {
			select {
			case event := <-watcher.Events:
				if event.Op&fsnotify.Write == fsnotify.Write {
					renderHTML(c, input, output, tplFile)
				}
			case err := <-watcher.Errors:
				fmt.Fprintln(c.App.Writer, err)
			}
		}
	}()

	err = watcher.Add(input)
	if err != nil {
		return err
	}

	if _, err = os.Stat(tplFile); err == nil {
		err = watcher.Add(tplFile)
		if err != nil {
			return err
		}
	}

	renderHTML(c, input, output, tplFile)
	serveHTML(bind, output)

	<-done

	return nil
}

func serveHTML(bind, output string) error {
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, output)
	})

	return http.ListenAndServe(bind, nil)
}

func serveMock(c *cli.Context, bind, input string) error {
	bp, err := snowboard.Load(input, engine)
	if err != nil {
		return err
	}

	fmt.Fprintf(c.App.Writer, "Mock server is ready. Use %s\n", bind)
	fmt.Fprintln(c.App.Writer, "Available Routes:")

	ms := mock.Mock(bp)
	for _, m := range ms {
		fmt.Fprintf(c.App.Writer, "%s\t%d\t%s\n", m.Method, m.StatusCode, m.Pattern)
	}

	h := mock.MockHandler(ms)
	return http.ListenAndServe(bind, h)
}
<commit_msg>fix double help usage<commit_after>package main

//go:generate esc -o templates.go ./templates

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"text/tabwriter"

	"github.com/bukalapak/snowboard/adapter/drafter"
	"github.com/bukalapak/snowboard/adapter/drafterc"
	"github.com/bukalapak/snowboard/mock"
	snowboard "github.com/bukalapak/snowboard/parser"
	"github.com/bukalapak/snowboard/render"
	"github.com/fsnotify/fsnotify"
	xerrors "github.com/pkg/errors"
	"github.com/urfave/cli"
)

var versionStr string
var (
	engine  snowboard.Parser
	engineC snowboard.Parser
)

func main() {
	engine = drafter.Engine{}
	engineC = drafterc.Engine{}

	cli.VersionPrinter = func(c *cli.Context) {
		fmt.Fprintf(c.App.Writer, "Snowboard version: %s\n", c.App.Version)
		fmt.Fprintf(c.App.Writer, "Drafter version: %s\n", engine.Version())
	}

	if versionStr == "" {
		versionStr = "HEAD"
	}

	app := cli.NewApp()
	app.Name = "snowboard"
	app.Usage = "API blueprint toolkit"
	app.Version = versionStr
	app.Before = func(c *cli.Context) error {
		if c.Args().Present() && c.Args().Get(1) == "" {
			cli.ShowCommandHelp(c, c.Args().Get(0))
		}

		return nil
	}
	app.Commands = []cli.Command{
		{
			Name:  "lint",
			Usage: "Validate API blueprint",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "c",
					Usage: "Use character index instead of line and row number",
				},
			},
			Action: func(c *cli.Context) error {
				if c.Args().Get(0) == "" {
					return nil
				}

				if err := validate(c, c.Args().Get(0), c.Bool("c")); err != nil {
					if strings.Contains(err.Error(), "read failed") {
						return xerrors.Cause(err)
					}

					io.WriteString(c.App.Writer, err.Error())
				}

				return nil
			},
		},
		{
			Name:  "html",
			Usage: "Render HTML documentation",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "o",
					Usage: "HTML file",
				},
				cli.StringFlag{
					Name:  "t",
					Value: "alpha",
					Usage: "Template for HTML documentation",
				},
				cli.BoolFlag{
					Name:  "s",
					Usage: "Serve HTML via HTTP server",
				},
				cli.StringFlag{
					Name:  "b",
					Value: "127.0.0.1:8088",
					Usage: "HTTP server listen address",
				},
			},
			Action: func(c *cli.Context) error {
				if c.Args().Get(0) == "" {
					return nil
				}

				if c.Bool("s") {
					return watchHTML(c, c.Args().Get(0), c.String("o"), c.String("t"), c.String("b"))
				}

				return renderHTML(c, c.Args().Get(0), c.String("o"), c.String("t"))
			},
		},
		{
			Name:  "apib",
			Usage: "Render API blueprint",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "o",
					Usage: "API blueprint output file",
				},
			},
			Action: func(c *cli.Context) error {
				if c.Args().Get(0) == "" {
					return nil
				}

				return renderAPIB(c, c.Args().Get(0), c.String("o"))
			},
		},
		{
			Name:  "json",
			Usage: "Render API element json",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "o",
					Usage: "API element output file",
				},
			},
			Action: func(c *cli.Context) error {
				if c.Args().Get(0) == "" {
					return nil
				}

				return renderJSON(c, c.Args().Get(0), c.String("o"))
			},
		},
		{
			Name:  "mock",
			Usage: "Run Mock server",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "b",
					Value: "127.0.0.1:8087",
					Usage: "HTTP server listen address",
				},
			},
			Action: func(c *cli.Context) error {
				if c.Args().Get(0) == "" {
					return nil
				}

				return serveMock(c, c.String("b"), c.Args().Get(0))
			},
		},
	}
	app.Run(os.Args)
}

func readFile(fn string) ([]byte, error) {
	info, err := os.Stat(fn)
	if err != nil {
		return nil, errors.New("File is not exist")
	}

	if info.IsDir() {
		return nil, errors.New("File is a directory")
	}

	return ioutil.ReadFile(fn)
}

func readTemplate(fn string) ([]byte, error) {
	tf, err := readFile(fn)
	if err == nil {
		return tf, nil
	}

	fs := FS(false)
	ff, err := fs.Open("/templates/" + fn + ".html")
	if err != nil {
		return nil, err
	}

	defer ff.Close()
	return ioutil.ReadAll(ff)
}

func renderHTML(c *cli.Context, input, output, tplFile string) error {
	bp, err := snowboard.Load(input, engine)
	if err != nil {
		return err
	}

	tf, err := readTemplate(tplFile)
	if err != nil {
		return err
	}

	if output == "" {
		var bf bytes.Buffer

		if err := render.HTML(string(tf), &bf, bp); err != nil {
			return err
		}

		fmt.Fprintln(c.App.Writer, bf.String())
		return nil
	}

	of, err := os.Create(output)
	if err != nil {
		return err
	}
	defer of.Close()

	err = render.HTML(string(tf), of, bp)
	if err != nil {
		return err
	}

	fmt.Fprintln(c.App.Writer, "HTML has been generated!")
	return nil
}

func renderAPIB(c *cli.Context, input, output string) error {
	b, err := snowboard.Read(input)
	if err != nil {
		return err
	}

	if output == "" {
		fmt.Fprintln(c.App.Writer, string(b))
		return nil
	}

	of, err := os.Create(output)
	if err != nil {
		return err
	}
	defer of.Close()

	_, err = io.Copy(of, bytes.NewReader(b))
	if err != nil {
		return err
	}

	fmt.Fprintln(c.App.Writer, "API blueprint has been generated!")
	return nil
}

func renderJSON(c *cli.Context, input, output string) error {
	b, err := snowboard.LoadAsJSON(input, engine)
	if err != nil {
		return err
	}

	if output == "" {
		fmt.Fprintln(c.App.Writer, string(b))
		return nil
	}

	of, err := os.Create(output)
	if err != nil {
		return err
	}
	defer of.Close()

	_, err = io.Copy(of, bytes.NewReader(b))
	if err != nil {
		return err
	}

	fmt.Fprintln(c.App.Writer, "API element JSON has been generated!")
	return nil
}

func validate(c *cli.Context, input string, charIdx bool) error {
	b, err := readFile(input)
	if err != nil {
		return xerrors.Wrap(err, "read failed")
	}

	bf := bytes.NewReader(b)

	if charIdx {
		out, err := snowboard.Validate(bf, engine)
		if err != nil {
			return err
		}

		if out == nil {
			fmt.Fprintln(c.App.Writer, "OK")
			return nil
		}

		var buf bytes.Buffer

		s := "--------"
		w := tabwriter.NewWriter(&buf, 8, 0, 0, ' ', tabwriter.Debug)
		fmt.Fprintln(w, "Char Index\tDescription")
		fmt.Fprintf(w, "%s\t%s\n", s, strings.Repeat(s, 8))

		for _, n := range out.Annotations {
			for _, m := range n.SourceMaps {
				fmt.Fprintf(w, "%d:%d\t%s\n", m.Row, m.Col, n.Description)
			}
		}

		w.Flush()

		if len(out.Annotations) > 0 {
			return errors.New(buf.String())
		}
	} else {
		b, err := engineC.Validate(bf)
		if err != nil {
			return err
		}

		var buf bytes.Buffer

		w := tabwriter.NewWriter(&buf, 8, 0, 0, ' ', tabwriter.Debug)
		fmt.Fprintln(w, "Location\tSeverity\tDescription")

		fmt.Fprintf(w, "%s\t%s\t%s\n", dash(42), dash(16), dash(80))

		if string(b) == "" {
			fmt.Fprintln(c.App.Writer, "OK")
			return nil
		}

		ns := strings.Split(string(b), "\n")

		for _, n := range ns {
			nn := strings.SplitN(n, "; ", 2)
			mm := strings.SplitN(nn[0], "  ", 2)
			fmt.Fprintf(w, "%s\t%s\t%s\n", nn[1], mm[0], mm[1])
		}

		w.Flush()
		return errors.New(buf.String())
	}

	return nil
}

func dash(n int) string {
	return strings.Repeat("-", n)
}

func watchHTML(c *cli.Context, input, output, tplFile, bind string) error {
	if output == "" {
		output = "index.html"
	}

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}
	defer watcher.Close()

	done := make(chan bool)
	go func() {
		for {
			select {
			case event := <-watcher.Events:
				if event.Op&fsnotify.Write == fsnotify.Write {
					renderHTML(c, input, output, tplFile)
				}
			case err := <-watcher.Errors:
				fmt.Fprintln(c.App.Writer, err)
			}
		}
	}()

	err = watcher.Add(input)
	if err != nil {
		return err
	}

	if _, err = os.Stat(tplFile); err == nil {
		err = watcher.Add(tplFile)
		if err != nil {
			return err
		}
	}

	renderHTML(c, input, output, tplFile)
	serveHTML(bind, output)

	<-done

	return nil
}

func serveHTML(bind, output string) error {
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, output)
	})

	return http.ListenAndServe(bind, nil)
}

func serveMock(c *cli.Context, bind, input string) error {
	bp, err := snowboard.Load(input, engine)
	if err != nil {
		return err
	}

	fmt.Fprintf(c.App.Writer, "Mock server is ready. Use %s\n", bind)
	fmt.Fprintln(c.App.Writer, "Available Routes:")

	ms := mock.Mock(bp)
	for _, m := range ms {
		fmt.Fprintf(c.App.Writer, "%s\t%d\t%s\n", m.Method, m.StatusCode, m.Pattern)
	}

	h := mock.MockHandler(ms)
	return http.ListenAndServe(bind, h)
}
<|endoftext|>