<commit_before>package main

import (
	"database/sql"
	"log"
	"net/http"
	"os"

	ledgerContext "github.com/RealImage/QLedger/context"
	"github.com/RealImage/QLedger/controllers"
	"github.com/RealImage/QLedger/middlewares"
	"github.com/julienschmidt/httprouter"
	"github.com/mattes/migrate"
	"github.com/mattes/migrate/database/postgres"
	_ "github.com/mattes/migrate/source/file"
)

func main() {
	db, err := sql.Open("postgres", os.Getenv("DATABASE_URL"))
	if err != nil {
		log.Panic("Unable to connect to Database:", err)
	}
	log.Println("Successfully established connection to database.")

	// Migrate DB changes
	migrateDB(db)

	appContext := &ledgerContext.AppContext{DB: db}
	router := httprouter.New()

	hostPrefix := os.Getenv("HOST_PREFIX")
	// Monitors
	router.HandlerFunc(http.MethodGet, hostPrefix+"/ping", controllers.Ping)

	// Create accounts and transactions
	router.HandlerFunc(http.MethodPost, hostPrefix+"/v1/accounts",
		middlewares.TokenAuthMiddleware(
			middlewares.ContextMiddleware(controllers.AddAccount, appContext)))
	router.HandlerFunc(http.MethodPost, hostPrefix+"/v1/transactions",
		middlewares.TokenAuthMiddleware(
			middlewares.ContextMiddleware(controllers.MakeTransaction, appContext)))

	// Read or search accounts and transactions
	router.HandlerFunc(http.MethodGet, hostPrefix+"/v1/accounts",
		middlewares.TokenAuthMiddleware(
			middlewares.ContextMiddleware(controllers.GetAccounts, appContext)))
	router.HandlerFunc(http.MethodPost, hostPrefix+"/v1/accounts/_search",
		middlewares.TokenAuthMiddleware(
			middlewares.ContextMiddleware(controllers.GetAccounts, appContext)))
	router.HandlerFunc(http.MethodGet, hostPrefix+"/v1/transactions",
		middlewares.TokenAuthMiddleware(
			middlewares.ContextMiddleware(controllers.GetTransactions, appContext)))
	router.HandlerFunc(http.MethodPost, hostPrefix+"/v1/transactions/_search",
		middlewares.TokenAuthMiddleware(
			middlewares.ContextMiddleware(controllers.GetTransactions, appContext)))

	// Update data of accounts and transactions
	router.HandlerFunc(http.MethodPut, hostPrefix+"/v1/accounts",
		middlewares.TokenAuthMiddleware(
			middlewares.ContextMiddleware(controllers.UpdateAccount, appContext)))
	router.HandlerFunc(http.MethodPut, hostPrefix+"/v1/transactions",
		middlewares.TokenAuthMiddleware(
			middlewares.ContextMiddleware(controllers.UpdateTransaction, appContext)))

	port := os.Getenv("PORT")
	if port == "" {
		port = "7000"
	}
	log.Println("Running server on port:", port)
	log.Fatal(http.ListenAndServe(":"+port, router))

	defer func() {
		if r := recover(); r != nil {
			log.Println("Server exited!!!", r)
		}
	}()
}

func migrateDB(db *sql.DB) {
	log.Println("Starting db schema migration...")
	driver, err := postgres.WithInstance(db, &postgres.Config{})
	if err != nil {
		log.Panic("Unable to create database instance for migration:", err)
	}

	migrationFilesPath := os.Getenv("MIGRATION_FILES_PATH")
	if migrationFilesPath == "" {
		migrationFilesPath = "file://migrations/postgres"
	}
	m, err := migrate.NewWithDatabaseInstance(
		migrationFilesPath,
		"postgres", driver)
	if err != nil {
		log.Panic("Unable to create Migrate instance for database:", err)
	}

	version, dirty, err := m.Version()
	if err != nil && err != migrate.ErrNilVersion {
		log.Panic("Unable to get existing migration version for database:", dirty, err)
	}
	log.Println("Current schema version:", version)
	err = m.Up()
	if err != nil && err != migrate.ErrNoChange && err != migrate.ErrLocked {
		log.Panic("Error while migration:", err)
	}
	version, dirty, err = m.Version()
	if err != nil {
		log.Panic("Unable to get new migration version for database:", dirty, err)
	}
	log.Println("Migrated schema version:", version)
}
<commit_msg>Skip lock error while migration (#25)<commit_after>package main

import (
	"database/sql"
	"log"
	"net/http"
	"os"

	ledgerContext "github.com/RealImage/QLedger/context"
	"github.com/RealImage/QLedger/controllers"
	"github.com/RealImage/QLedger/middlewares"
	"github.com/julienschmidt/httprouter"
	"github.com/mattes/migrate"
	"github.com/mattes/migrate/database"
	"github.com/mattes/migrate/database/postgres"
	_ "github.com/mattes/migrate/source/file"
)

func main() {
	db, err := sql.Open("postgres", os.Getenv("DATABASE_URL"))
	if err != nil {
		log.Panic("Unable to connect to Database:", err)
	}
	log.Println("Successfully established connection to database.")

	// Migrate DB changes
	migrateDB(db)

	appContext := &ledgerContext.AppContext{DB: db}
	router := httprouter.New()

	hostPrefix := os.Getenv("HOST_PREFIX")
	// Monitors
	router.HandlerFunc(http.MethodGet, hostPrefix+"/ping", controllers.Ping)

	// Create accounts and transactions
	router.HandlerFunc(http.MethodPost, hostPrefix+"/v1/accounts",
		middlewares.TokenAuthMiddleware(
			middlewares.ContextMiddleware(controllers.AddAccount, appContext)))
	router.HandlerFunc(http.MethodPost, hostPrefix+"/v1/transactions",
		middlewares.TokenAuthMiddleware(
			middlewares.ContextMiddleware(controllers.MakeTransaction, appContext)))

	// Read or search accounts and transactions
	router.HandlerFunc(http.MethodGet, hostPrefix+"/v1/accounts",
		middlewares.TokenAuthMiddleware(
			middlewares.ContextMiddleware(controllers.GetAccounts, appContext)))
	router.HandlerFunc(http.MethodPost, hostPrefix+"/v1/accounts/_search",
		middlewares.TokenAuthMiddleware(
			middlewares.ContextMiddleware(controllers.GetAccounts, appContext)))
	router.HandlerFunc(http.MethodGet, hostPrefix+"/v1/transactions",
		middlewares.TokenAuthMiddleware(
			middlewares.ContextMiddleware(controllers.GetTransactions, appContext)))
	router.HandlerFunc(http.MethodPost, hostPrefix+"/v1/transactions/_search",
		middlewares.TokenAuthMiddleware(
			middlewares.ContextMiddleware(controllers.GetTransactions, appContext)))

	// Update data of accounts and transactions
	router.HandlerFunc(http.MethodPut, hostPrefix+"/v1/accounts",
		middlewares.TokenAuthMiddleware(
			middlewares.ContextMiddleware(controllers.UpdateAccount, appContext)))
	router.HandlerFunc(http.MethodPut, hostPrefix+"/v1/transactions",
		middlewares.TokenAuthMiddleware(
			middlewares.ContextMiddleware(controllers.UpdateTransaction, appContext)))

	port := os.Getenv("PORT")
	if port == "" {
		port = "7000"
	}
	log.Println("Running server on port:", port)
	log.Fatal(http.ListenAndServe(":"+port, router))

	defer func() {
		if r := recover(); r != nil {
			log.Println("Server exited!!!", r)
		}
	}()
}

func migrateDB(db *sql.DB) {
	log.Println("Starting db schema migration...")
	driver, err := postgres.WithInstance(db, &postgres.Config{})
	if err != nil {
		log.Panic("Unable to create database instance for migration:", err)
	}

	migrationFilesPath := os.Getenv("MIGRATION_FILES_PATH")
	if migrationFilesPath == "" {
		migrationFilesPath = "file://migrations/postgres"
	}
	m, err := migrate.NewWithDatabaseInstance(
		migrationFilesPath,
		"postgres", driver)
	if err != nil {
		log.Panic("Unable to create Migrate instance for database:", err)
	}

	version, dirty, err := m.Version()
	if err != nil && err != migrate.ErrNilVersion {
		log.Panic("Unable to get existing migration version for database:", dirty, err)
	}
	log.Println("Current schema version:", version)
	err = m.Up()
	if err != nil {
		switch err {
		case migrate.ErrNoChange:
			log.Println("No changes to migrate")
		case migrate.ErrLocked, database.ErrLocked:
			log.Println("Database locked. Skipping migration assuming another instance working on it")
		default:
			log.Panic("Error while migration:", err)
		}
	}
	version, dirty, err = m.Version()
	if err != nil {
		log.Panic("Unable to get new migration version for database:", dirty, err)
	}
	log.Println("Migrated schema version:", version)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	"github.com/vito/gocart/command_runner"
	"github.com/vito/gocart/dependency"
	"github.com/vito/gocart/dependency_fetcher"
	"github.com/vito/gocart/dependency_reader"
	"github.com/vito/gocart/gopath"
	"github.com/vito/gocart/locker"
)

const GocartVersion = "0.1.0"

const CartridgeFile = "Cartridge"
const CartridgeLockFile = "Cartridge.lock"

var GOPATH string

var FetchedDependencies = make(map[string]dependency.Dependency)
var TrickleDownDependencies = make(map[string]dependency.Dependency)

var recursive = flag.Bool(
	"r",
	false,
	"recursively fetch dependencies and check for conflicts",
)

var showHelp = flag.Bool(
	"h",
	false,
	"show command usage (this text)",
)

var showVersion = flag.Bool(
	"v",
	false,
	"show gocart version",
)

var trickleDown = flag.Bool(
	"t",
	false,
	"trickle down dependencies into nested Cartridges",
)

var aggregate = flag.Bool(
	"a",
	false,
	"collect recursive dependencies into a single .lock file",
)

func main() {
	flag.Parse()

	args := flag.Args()

	command := ""

	gopath, err := gopath.InstallationDirectory(os.Getenv("GOPATH"))
	if err != nil {
		fatal("GOPATH is not set.")
	}

	GOPATH = gopath

	if len(args) == 0 {
		command = "install"
	} else {
		command = args[0]
	}

	if *showHelp || command == "help" {
		help()
		return
	}

	if *showVersion || command == "version" {
		version()
		return
	}

	if command == "install" {
		install(".", *recursive, *aggregate, *trickleDown, 0)
		return
	}

	unknownCommand()
}

func install(root string, recursive bool, aggregate bool, trickleDown bool, depth int) {
	if _, err := os.Stat(path.Join(root, CartridgeFile)); err != nil {
		println("no Cartridge file!")
		os.Exit(1)
		return
	}

	requestedDependencies := loadFile(path.Join(root, CartridgeFile))
	lockedDependencies := loadFile(path.Join(root, CartridgeLockFile))

	dependencies := locker.GenerateLock(requestedDependencies, lockedDependencies)

	newLockedDependencies, err := installDependencies(dependencies, recursive, trickleDown, depth)
	if err != nil {
		fatal(err.Error())
	}

	file, err := os.Create(path.Join(root, "Cartridge.lock"))
	if err != nil {
		fatal(err.Error())
	}
	defer file.Close()

	var dependenciesToBeWritten []dependency.Dependency

	if aggregate {
		for _, dep := range FetchedDependencies {
			dependenciesToBeWritten = append(dependenciesToBeWritten, dep)
		}
	} else {
		dependenciesToBeWritten = newLockedDependencies
	}

	err = updateLockFile(file, dependenciesToBeWritten)
	if err != nil {
		fatal(err.Error())
	}

	if depth == 0 {
		fmt.Println("\x1b[32mOK\x1b[0m")
	}
}

func help() {
	fmt.Println(`gocart: a go package manager

Usage:

    'gocart install' or 'gocart':
        Install dependencies described by Cartridge.lock or Cartridge, and
        update Cartridge.lock with locked-down dependency versions.

Place your dependencies in a file called Cartridge with this format:

[import path]	[vcs ref]

'gocart install' will 'go get' each import path and switch it to the given ref.
After getting them, it will take the "hard" ref (e.g. the sha in git), and save
it in Cartridge.lock. The Cartridge.lock has the same format as Cartridge and
has the same semantics; it will later be used by 'gocart install' if it exists.

To update an individual dependency, simply remove its line from Cartridge.lock
and run 'gocart install'. To update all dependencies, remove Cartridge.lock.
`)
}

func version() {
	fmt.Println(GocartVersion)
}

func unknownCommand() {
	fmt.Println("unknown command:", os.Args[1])
	fmt.Println()
	help()
	os.Exit(1)
}

func loadFile(fileName string) []dependency.Dependency {
	cartridge, err := os.Open(fileName)
	if err != nil {
		return []dependency.Dependency{}
	}

	reader := dependency_reader.New(cartridge)

	dependencies, err := reader.ReadAll()
	if err != nil {
		fatal(err.Error())
	}

	return dependencies
}

func installDependencies(dependencies []dependency.Dependency, recursive bool, trickleDown bool, depth int) ([]dependency.Dependency, error) {
	runner := command_runner.New()

	lockedDependencies := []dependency.Dependency{}

	fetcher, err := dependency_fetcher.New(runner)
	if err != nil {
		return []dependency.Dependency{}, err
	}

	maxWidth := 0

	for _, dep := range dependencies {
		if len(dep.Path) > maxWidth {
			maxWidth = len(dep.Path)
		}
	}

	nextLevel := []dependency.Dependency{}

	for _, dep := range dependencies {
		fmt.Println(bold(dep.Path) + padding(maxWidth-len(dep.Path)+2) + cyan(dep.Version))

		dep = trickledDown(dep)

		lockedDependency, err := fetcher.Fetch(dep)
		if err != nil {
			return []dependency.Dependency{}, err
		}

		checkForConflicts(lockedDependency)

		FetchedDependencies[lockedDependency.Path] = lockedDependency

		lockedDependencies = append(lockedDependencies, lockedDependency)

		if depth == 0 && trickleDown {
			TrickleDownDependencies[lockedDependency.Path] = lockedDependency
		}

		nextLevel = append(nextLevel, lockedDependency)
	}

	if recursive {
		installNextLevel(nextLevel, depth+1)
	}

	return lockedDependencies, nil
}

func updateLockFile(writer io.Writer, dependencies []dependency.Dependency) error {
	for _, dependency := range dependencies {
		writer.Write([]byte(fmt.Sprintf("%s\t%s\n", dependency.Path, dependency.Version)))
	}

	return nil
}

func installNextLevel(deps []dependency.Dependency, newDepth int) {
	for _, dependency := range deps {
		dependencyPath := dependency.FullPath(GOPATH)

		if _, err := os.Stat(path.Join(dependencyPath, "Cartridge")); err == nil {
			fmt.Println("\nfetching dependencies for", dependency.Path)
			install(dependencyPath, true, false, false, newDepth)
		}
	}
}

func trickledDown(dep dependency.Dependency) dependency.Dependency {
	trickled, found := TrickleDownDependencies[dep.Path]
	if found {
		return trickled
	}

	return dep
}

func checkForConflicts(lockedDependency dependency.Dependency) {
	currentVersion, found := FetchedDependencies[lockedDependency.Path]
	if found && currentVersion.Version != lockedDependency.Version {
		fatal("version conflict for " + currentVersion.Path)
	}
}

func bold(str string) string {
	return "\x1b[1m" + str + "\x1b[0m"
}

func red(str string) string {
	return "\x1b[31m" + str + "\x1b[0m"
}

func cyan(str string) string {
	return "\x1b[36m" + str + "\x1b[0m"
}

func padding(size int) string {
	return strings.Repeat(" ", size)
}

func fatal(message string) {
	fmt.Println(red(message))
	os.Exit(1)
}
<commit_msg>make things prettier<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	"github.com/vito/gocart/command_runner"
	"github.com/vito/gocart/dependency"
	"github.com/vito/gocart/dependency_fetcher"
	"github.com/vito/gocart/dependency_reader"
	"github.com/vito/gocart/gopath"
	"github.com/vito/gocart/locker"
)

const GocartVersion = "0.1.0"

const CartridgeFile = "Cartridge"
const CartridgeLockFile = "Cartridge.lock"

var GOPATH string

var FetchedDependencies = make(map[string]dependency.Dependency)
var TrickleDownDependencies = make(map[string]dependency.Dependency)

var recursive = flag.Bool(
	"r",
	false,
	"recursively fetch dependencies and check for conflicts",
)

var showHelp = flag.Bool(
	"h",
	false,
	"show command usage (this text)",
)

var showVersion = flag.Bool(
	"v",
	false,
	"show gocart version",
)

var trickleDown = flag.Bool(
	"t",
	false,
	"trickle down dependencies into nested Cartridges",
)

var aggregate = flag.Bool(
	"a",
	false,
	"collect recursive dependencies into a single .lock file",
)

func main() {
	flag.Parse()

	args := flag.Args()

	command := ""

	gopath, err := gopath.InstallationDirectory(os.Getenv("GOPATH"))
	if err != nil {
		fatal("GOPATH is not set.")
	}

	GOPATH = gopath

	if len(args) == 0 {
		command = "install"
	} else {
		command = args[0]
	}

	if *showHelp || command == "help" {
		help()
		return
	}

	if *showVersion || command == "version" {
		version()
		return
	}

	if command == "install" {
		install(".", *recursive, *aggregate, *trickleDown, 0)
		return
	}

	unknownCommand()
}

func install(root string, recursive bool, aggregate bool, trickleDown bool, depth int) {
	if _, err := os.Stat(path.Join(root, CartridgeFile)); err != nil {
		println("no Cartridge file!")
		os.Exit(1)
		return
	}

	requestedDependencies := loadFile(path.Join(root, CartridgeFile))
	lockedDependencies := loadFile(path.Join(root, CartridgeLockFile))

	dependencies := locker.GenerateLock(requestedDependencies, lockedDependencies)

	newLockedDependencies := installDependencies(dependencies, recursive, trickleDown, depth)

	file, err := os.Create(path.Join(root, "Cartridge.lock"))
	if err != nil {
		fatal(err.Error())
	}
	defer file.Close()

	var dependenciesToBeWritten []dependency.Dependency

	if aggregate {
		for _, dep := range FetchedDependencies {
			dependenciesToBeWritten = append(dependenciesToBeWritten, dep)
		}
	} else {
		dependenciesToBeWritten = newLockedDependencies
	}

	err = updateLockFile(file, dependenciesToBeWritten)
	if err != nil {
		fatal(err.Error())
	}

	if depth == 0 {
		fmt.Println("\x1b[32mOK\x1b[0m")
	}
}

func help() {
	fmt.Println(`gocart: a go package manager

Usage:

    'gocart install' or 'gocart':
        Install dependencies described by Cartridge.lock or Cartridge, and
        update Cartridge.lock with locked-down dependency versions.

Place your dependencies in a file called Cartridge with this format:

[import path]	[vcs ref]

'gocart install' will 'go get' each import path and switch it to the given ref.
After getting them, it will take the "hard" ref (e.g. the sha in git), and save
it in Cartridge.lock. The Cartridge.lock has the same format as Cartridge and
has the same semantics; it will later be used by 'gocart install' if it exists.

To update an individual dependency, simply remove its line from Cartridge.lock
and run 'gocart install'. To update all dependencies, remove Cartridge.lock.
`)
}

func version() {
	fmt.Println(GocartVersion)
}

func unknownCommand() {
	fmt.Println("unknown command:", os.Args[1])
	fmt.Println()
	help()
	os.Exit(1)
}

func loadFile(fileName string) []dependency.Dependency {
	cartridge, err := os.Open(fileName)
	if err != nil {
		return []dependency.Dependency{}
	}

	reader := dependency_reader.New(cartridge)

	dependencies, err := reader.ReadAll()
	if err != nil {
		fatal(err.Error())
	}

	return dependencies
}

func installDependencies(dependencies []dependency.Dependency, recursive bool, trickleDown bool, depth int) []dependency.Dependency {
	runner := command_runner.New()

	fetcher, err := dependency_fetcher.New(runner)
	if err != nil {
		fatal(err.Error())
	}

	maxWidth := 0

	for _, dep := range dependencies {
		if len(dep.Path) > maxWidth {
			maxWidth = len(dep.Path)
		}
	}

	lockedDependencies := []dependency.Dependency{}

	for _, dep := range dependencies {
		fmt.Println(bold(dep.Path) + padding(maxWidth-len(dep.Path)+2) + cyan(dep.Version))

		lockedDependency := processDependency(fetcher, dep)

		if depth == 0 && trickleDown {
			TrickleDownDependencies[lockedDependency.Path] = lockedDependency
		}

		lockedDependencies = append(lockedDependencies, lockedDependency)
	}

	if recursive {
		installNextLevel(lockedDependencies, depth+1)
	}

	return lockedDependencies
}

func processDependency(fetcher *dependency_fetcher.DependencyFetcher, dep dependency.Dependency) dependency.Dependency {
	dep = trickledDown(dep)

	lockedDependency, err := fetcher.Fetch(dep)
	if err != nil {
		fatal(err.Error())
	}

	checkForConflicts(lockedDependency)

	FetchedDependencies[lockedDependency.Path] = lockedDependency

	return lockedDependency
}

func updateLockFile(writer io.Writer, dependencies []dependency.Dependency) error {
	for _, dependency := range dependencies {
		_, err := writer.Write([]byte(fmt.Sprintf("%s\t%s\n", dependency.Path, dependency.Version)))
		if err != nil {
			return err
		}
	}

	return nil
}

func installNextLevel(deps []dependency.Dependency, newDepth int) {
	for _, dependency := range deps {
		dependencyPath := dependency.FullPath(GOPATH)

		if _, err := os.Stat(path.Join(dependencyPath, "Cartridge")); err == nil {
			fmt.Println("\nfetching dependencies for", dependency.Path)
			install(dependencyPath, true, false, false, newDepth)
		}
	}
}

func trickledDown(dep dependency.Dependency) dependency.Dependency {
	trickled, found := TrickleDownDependencies[dep.Path]
	if found {
		return trickled
	}

	return dep
}

func checkForConflicts(lockedDependency dependency.Dependency) {
	currentVersion, found := FetchedDependencies[lockedDependency.Path]
	if found && currentVersion.Version != lockedDependency.Version {
		fatal("version conflict for " + currentVersion.Path)
	}
}

func bold(str string) string {
	return "\x1b[1m" + str + "\x1b[0m"
}

func red(str string) string {
	return "\x1b[31m" + str + "\x1b[0m"
}

func cyan(str string) string {
	return "\x1b[36m" + str + "\x1b[0m"
}

func padding(size int) string {
	return strings.Repeat(" ", size)
}

func fatal(message string) {
	fmt.Fprintln(os.Stderr, red(message))
	os.Exit(1)
}
<|endoftext|>


<commit_before>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"bufio"
	"crypto/tls"
	"crypto/x509"
	"flag"
	"fmt"
	"github.com/nmeum/marvin/irc"
	"github.com/nmeum/marvin/modules"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/user"
	"path/filepath"
	"strings"
	"time"
)

const (
	appName  = "marvin"
	appUsage = "[options] CHANNEL..."
)

var (
	nick = flag.String("n", "marvin", "nickname")
	name = flag.String("r", "Marvin Bot", "realname")
	host = flag.String("h", "irc.hackint.eu", "host")
	cert = flag.String("c", "", "certificate")
	verb = flag.Bool("v", false, "verbose output")
	port = flag.Int("p", 6697, "port")
)

func main() {
	flag.Parse()
	logger := log.New(os.Stderr, "ERROR: ", 0)
	if flag.NArg() < 1 {
		logger.Fatalf("USAGE: %s %s", appName, appUsage)
	}

	conn, err := connect("tcp", fmt.Sprintf("%s:%d", *host, *port))
	if err != nil {
		logger.Fatal(err)
	}
	defer conn.Close()

	errChan := make(chan error)
	go func() {
		for err := range errChan {
			logger.Println(err)
		}
	}()

	for {
		ircBot, err := setup(&conn, flag.Args())
		if err != nil {
			logger.Println(err)
		}

		reader := bufio.NewReader(conn)
		for {
			line, err := reader.ReadString('\n')
			if err != nil {
				logger.Println(err)
				break
			}

			line = strings.Trim(line, "\n")
			line = strings.Trim(line, "\r")

			if *verb {
				fmt.Println(line)
			}

			ircBot.Handle(line, errChan)
		}

		conn = reconnect(conn)
	}
}

func setup(conn *net.Conn, channels []string) (client *irc.Client, err error) {
	client = irc.NewClient(conn)
	client.CmdHook("001", func(c *irc.Client, m irc.Message) error {
		time.Sleep(3 * time.Second) // Wait for NickServ etc
		return c.Write("JOIN %s", strings.Join(channels, ","))
	})

	client.CmdHook("kick", func(c *irc.Client, m irc.Message) error {
		params := strings.Fields(m.Receiver)
		return c.Write("JOIN %s", params[0])
	})

	client.Write("USER %s %s * :%s", *nick, *host, *name)
	client.Write("NICK %s", *nick)

	if err = initializeModules(client); err != nil {
		return
	}

	return
}

func initializeModules(c *irc.Client) error {
	config := os.Getenv("XDG_CONFIG_HOME")
	if len(config) <= 0 {
		user, err := user.Current()
		if err != nil {
			return err
		}

		config = filepath.Join(user.HomeDir, ".config")
	}

	moduleSet := modules.NewModuleSet(c, filepath.Join(config, appName))
	for _, fn := range moduleInits {
		fn(moduleSet)
	}

	return moduleSet.LoadAll()
}

func connect(network, address string) (conn net.Conn, err error) {
	if len(*cert) >= 1 {
		certFile, err := ioutil.ReadFile(*cert)
		if err != nil {
			return nil, err
		}

		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(certFile)

		config := &tls.Config{RootCAs: caCertPool}
		return tls.Dial(network, address, config)
	}

	return net.Dial(network, address)
}

func reconnect(c net.Conn) (conn net.Conn) {
	addr := c.RemoteAddr()
	c.Close()

	var err error
	for i := 1; ; i++ {
		conn, err = connect(addr.Network(), addr.String())
		if err == nil {
			break
		}

		time.Sleep(time.Duration(i*3) * time.Second)
	}

	return
}
<commit_msg>setup errors are fatal<commit_after>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"bufio"
	"crypto/tls"
	"crypto/x509"
	"flag"
	"fmt"
	"github.com/nmeum/marvin/irc"
	"github.com/nmeum/marvin/modules"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/user"
	"path/filepath"
	"strings"
	"time"
)

const (
	appName  = "marvin"
	appUsage = "[options] CHANNEL..."
)

var (
	nick = flag.String("n", "marvin", "nickname")
	name = flag.String("r", "Marvin Bot", "realname")
	host = flag.String("h", "irc.hackint.eu", "host")
	cert = flag.String("c", "", "certificate")
	verb = flag.Bool("v", false, "verbose output")
	port = flag.Int("p", 6697, "port")
)

func main() {
	flag.Parse()
	logger := log.New(os.Stderr, "ERROR: ", 0)
	if flag.NArg() < 1 {
		logger.Fatalf("USAGE: %s %s", appName, appUsage)
	}

	conn, err := connect("tcp", fmt.Sprintf("%s:%d", *host, *port))
	if err != nil {
		logger.Fatal(err)
	}
	defer conn.Close()

	errChan := make(chan error)
	go func() {
		for err := range errChan {
			logger.Println(err)
		}
	}()

	for {
		ircBot, err := setup(&conn, flag.Args())
		if err != nil {
			logger.Fatal(err)
		}

		reader := bufio.NewReader(conn)
		for {
			line, err := reader.ReadString('\n')
			if err != nil {
				logger.Println(err)
				break
			}

			line = strings.Trim(line, "\n")
			line = strings.Trim(line, "\r")

			if *verb {
				fmt.Println(line)
			}

			ircBot.Handle(line, errChan)
		}

		conn = reconnect(conn)
	}
}

func setup(conn *net.Conn, channels []string) (client *irc.Client, err error) {
	client = irc.NewClient(conn)
	client.CmdHook("001", func(c *irc.Client, m irc.Message) error {
		time.Sleep(3 * time.Second) // Wait for NickServ etc
		return c.Write("JOIN %s", strings.Join(channels, ","))
	})

	client.CmdHook("kick", func(c *irc.Client, m irc.Message) error {
		params := strings.Fields(m.Receiver)
		return c.Write("JOIN %s", params[0])
	})

	client.Write("USER %s %s * :%s", *nick, *host, *name)
	client.Write("NICK %s", *nick)

	if err = initializeModules(client); err != nil {
		return
	}

	return
}

func initializeModules(c *irc.Client) error {
	config := os.Getenv("XDG_CONFIG_HOME")
	if len(config) <= 0 {
		user, err := user.Current()
		if err != nil {
			return err
		}

		config = filepath.Join(user.HomeDir, ".config")
	}

	moduleSet := modules.NewModuleSet(c, filepath.Join(config, appName))
	for _, fn := range moduleInits {
		fn(moduleSet)
	}

	return moduleSet.LoadAll()
}

func connect(network, address string) (conn net.Conn, err error) {
	if len(*cert) >= 1 {
		certFile, err := ioutil.ReadFile(*cert)
		if err != nil {
			return nil, err
		}

		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(certFile)

		config := &tls.Config{RootCAs: caCertPool}
		return tls.Dial(network, address, config)
	}

	return net.Dial(network, address)
}

func reconnect(c net.Conn) (conn net.Conn) {
	addr := c.RemoteAddr()
	c.Close()

	var err error
	for i := 1; ; i++ {
		conn, err = connect(addr.Network(), addr.String())
		if err == nil {
			break
		}

		time.Sleep(time.Duration(i*3) * time.Second)
	}

	return
}
<|endoftext|>


<commit_before>/*
Copyright 2016, RadiantBlue Technologies, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/julienschmidt/httprouter"
)

func check(e error) {
	if e != nil {
		panic(e)
	}
}

// LasHeader is the LAS header.
type LasHeader struct {
	FileSignature                [4]byte
	FileSourceID                 uint16
	GlobalEncoding               uint16
	GUID1                        uint32
	GUID2                        uint16
	GUID3                        uint16
	GUID4                        [8]uint8
	VersionMajor, VersionMinor   uint8
	SystemIdentifier             [32]byte
	GeneratingSoftware           [32]byte
	FileCreationDayOfYear        uint16
	FileCreationYear             uint16
	HeaderSize                   uint16
	OffsetToPointData            uint32
	NumberOfVLRs                 uint32
	PointDataRecordFormat        uint8
	PointDataRecordLength        uint16
	LegacyNumberOfPointRecords   uint32
	LegacyNumberOfPointsByReturn [5]uint32
	XScale, YScale, ZScale       float64
	XOffset, YOffset, ZOffset    float64
	MaxX, MinX                   float64
	MaxY, MinY                   float64
	MaxZ, MinZ                   float64
	// missing some waveform packet stuff for 1.4
}

// Format0 is Point Data Record Format 0.
type Format0 struct {
	X, Y, Z        int32
	Intensity      uint16
	Foo            byte
	Classification uint8
	ScanAngleRank  byte
	UserData       uint8
	PointSourceID  uint16
}

// Format1 is Point Data Record Format 1.
type Format1 struct {
	Format0
	GPSTime float64
}

// Format3 is Point Date Record Format 3.
type Format3 struct {
	Format1
	Red, Blue, Green uint16
}

// ReadLas reads an LAS file.
func ReadLas(fname string) (h LasHeader, p []Format1, err error) {
	f, err := os.Open(fname)
	if err != nil {
		return h, p, err
	}
	defer f.Close()

	binary.Read(f, binary.LittleEndian, &h)
	fmt.Printf("%+v\n", h)

	f.Seek(int64(h.OffsetToPointData), 0)
	if h.PointDataRecordFormat == 0 {
		fmt.Println("format = 0")
		points := make([]Format0, h.LegacyNumberOfPointRecords)
		binary.Read(f, binary.LittleEndian, points)
		fmt.Printf("%+v\n", points[:5])
		fmt.Printf("%+v\n", points[len(points)-5:])
	} else if h.PointDataRecordFormat == 1 {
		fmt.Println("format = 1")
		p = make([]Format1, h.LegacyNumberOfPointRecords)
		binary.Read(f, binary.LittleEndian, p)
		fmt.Printf("%+v\n", p[:5])
		fmt.Printf("%+v\n", p[len(p)-5:])
	} else if h.PointDataRecordFormat == 3 {
		fmt.Println("format = 3")
		points := make([]Format3, h.LegacyNumberOfPointRecords)
		binary.Read(f, binary.LittleEndian, points)
		fmt.Printf("%+v\n", points[:5])
		fmt.Printf("%+v\n", points[len(points)-5:])
	}
	return h, p, nil
}

func main() {
	router := httprouter.New()

	router.GET("/", func(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) {
		fmt.Fprintf(w, "Hi!")
	})

	router.POST("/info", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
		// var inputName string
		// var fileIn *os.File

		// Split the source S3 key string, interpreting the last element as the
		// input filename. Create the input file, throwing 500 on error.
		// inputName = s3.ParseFilenameFromKey("pointcloud/samp11-utm.las")
		// fileIn, err := os.Create(inputName)
		// if err != nil {
		// 	http.Error(w, err.Error(), http.StatusInternalServerError)
		// 	return
		// }
		// defer fileIn.Close()

		type infoOptions struct {
			URL string `json:"url"`
		}

		var opts infoOptions
		if r.Body == nil {
			// bad request
		}

		b, err := ioutil.ReadAll(r.Body)
		if err != nil {
			// internal error
		}

		if err := json.Unmarshal(b, &opts); err != nil {
			// bad request
		}

		// rawURL := "https://s3.amazonaws.com/venicegeo-sample-data/pointcloud/samp11-utm.las"
		fileURL, err := url.Parse(opts.URL)
		path := fileURL.Path
		segments := strings.Split(path, "/")
		inputName := segments[len(segments)-1]
		fileIn, err := os.Create(inputName)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		defer fileIn.Close()

		// Download the source data from S3, throwing 500 on error.
		// err = s3.Download(fileIn, "venicegeo-sample-data", "pointcloud/samp11-utm.las")
		check := http.Client{
			CheckRedirect: func(r *http.Request, via []*http.Request) error {
				r.URL.Opaque = r.URL.Path
				return nil
			},
		}
		resp, err := check.Get(opts.URL)
		defer resp.Body.Close()
		_, err = io.Copy(fileIn, resp.Body)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		h, _, err := ReadLas(inputName)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}

		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(http.StatusOK)
		if err := json.NewEncoder(w).Encode(h); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	})

	var defaultPort = os.Getenv("PORT")
	if defaultPort == "" {
		defaultPort = "8080"
	}

	log.Println("Starting on ", defaultPort)
	if err := http.ListenAndServe(":"+defaultPort, router); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Better error messages, return on error<commit_after>/*
Copyright 2016, RadiantBlue Technologies, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/julienschmidt/httprouter"
)

func check(e error) {
	if e != nil {
		panic(e)
	}
}

// LasHeader is the LAS header.
type LasHeader struct {
	FileSignature                [4]byte
	FileSourceID                 uint16
	GlobalEncoding               uint16
	GUID1                        uint32
	GUID2                        uint16
	GUID3                        uint16
	GUID4                        [8]uint8
	VersionMajor, VersionMinor   uint8
	SystemIdentifier             [32]byte
	GeneratingSoftware           [32]byte
	FileCreationDayOfYear        uint16
	FileCreationYear             uint16
	HeaderSize                   uint16
	OffsetToPointData            uint32
	NumberOfVLRs                 uint32
	PointDataRecordFormat        uint8
	PointDataRecordLength        uint16
	LegacyNumberOfPointRecords   uint32
	LegacyNumberOfPointsByReturn [5]uint32
	XScale, YScale, ZScale       float64
	XOffset, YOffset, ZOffset    float64
	MaxX, MinX                   float64
	MaxY, MinY                   float64
	MaxZ, MinZ                   float64
	// missing some waveform packet stuff for 1.4
}

// Format0 is Point Data Record Format 0.
type Format0 struct {
	X, Y, Z        int32
	Intensity      uint16
	Foo            byte
	Classification uint8
	ScanAngleRank  byte
	UserData       uint8
	PointSourceID  uint16
}

// Format1 is Point Data Record Format 1.
type Format1 struct {
	Format0
	GPSTime float64
}

// Format3 is Point Date Record Format 3.
type Format3 struct {
	Format1
	Red, Blue, Green uint16
}

// ReadLas reads an LAS file.
func ReadLas(fname string) (h LasHeader, p []Format1, err error) {
	f, err := os.Open(fname)
	if err != nil {
		return h, p, err
	}
	defer f.Close()

	binary.Read(f, binary.LittleEndian, &h)
	fmt.Printf("%+v\n", h)

	f.Seek(int64(h.OffsetToPointData), 0)
	if h.PointDataRecordFormat == 0 {
		fmt.Println("format = 0")
		points := make([]Format0, h.LegacyNumberOfPointRecords)
		binary.Read(f, binary.LittleEndian, points)
		fmt.Printf("%+v\n", points[:5])
		fmt.Printf("%+v\n", points[len(points)-5:])
	} else if h.PointDataRecordFormat == 1 {
		fmt.Println("format = 1")
		p = make([]Format1, h.LegacyNumberOfPointRecords)
		binary.Read(f, binary.LittleEndian, p)
		fmt.Printf("%+v\n", p[:5])
		fmt.Printf("%+v\n", p[len(p)-5:])
	} else if h.PointDataRecordFormat == 3 {
		fmt.Println("format = 3")
		points := make([]Format3, h.LegacyNumberOfPointRecords)
		binary.Read(f, binary.LittleEndian, points)
		fmt.Printf("%+v\n", points[:5])
		fmt.Printf("%+v\n", points[len(points)-5:])
	}
	return h, p, nil
}

func main() {
	router := httprouter.New()

	router.GET("/", func(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) {
		fmt.Fprintf(w, "Hi!")
	})

	router.POST("/info", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
		type infoOptions struct {
			URL string `json:"url"`
		}

		var opts infoOptions
		if r.Body == nil {
			http.Error(w, "No body", http.StatusBadRequest)
			return
		}

		b, err := ioutil.ReadAll(r.Body)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		if err := json.Unmarshal(b, &opts); err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		fileURL, err := url.Parse(opts.URL)
		path := fileURL.Path
		segments := strings.Split(path, "/")
		inputName := segments[len(segments)-1]
		fileIn, err := os.Create(inputName)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		defer fileIn.Close()

		// Download the source data, throwing 500 on error.
		check := http.Client{
			CheckRedirect: func(r *http.Request, via []*http.Request) error {
				r.URL.Opaque = r.URL.Path
				return nil
			},
		}
		resp, err := check.Get(opts.URL)
		defer resp.Body.Close()
		_, err = io.Copy(fileIn, resp.Body)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		h, _, err := ReadLas(inputName)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(http.StatusOK)
		if err := json.NewEncoder(w).Encode(h); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	})

	var defaultPort = os.Getenv("PORT")
	if defaultPort == "" {
		defaultPort = "8080"
	}

	log.Println("Starting on ", defaultPort)
	if err := http.ListenAndServe(":"+defaultPort, router); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"net/url"
	"os"

	"github.com/CenturyLinkLabs/dray/api"
	"github.com/CenturyLinkLabs/dray/job"
	log "github.com/Sirupsen/logrus"
)

const (
	DefaultDockerEndpoint = "unix:///var/run/docker.sock"
)

func main() {
	a := job.NewJobAccessor(redisHost())
	cf := job.NewContainerFactory(dockerEndpoint())
	jm := job.NewJobManager(a, cf)

	s := api.NewServer(jm)
	s.Start(2000)
}

func redisHost() string {
	redisPort := os.Getenv("REDIS_PORT")

	if len(redisPort) == 0 {
		log.Error("Missing required REDIS_PORT environment variable")
	}

	u, err := url.Parse(redisPort)
	if err != nil {
		log.Errorf("Invalid Redis URL: %s", err)
		panic(err)
	}

	return u.Host
}

func dockerEndpoint() string {
	endpoint := os.Getenv("DOCKER_HOST")

	if len(endpoint) == 0 {
		endpoint = DefaultDockerEndpoint
	}

	return endpoint
}

// For streaming responses
//type flushWriter struct {
//flusher http.Flusher
//writer  io.Writer
//}

//func newFlushWriter(writer io.Writer) flushWriter {
//fw := flushWriter{writer: writer}
//if flusher, ok := writer.(http.Flusher); ok {
//fw.flusher = flusher
//}

//return fw
//}

//func (fw *flushWriter) Write(p []byte) (n int, err error) {
//n, err = fw.writer.Write(p)
//if fw.flusher != nil {
//fw.flusher.Flush()
//}
//return
//}
<commit_msg>Add command-line flag for server port<commit_after>package main

import (
	"flag"
	"net/url"
	"os"

	"github.com/CenturyLinkLabs/dray/api"
	"github.com/CenturyLinkLabs/dray/job"
	log "github.com/Sirupsen/logrus"
)

const (
	DefaultDockerEndpoint = "unix:///var/run/docker.sock"
)

func main() {
	port := flag.Int("p", 3000, "port on which the server will run")
	flag.Parse()

	a := job.NewJobAccessor(redisHost())
	cf := job.NewContainerFactory(dockerEndpoint())
	jm := job.NewJobManager(a, cf)

	s := api.NewServer(jm)
	s.Start(*port)
}

func redisHost() string {
	redisPort := os.Getenv("REDIS_PORT")

	if len(redisPort) == 0 {
		log.Error("Missing required REDIS_PORT environment variable")
	}

	u, err := url.Parse(redisPort)
	if err != nil {
		log.Errorf("Invalid Redis URL: %s", err)
		panic(err)
	}

	return u.Host
}

func dockerEndpoint() string {
	endpoint := os.Getenv("DOCKER_HOST")

	if len(endpoint) == 0 {
		endpoint = DefaultDockerEndpoint
	}

	return endpoint
}

// For streaming responses
//type flushWriter struct {
//flusher http.Flusher
//writer  io.Writer
//}

//func newFlushWriter(writer io.Writer) flushWriter {
//fw := flushWriter{writer: writer}
//if flusher, ok := writer.(http.Flusher); ok {
//fw.flusher = flusher
//}

//return fw
//}

//func (fw *flushWriter) Write(p []byte) (n int, err error) {
//n, err = fw.writer.Write(p)
//if fw.flusher != nil {
//fw.flusher.Flush()
//}
//return
//}
<|endoftext|>


<commit_before>/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* TODO: We have a problem with file dependencies. What if header files change?
 * All files dependent to this one should also be updated recursively. What
 * if the header is removed? Where would all the symbols go? What if the header
 * shows up again?
 *
 * We need to solve all this issues and it may require plenty of changes in the
 * symbols DB.
 */

package main

import (
	"container/list"
	"flag"
	fsnotify "gopkg.in/fsnotify.v1"
	"log"
	"net"
	"net/rpc"
	"net/rpc/jsonrpc"
	"os"
	"os/signal"
	"path/filepath"
	"regexp"
	"sync"
)

func processFile(files chan string, wg *sync.WaitGroup, parser *Parser) {
	wg.Add(1)
	defer wg.Done()

	// start exploring files
	for {
		file, ok := <-files

		if !ok {
			return
		}

		log.Println("parsing", file)
		parser.Parse(file)
	}
}

func explorePathToParse(path string,
	visitDir func(string),
	visitC func(string)) *list.List {

	path = filepath.Clean(path)

	f, err := os.Open(path)
	if err != nil {
		log.Println(err, "opening", path, ", ignoring")
		return nil
	}
	defer f.Close()

	info, err := f.Stat()
	if err != nil {
		log.Println(err, "stating", path, ", ignoring")
		return nil
	}

	// visit file
	if !info.IsDir() {
		// ignore non-C files
		validC, _ := regexp.MatchString(`.*\.[ch]$`, path)
		if validC {
			visitC(path)
		}
		return nil
	} else {
		visitDir(path)
	}

	// add all the files in the directory to explore
	dirFiles, err := f.Readdir(0)
	if err != nil {
		log.Println(err, " readdir ", path, ", ignoring")
		return nil
	}

	toExplore := list.New()
	for _, subf := range dirFiles {
		// ignore hidden files
		if subf.Name()[0] == '.' {
			continue
		}

		toExplore.PushBack(path + "/" + subf.Name())
	}
	return toExplore
}

func handleChange(event fsnotify.Event,
	db *SymbolsDB,
	watcher *fsnotify.Watcher,
	files chan string) {

	visitorDir := func(path string) {
		// add watcher to directory
		watcher.Add(path)
	}
	visitorC := func(path string) {
		// add watcher
		watcher.Add(path)
		// put file in channel
		files <- path
	}

	switch {
	case event.Op&(fsnotify.Write|fsnotify.Create|fsnotify.Chmod) != 0:
		explorePathToParse(event.Name, visitorDir, visitorC)
	case event.Op&(fsnotify.Remove|fsnotify.Rename) != 0:
		watcher.Remove(event.Name)
		tx := db.BeginTx()
		tx.RemoveFileReferences(event.Name)
		tx.Close()
	}
}

func main() {
	// path to symbols DB
	var dbFile string
	flag.StringVar(&dbFile, "db", ".navc_dbsymbols", "Path to symbols path")

	// number of parallel indexing threads
	var nIndexingThreads int
	flag.IntVar(&nIndexingThreads, "numThreads", 1,
		"Number of indexing threads (don't use)")

	// reset DB
	var resetDb bool
	flag.BoolVar(&resetDb, "resetDb", false, "Reset symbols DB and start over")

	flag.Parse()

	// socket file for communication with daemon
	socketFile := "/tmp/navc.sock"

	// list of directores with source to index
	var indexDir []string
	if len(flag.Args()) > 0 {
		indexDir = flag.Args()
	} else {
		indexDir = []string{"."}
	}

	// handle interrup and kill signals
	intr := make(chan os.Signal, 1)
	signal.Notify(intr, os.Interrupt, os.Kill)
	defer close(intr)

	var wg sync.WaitGroup
	defer wg.Wait()

	// if we need to reset the database, erase the old one
	if resetDb {
		os.Remove(dbFile)

	}

	// open databased of symbols
	db := OpenSymbolsDB(dbFile)
	defer db.Close()

	// create parser
	parser := NewParser(db, indexDir)

	// start indexing threads
	files := make(chan string, nIndexingThreads)
	defer close(files)

	// start threads to process files
	for i := 0; i < nIndexingThreads; i++ {
		go processFile(files, &wg, parser)
	}

	// start file watcher
	watcher, _ := fsnotify.NewWatcher()
	defer watcher.Close()
	go func() {
		wg.Add(1)
		defer wg.Done()
		for {
			select {
			case event, ok := <-watcher.Events:
				if !ok {
					return
				}
				handleChange(event, db, watcher, files)
			case err, ok := <-watcher.Errors:
				if !ok {
					return
				}

				log.Println("watcher error: ", err)
			}
		}
	}()

	// explore all the paths in indexDir and process all files
	removedFilesSet := db.GetSetFilesInDB()
	visitorDir := func(path string) {
		// add watcher to directory
		watcher.Add(path)
	}
	visitorC := func(path string) {
		// update set of removed files
		delete(removedFilesSet, path)
		// add watcher
		watcher.Add(path)
		// put file in channel
		files <- path
	}
	toExplore := list.New()
	for _, path := range indexDir {
		toExplore.PushBack(path)
	}
	for toExplore.Len() > 0 {
		// dequeue first path
		path := toExplore.Front()
		toExplore.Remove(path)

		newDirs := explorePathToParse(
			path.Value.(string),
			visitorDir,
			visitorC)
		if newDirs != nil {
			toExplore.PushBackList(newDirs)
		}
	}

	// remove from DB deleted files
	tx := db.BeginTx()
	for path := range removedFilesSet {
		tx.RemoveFileReferences(path)
	}
	tx.Close()

	// start serving requests
	os.Remove(socketFile)
	lis, err := net.Listen("unix", socketFile)
	if err != nil {
		log.Println("error opening socket", err)
		return
	}
	defer os.Remove(socketFile)
	defer lis.Close()

	handler := rpc.NewServer()
	handler.Register(&RequestHandler{db})
	go func() {
		wg.Add(1)
		defer wg.Done()

		for {
			conn, err := lis.Accept()
			if err != nil {
				log.Println("accepting connection (breaking):", err)
				return
			}

			codec := jsonrpc.NewServerCodec(conn)
			err = handler.ServeRequest(codec)
			if err != nil {
				log.Println("handling request (ignoring):", err)
			}
			codec.Close()
		}
	}()

	// wait until ctl-c is pressed
	select {
	case <-intr:
	}
}
<commit_msg>Add traverse path function to be used by main and the watcher handler<commit_after>/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* TODO: We have a problem with file dependencies. What if header files change?
 * All files dependent to this one should also be updated recursively. What
 * if the header is removed? Where would all the symbols go? What if the header
 * shows up again?
 *
 * We need to solve all this issues and it may require plenty of changes in the
 * symbols DB.
 */

package main

import (
	"container/list"
	"flag"
	fsnotify "gopkg.in/fsnotify.v1"
	"log"
	"net"
	"net/rpc"
	"net/rpc/jsonrpc"
	"os"
	"os/signal"
	"path/filepath"
	"regexp"
	"sync"
)

func processFile(files chan string, wg *sync.WaitGroup, parser *Parser) {
	wg.Add(1)
	defer wg.Done()

	// start exploring files
	for {
		file, ok := <-files

		if !ok {
			return
		}

		log.Println("parsing", file)
		parser.Parse(file)
	}
}

func explorePathToParse(path string,
	visitDir func(string),
	visitC func(string)) *list.List {

	path = filepath.Clean(path)

	f, err := os.Open(path)
	if err != nil {
		log.Println(err, "opening", path, ", ignoring")
		return nil
	}
	defer f.Close()

	info, err := f.Stat()
	if err != nil {
		log.Println(err, "stating", path, ", ignoring")
		return nil
	}

	// visit file
	if !info.IsDir() {
		// ignore non-C files
		validC, _ := regexp.MatchString(`.*\.[ch]$`, path)
		if validC {
			visitC(path)
		}
		return nil
	} else {
		visitDir(path)
	}

	// add all the files in the directory to explore
	dirFiles, err := f.Readdir(0)
	if err != nil {
		log.Println(err, " readdir ", path, ", ignoring")
		return nil
	}

	toExplore := list.New()
	for _, subf := range dirFiles {
		// ignore hidden files
		if subf.Name()[0] == '.' {
			continue
		}

		toExplore.PushBack(path + "/" + subf.Name())
	}
	return toExplore
}

func traversePath(path string, visitDir func(string), visitC func(string)) {
	toExplore := list.New()
	toExplore.PushBack(path)

	for toExplore.Len() > 0 {
		// dequeue first path
		path := toExplore.Front()
		toExplore.Remove(path)

		newDirs := explorePathToParse(
			path.Value.(string),
			visitDir,
			visitC)
		if newDirs != nil {
			toExplore.PushBackList(newDirs)
		}
	}
}

func handleChange(event fsnotify.Event,
	db *SymbolsDB,
	watcher *fsnotify.Watcher,
	files chan string) {

	visitorDir := func(path string) {
		// add watcher to directory
		watcher.Add(path)
	}
	visitorC := func(path string) {
		// add watcher
		watcher.Add(path)
		// put file in channel
		files <- path
	}

	switch {
	case event.Op&(fsnotify.Write|fsnotify.Create|fsnotify.Chmod) != 0:
		traversePath(event.Name, visitorDir, visitorC)
	case event.Op&(fsnotify.Remove|fsnotify.Rename) != 0:
		watcher.Remove(event.Name)
		tx := db.BeginTx()
		tx.RemoveFileReferences(filepath.Clean(event.Name))
		tx.Close()
	}
}

func main() {
	// path to symbols DB
	var dbFile string
	flag.StringVar(&dbFile, "db", ".navc_dbsymbols", "Path to symbols path")

	// number of parallel indexing threads
	var nIndexingThreads int
	flag.IntVar(&nIndexingThreads, "numThreads", 1,
		"Number of indexing threads (don't use)")

	// reset DB
	var resetDb bool
	flag.BoolVar(&resetDb, "resetDb", false,
		"Reset symbols DB and start over")

	flag.Parse()

	// socket file for communication with daemon
	socketFile := "/tmp/navc.sock"

	// list of directores with source to index
	var indexDir []string
	if len(flag.Args()) > 0 {
		indexDir = flag.Args()
	} else {
		indexDir = []string{"."}
	}

	// handle interrup and kill signals
	intr := make(chan os.Signal, 1)
	signal.Notify(intr, os.Interrupt, os.Kill)
	defer close(intr)

	var wg sync.WaitGroup
	defer wg.Wait()

	// if we need to reset the database, erase the old one
	if resetDb {
		os.Remove(dbFile)
	}

	// open databased of symbols
	db := OpenSymbolsDB(dbFile)
	defer db.Close()

	// create parser
	parser := NewParser(db, indexDir)

	// start indexing threads
	files := make(chan string, nIndexingThreads)
	defer close(files)

	// start threads to process files
	for i := 0; i < nIndexingThreads; i++ {
		go processFile(files, &wg, parser)
	}

	// start file watcher
	watcher, _ := fsnotify.NewWatcher()
	defer watcher.Close()
	go func() {
		wg.Add(1)
		defer wg.Done()
		for {
			select {
			case event, ok := <-watcher.Events:
				if !ok {
					return
				}
				handleChange(event, db, watcher, files)
			case err, ok := <-watcher.Errors:
				if !ok {
					return
				}

				log.Println("watcher error: ", err)
			}
		}
	}()

	// explore all the paths in indexDir and process all files
	removedFilesSet := db.GetSetFilesInDB()
	visitorDir := func(path string) {
		// add watcher to directory
		watcher.Add(path)
	}
	visitorC := func(path string) {
		// update set of removed files
		delete(removedFilesSet, path)
		// add watcher
		watcher.Add(path)
		// put file in channel
		files <- path
	}
	for _, path := range indexDir {
		traversePath(path, visitorDir, visitorC)
	}

	// remove from DB deleted files
	tx := db.BeginTx()
	for path := range removedFilesSet {
		tx.RemoveFileReferences(path)
	}
	tx.Close()

	// start serving requests
	os.Remove(socketFile)
	lis, err := net.Listen("unix", socketFile)
	if err != nil {
		log.Println("error opening socket", err)
		return
	}
	defer os.Remove(socketFile)
	defer lis.Close()

	handler := rpc.NewServer()
	handler.Register(&RequestHandler{db})
	go func() {
		wg.Add(1)
		defer wg.Done()

		for {
			conn, err := lis.Accept()
			if err != nil {
				log.Println("accepting connection (breaking):",
					err)
				return
			}

			codec := jsonrpc.NewServerCodec(conn)
			err = handler.ServeRequest(codec)
			if err != nil {
				log.Println("handling request (ignoring):", err)
			}
			codec.Close()
		}
	}()

	// wait until ctl-c is pressed
	select {
	case <-intr:
	}
}
<|endoftext|>


<commit_before>package main

import (
    "encoding/json"
    "fmt"
    "os"
)

func parseObj(in interface{}, out map[string]interface{}, prefix string) {
    switch vv := in.(type) {
    case map[string]interface{}:
        for key, value := range vv {
            parseObj(value, out, fmt.Sprintf("%s.%s", prefix, key))
        }
    case []interface{}:
        for index, value := range vv {
            parseObj(value, out, fmt.Sprintf("%s.%d", prefix, index))
        }
    case string:
        out[prefix[1:]] = vv
    case float64:
        out[prefix[1:]] = vv
    case bool:
        out[prefix[1:]] = vv
    default:
        fmt.Fprintln(os.Stderr, "Input appears to be invalid json")
        os.Exit(1)
    }
}

func main() {
    var in interface{}

    out := make(map[string]interface{})

    dec := json.NewDecoder(os.Stdin)

    dec.Decode(&in)

    parseObj(in, out, "")

    if len(os.Args) > 1 {
        key := os.Args[1]

        if value, ok := out[key]; ok {
            fmt.Println(value)
        }
    } else {
        for key, value := range out {
            fmt.Printf("%s=%v\n", key, value)
        }
    }
}
<commit_msg>Deal with nulls<commit_after>package main

import (
    "encoding/json"
    "fmt"
    "os"
)

func parseObj(in interface{}, out map[string]interface{}, prefix string) {
    switch vv := in.(type) {
    case map[string]interface{}:
        for key, value := range vv {
            parseObj(value, out, fmt.Sprintf("%s.%s", prefix, key))
        }
    case []interface{}:
        for index, value := range vv {
            parseObj(value, out, fmt.Sprintf("%s.%d", prefix, index))
        }
    case string:
        out[prefix[1:]] = vv
    case float64:
        out[prefix[1:]] = vv
    case bool:
        out[prefix[1:]] = vv
    case nil:
        // Do nothing!
    default:
        fmt.Fprintln(os.Stderr, "Input appears to be invalid json")
        os.Exit(1)
    }
}

func main() {
    var in interface{}

    out := make(map[string]interface{})

    dec := json.NewDecoder(os.Stdin)

    dec.Decode(&in)

    parseObj(in, out, "")

    if len(os.Args) > 1 {
        key := os.Args[1]

        if value, ok := out[key]; ok {
            fmt.Println(value)
        }
    } else {
        for key, value := range out {
            fmt.Printf("%s=%v\n", key, value)
        }
    }
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"

	"github.com/hashicorp/hcl"
	"github.com/hashicorp/hcl/hcl/ast"
	"github.com/kevinswiber/apg-hcl/config"
)

func main() {
	file := "./examples/helloworld.hcl"
	//file := "./examples/conditional_policy.hcl"

	d, err := ioutil.ReadFile(file)
	if err != nil {
		log.Fatalf("err: %s", err)
	}

	hclRoot, err := hcl.Parse(string(d))
	if err != nil {
		log.Fatalf("err: %s", err)
	}

	list, ok := hclRoot.Node.(*ast.ObjectList)
	if !ok {
		log.Fatalf("error parsing: file doesn't contain root object")
	}

	//config, err := config.LoadConfigFromHCL(list)
	c, err := config.LoadConfigFromHCL(list)
	if err != nil {
		log.Fatalf("err: %s", err)
	}

	var output []byte
	if output, err = xml.MarshalIndent(c.Proxy, "", "    "); err != nil {
		log.Fatalf("err: %s", err)
	}
	fmt.Printf("%s\n", string(output))

	/*
		for _, proxyEndpoint := range config.ProxyEndpoints {
			var output []byte
			output, err := xml.MarshalIndent(proxyEndpoint, "", "    ")
			if err != nil {
				log.Fatalf("err: %s", err)
			}
			fmt.Printf("%s\n", string(output))
		}

		for _, targetEndpoint := range config.TargetEndpoints {
			var output []byte
			output, err := xml.MarshalIndent(targetEndpoint, "", "    ")
			if err != nil {
				log.Fatalf("err: %s", err)
			}
			//fmt.Printf("%s\n", string(output))
		}
	*/
	for _, policy := range c.Policies {
		var output []byte
		output, err := xml.MarshalIndent(policy, "", "    ")
		if err != nil {
			log.Fatalf("err: %s", err)
		}
		fmt.Printf("%s\n", string(output))
	}
}
<commit_msg>Printing out the full bundle.<commit_after>package main

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"

	"github.com/hashicorp/hcl"
	"github.com/hashicorp/hcl/hcl/ast"
	"github.com/kevinswiber/apg-hcl/config"
)

func main() {
	file := "./examples/helloworld.hcl"
	//file := "./examples/conditional_policy.hcl"

	d, err := ioutil.ReadFile(file)
	if err != nil {
		log.Fatalf("err: %s", err)
	}

	hclRoot, err := hcl.Parse(string(d))
	if err != nil {
		log.Fatalf("err: %s", err)
	}

	list, ok := hclRoot.Node.(*ast.ObjectList)
	if !ok {
		log.Fatalf("error parsing: file doesn't contain root object")
	}

	c, err := config.LoadConfigFromHCL(list)
	if err != nil {
		log.Fatalf("err: %s", err)
	}

	var output []byte
	if output, err = xml.MarshalIndent(c.Proxy, "", "    "); err != nil {
		log.Fatalf("err: %s", err)
	}
	fmt.Printf("%s\n", string(output))

	for _, proxyEndpoint := range c.ProxyEndpoints {
		var output []byte
		output, err := xml.MarshalIndent(proxyEndpoint, "", "    ")
		if err != nil {
			log.Fatalf("err: %s", err)
		}
		fmt.Printf("%s\n", string(output))
	}

	for _, targetEndpoint := range c.TargetEndpoints {
		var output []byte
		output, err := xml.MarshalIndent(targetEndpoint, "", "    ")
		if err != nil {
			log.Fatalf("err: %s", err)
		}
		fmt.Printf("%s\n", string(output))
	}

	for _, policy := range c.Policies {
		var output []byte
		output, err := xml.MarshalIndent(policy, "", "    ")
		if err != nil {
			log.Fatalf("err: %s", err)
		}
		fmt.Printf("%s\n", string(output))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/fcgi"
	"os"
	"os/signal"
	"runtime"
	"syscall"
	"time"

	geoip2 "github.com/oschwald/geoip2-golang"
	"github.com/wlbr/scrummy/gotils"
)

//go:generate templify -o myip.go myip.tpl

const GEOIPFILENAME = "GeoLite2-City.mmdb"

const GEOIPURL = "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz"

//const GEOIPURL = "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz"
const ETAGFILE = GEOIPFILENAME + ".etag"

//set by linker. If AnalyticsId not set, then the tracking code will be omitted.
var AnalyticsId = ""
var AnalyticsSite = ""

//set by linker. If LogLevel not set, then the logging is shut off.
var LogLevel = ""

//set by linker. If LogFilet not set, then the logging is set to os.stdout
var LogFile = ""
var logger *gotils.Logger

type MyIpDate struct {
	Time                     string
	Req                      *http.Request
	RequestIP                string
	LookupIPs                []string
	LookupHostnames          []string
	Geo                      *geoip2.City
	City                     string
	Country                  string
	GeoIpFileLastUpdate      string
	GeoIpFileLastUpdateCheck string
	GoogleAnalyticsId        string
	GoogleAnalyticsSite      string
}

func NewMyIpDate(r *http.Request, ip string, lookupips, lookuphostnames []string, geo *geoip2.City, analyticsid, analyticssite string) *MyIpDate {
	return &MyIpDate{Time: time.Now().Format("January 2, 2006 15:04:05"),
		Req: r, RequestIP: ip, LookupIPs: lookupips, LookupHostnames: lookuphostnames,
		Geo: geo, City: geo.City.Names["en"], Country: geo.Country.Names["en"],
		GoogleAnalyticsId: analyticsid, GoogleAnalyticsSite: analyticssite}
}

func NewMyIpDateWithUpdate(r *http.Request, ip string, lookupips, lookuphostnames []string, geo *geoip2.City,
	analyticsid, analyticssite string, lastupdate time.Time, lastcheck time.Time) *MyIpDate {
	gipd := NewMyIpDate(r, ip, lookupips, lookuphostnames, geo, analyticsid, analyticssite)
	gipd.GeoIpFileLastUpdate = lastupdate.Format("January 2, 2006 15:04:05")
	gipd.GeoIpFileLastUpdateCheck = lastcheck.Format("January 2, 2006 15:04:05")
	return gipd
}

func gentemplate() *template.Template {
	tpl := myipTemplate()
	tmpl, _ := template.New("myip").Parse(tpl)
	return tmpl
}

func dialTCP4(network, addr string) (net.Conn, error) {
	return net.Dial("tcp4", addr)
}

/*
	checkDownload checks if there is a new GeoIpDatabase available on Maxminds site.
	To save bandwidth it checks the etag on the server (comparing this to a local etag file).
	The db will be downloaded only if the two etags differ.
	The etag is checked on the server only if the last check is more than a day ago.
*/

func checkDownload(uri string, file string, c chan bool) {
	var download bool
	var etag string

	tr := &http.Transport{Dial: dialTCP4}
	httpclient := &http.Client{Transport: tr}

	etagfildate, err := os.Stat(ETAGFILE)

	if err != nil || time.Now().After(etagfildate.ModTime().AddDate(0, 0, 1)) {
		//Checking Etag, as no etag file found or older than 1 day.
		download = false
		head, err := httpclient.Head(uri)

		if err != nil {
			logger.Error("Error retrieving GeoIpFile. %s", err.Error())
		}
		if head.Status != "200 OK" {
			logger.Error("Error retrieving GeoIpFile. Url: %s Status: %s", uri, head.Status)
			download = false
		} else {
			etag := head.Header.Get("Etag")
			if etag != "" { //etag in header
				fileetag, err := ioutil.ReadFile(ETAGFILE)
				if err != nil { //no old etag found, download
					download = true
				}
				os.Chtimes(ETAGFILE, time.Now(), time.Now())
				if etag != string(fileetag) { //old etag differs from servers one, so download
					download = true
				} else { //old filtype same as servers, do not download
					download = false
				}
			} else { //no etag in header, always download
				download = true
			}
		}
	} else {
		//Not checking Etag, last check less than 1 day ago
		download = false
	}

	if download {
		logger.Info("Downloading GeoIp database from %s.", GEOIPURL)
		out, err := ioutil.TempFile(".", "myip-geoiptmp-")
		defer out.Close()
		if err == nil {
			resp, _ := httpclient.Get(uri)
			etag = resp.Header.Get("Etag")
			serverdate, derr := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
			r, _ := gzip.NewReader(resp.Body)
			defer resp.Body.Close()
			if _, err := io.Copy(out, r); err != nil {
				logger.Fatal("Cannot read from stream!")
			} else {
				os.Rename(out.Name(), file)
				etagout, _ := ioutil.TempFile(".", "myip-etagtmp-")
				defer etagout.Close()
				etagout.Write([]byte(etag))
				os.Rename(etagout.Name(), ETAGFILE)
				if derr == nil {
					os.Chtimes(file, time.Now(), serverdate)
				}
			}
		} else {
			logger.Fatal("Cannot create file %s. %s", file+out.Name(), err.Error())
		}

	} else {
		logger.Info("Not downloading, still the same etag.")
	}
	c <- true
}

func getGeoIp(ip string, w http.ResponseWriter) (*geoip2.City, error) {
	logger.Info("Opening GeoIP database: %s", GEOIPFILENAME)
	db, err := geoip2.Open(GEOIPFILENAME)

	if err != nil {
		fmt.Fprintf(w, "Error finding GeoIpFile: %s
 \n\n", err)
		logger.Error("Error finding GeoIpFile: %s", err)
		return &geoip2.City{}, err
		//panic("Error open GeoIp: %s\n" + err.Error())
	}
	defer db.Close()

	// If you are using strings that may be invalid, check that ip is not nil
	pip := net.ParseIP(ip)

	record, err := db.City(pip)
	if err != nil {
		logger.Error("Error finding city for ip %s: %s", ip, err)
		fmt.Fprintf(w, "Error finding city: %s\n", err)
		return &geoip2.City{}, err
		//panic("Error find City: %s\n" + err.Error())
	} else {
		logger.Info("Found city for ip: %s", record.City.Names)
	}
	return record, err
}

func handler(w http.ResponseWriter, r *http.Request) {

	logger.Info("Starting request handler.")

	reqip := r.Header.Get("X-Forwarded-For")
	if reqip != "" { // ip from header, could be a cdn
		logger.Info("Got ip from X-Forwarded-For-Header")
		cfip := r.Header.Get("Cf-Connecting-Ip")
		if cfip != "" && cfip != reqip {
			logger.Warn("X-Forwarded-For-Header ip and Cf-Connecting-Ip differ. %s !0 %s", reqip, cfip)
		}
	} else {
		reqip = r.Header.Get("Cf-Connecting-Ip")
		if reqip != "" { //reading cloudflare header
			logger.Info("Got request through cloudflare, connecting ip is: %s", reqip)
		} else { // direct access without cloudflare
			var e error
			reqip, _, e = net.SplitHostPort(r.RemoteAddr)
			if e != nil {
				logger.Error("Error getting request ip. %s", e)
			} else {
				logger.Info("Got request from ip: %s", reqip)
			}
		}
	}

	lookuphosts, err := net.LookupAddr(reqip)
	var lookupips []string
	if err != nil {
		logger.Error("Error resolving ip address: %s error: %s", reqip, err)
	} else {
		for _, host := range lookuphosts {
			logger.Info("Resolving IPs for hostname: %s", host)
			ips, err := net.LookupHost(host)
			if err != nil {
				logger.Error("Error looking up host. %s error: %s", lookuphosts, err)
			} else {
				lookupips = append(lookupips, ips...)
			}
		}
	}

	record, err := getGeoIp(reqip, w)
	if err != nil {
		logger.Error("Error getting GeoIP for %s : $s", reqip, err)
		w.Header().Set("Refresh", fmt.Sprintf("5;url=%s", r.RequestURI))
		w.WriteHeader(http.StatusOK)
		fmt.Fprintf(w, "<meta http-equiv=\"refresh\" content=\"1;url=%s\"/>", r.RequestURI)
		fmt.Fprintf(w, "Downloading GeoIPDatabase ...\n")
	} else {
		geofiledate, err := os.Stat(GEOIPFILENAME)
		etagfiledate, eerr := os.Stat(ETAGFILE)
		if err == nil && eerr == nil {
			err = gentemplate().Execute(w, NewMyIpDateWithUpdate(r, reqip, lookupips, lookuphosts, record, AnalyticsId,
				AnalyticsSite, geofiledate.ModTime(), etagfiledate.ModTime()))
		} else {
			err = gentemplate().Execute(w, NewMyIpDate(r, reqip, lookupips, lookuphosts, record, AnalyticsId, AnalyticsSite))
		}
		if err != nil {
			fmt.Fprintf(w, "Error template: %s\n", err)
		}
	}
}

func main() {
	llevel, _ := gotils.LogLevelString(LogLevel)
	logger = gotils.NewLogger(LogFile, llevel)
	logger.Info("Starting up")

	d := make(chan bool, 10)
	go checkDownload(GEOIPURL, GEOIPFILENAME, d)

	runtime.GOMAXPROCS(runtime.NumCPU()) // use all CPU cores
	n := runtime.NumGoroutine() + 1      // initial number of Goroutines

	// install signal handler
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGTERM)
	// Spawn request handler

	go func() {
		err := fcgi.Serve(nil, http.HandlerFunc(handler))
		if err != nil {
			logger.Info("Not in fcgi mode so not spawing handler.")
			c <- syscall.SIGTERM
			//panic(err)
		} else {
			logger.Info("Spawing handler.")
		}
	}()

	// catch signal
	_ = <-d

	// give pending requests in fcgi.Serve() some time to enter the request handler
	time.Sleep(time.Millisecond * 100)

	// wait at most 3 seconds for request handlers to finish
	//inc ase we are downloading the GeoIPFile that may take a while
	for i := 0; i < 30; i++ {
		if runtime.NumGoroutine() <= n {
			return
		}
		time.Sleep(time.Millisecond * 100)
	}

	// catch finished downloader signal
	_ = <-c
}
<commit_msg>Improved logging.<commit_after>package main

import (
	"compress/gzip"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/fcgi"
	"os"
	"os/signal"
	"runtime"
	"syscall"
	"time"

	geoip2 "github.com/oschwald/geoip2-golang"
	"github.com/wlbr/scrummy/gotils"
)

//go:generate templify -o myip.go myip.tpl

const GEOIPFILENAME = "GeoLite2-City.mmdb"

const GEOIPURL = "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz"

//const GEOIPURL = "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz"
const ETAGFILE = GEOIPFILENAME + ".etag"

//set by linker. If AnalyticsId not set, then the tracking code will be omitted.
var AnalyticsId = ""
var AnalyticsSite = ""

//set by linker. If LogLevel not set, then the logging is shut off.
var LogLevel = ""

//set by linker. If LogFilet not set, then the logging is set to os.stdout
var LogFile = ""
var logger *gotils.Logger

type MyIpDate struct {
	Time                     string
	Req                      *http.Request
	RequestIP                string
	LookupIPs                []string
	LookupHostnames          []string
	Geo                      *geoip2.City
	City                     string
	Country                  string
	GeoIpFileLastUpdate      string
	GeoIpFileLastUpdateCheck string
	GoogleAnalyticsId        string
	GoogleAnalyticsSite      string
}

func NewMyIpDate(r *http.Request, ip string, lookupips, lookuphostnames []string, geo *geoip2.City, analyticsid, analyticssite string) *MyIpDate {
	return &MyIpDate{Time: time.Now().Format("January 2, 2006 15:04:05"),
		Req: r, RequestIP: ip, LookupIPs: lookupips, LookupHostnames: lookuphostnames,
		Geo: geo, City: geo.City.Names["en"], Country: geo.Country.Names["en"],
		GoogleAnalyticsId: analyticsid, GoogleAnalyticsSite: analyticssite}
}

func NewMyIpDateWithUpdate(r *http.Request, ip string, lookupips, lookuphostnames []string, geo *geoip2.City,
	analyticsid, analyticssite string, lastupdate time.Time, lastcheck time.Time) *MyIpDate {
	gipd := NewMyIpDate(r, ip, lookupips, lookuphostnames, geo, analyticsid, analyticssite)
	gipd.GeoIpFileLastUpdate = lastupdate.Format("January 2, 2006 15:04:05")
	gipd.GeoIpFileLastUpdateCheck = lastcheck.Format("January 2, 2006 15:04:05")
	return gipd
}

func gentemplate() *template.Template {
	tpl := myipTemplate()
	tmpl, _ := template.New("myip").Parse(tpl)
	return tmpl
}

func dialTCP4(network, addr string) (net.Conn, error) {
	return net.Dial("tcp4", addr)
}

/*
	checkDownload checks if there is a new GeoIpDatabase available on Maxminds site.
	To save bandwidth it checks the etag on the server (comparing this to a local etag file).
	The db will be downloaded only if the two etags differ.
	The etag is checked on the server only if the last check is more than a day ago.
*/

func checkDownload(uri string, file string, c chan bool) {
	var download bool
	var etag string

	tr := &http.Transport{Dial: dialTCP4}
	httpclient := &http.Client{Transport: tr}

	etagfildate, err := os.Stat(ETAGFILE)

	if err != nil || time.Now().After(etagfildate.ModTime().AddDate(0, 0, 1)) {
		//Checking Etag, as no etag file found or older than 1 day.
		logger.Info("Checking download from %s", GEOIPURL)
		download = false
		head, err := httpclient.Head(uri)

		if err != nil {
			logger.Error("Error retrieving GeoIpFile. %s", err.Error())
		}
		if head.Status != "200 OK" {
			logger.Error("Error retrieving GeoIpFile. Url: %s Status: %s", uri, head.Status)
			download = false
		} else {
			etag := head.Header.Get("Etag")
			if etag != "" { //etag in header
				fileetag, err := ioutil.ReadFile(ETAGFILE)
				if err != nil { //no old etag found, download
					download = true
					logger.Info("No local etag found, downloading.")
				}
				os.Chtimes(ETAGFILE, time.Now(), time.Now())
				if etag != string(fileetag) { //old etag differs from servers one, so download
					download = true
					logger.Info("Etag differs from server, downloading.")
				} else { //old filetype same as servers, do not download
					download = false
					logger.Info("Same etag, not downloading.")
				}
			} else { //no etag in header, always download
				download = true
				logger.Info("No etag found on server, downloading.")
			}
		}
	} else {
		//Not checking Etag, last check less than 1 day ago
		download = false
		logger.Info("Not checking for download, last check less than a day ago.")
	}

	if download {
		logger.Info("Downloading GeoIp database from %s.", GEOIPURL)
		out, err := ioutil.TempFile(".", "myip-geoiptmp-")
		defer out.Close()
		if err == nil {
			resp, _ := httpclient.Get(uri)
			etag = resp.Header.Get("Etag")
			serverdate, derr := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
			r, _ := gzip.NewReader(resp.Body)
			defer resp.Body.Close()
			if _, err := io.Copy(out, r); err != nil {
				logger.Fatal("Cannot read from stream!")
			} else {
				os.Rename(out.Name(), file)
				etagout, _ := ioutil.TempFile(".", "myip-etagtmp-")
				defer etagout.Close()
				etagout.Write([]byte(etag))
				os.Rename(etagout.Name(), ETAGFILE)
				if derr == nil {
					os.Chtimes(file, time.Now(), serverdate)
				}
			}
		} else {
			logger.Fatal("Cannot create file %s. %s", file+out.Name(), err.Error())
		}

	}
	c <- true
}

func getGeoIp(ip string, w http.ResponseWriter) (*geoip2.City, error) {
	logger.Info("Opening GeoIP database: %s", GEOIPFILENAME)
	db, err := geoip2.Open(GEOIPFILENAME)

	if err != nil {
		fmt.Fprintf(w, "Error finding GeoIpFile: %s
 \n\n", err)
		logger.Error("Error finding GeoIpFile: %s", err)
		return &geoip2.City{}, err
		//panic("Error open GeoIp: %s\n" + err.Error())
	}
	defer db.Close()

	// If you are using strings that may be invalid, check that ip is not nil
	pip := net.ParseIP(ip)

	record, err := db.City(pip)
	if err != nil {
		logger.Error("Error finding city for ip %s: %s", ip, err)
		fmt.Fprintf(w, "Error finding city: %s\n", err)
		return &geoip2.City{}, err
		//panic("Error find City: %s\n" + err.Error())
	} else {
		logger.Info("Found city for ip: %s", record.City.Names)
	}
	return record, err
}

func handler(w http.ResponseWriter, r *http.Request) {

	logger.Info("Starting request handler.")

	reqip := r.Header.Get("X-Forwarded-For")
	if reqip != "" { // ip from header, could be a cdn
		logger.Info("Got ip from X-Forwarded-For-Header")
		cfip := r.Header.Get("Cf-Connecting-Ip")
		if cfip != "" && cfip != reqip {
			logger.Warn("X-Forwarded-For-Header ip and Cf-Connecting-Ip differ. %s !0 %s", reqip, cfip)
		}
	} else {
		reqip = r.Header.Get("Cf-Connecting-Ip")
		if reqip != "" { //reading cloudflare header
			logger.Info("Got request through cloudflare, connecting ip is: %s", reqip)
		} else { // direct access without cloudflare
			var e error
			reqip, _, e = net.SplitHostPort(r.RemoteAddr)
			if e != nil {
				logger.Error("Error getting request ip. %s", e)
			} else {
				logger.Info("Got request from ip: %s", reqip)
			}
		}
	}

	lookuphosts, err := net.LookupAddr(reqip)
	var lookupips []string
	if err != nil {
		logger.Error("Error resolving ip address: %s error: %s", reqip, err)
	} else {
		for _, host := range lookuphosts {
			logger.Info("Resolving IPs for hostname: %s", host)
			ips, err := net.LookupHost(host)
			if err != nil {
				logger.Error("Error looking up host. %s error: %s", lookuphosts, err)
			} else {
				lookupips = append(lookupips, ips...)
			}
		}
	}

	record, err := getGeoIp(reqip, w)
	if err != nil {
		logger.Error("Error getting GeoIP for %s : $s", reqip, err)
		w.Header().Set("Refresh", fmt.Sprintf("5;url=%s", r.RequestURI))
		w.WriteHeader(http.StatusOK)
		fmt.Fprintf(w, "<meta http-equiv=\"refresh\" content=\"1;url=%s\"/>", r.RequestURI)
		fmt.Fprintf(w, "Downloading GeoIPDatabase ...\n")
	} else {
		geofiledate, err := os.Stat(GEOIPFILENAME)
		etagfiledate, eerr := os.Stat(ETAGFILE)
		if err == nil && eerr == nil {
			err = gentemplate().Execute(w, NewMyIpDateWithUpdate(r, reqip, lookupips, lookuphosts, record, AnalyticsId,
				AnalyticsSite, geofiledate.ModTime(), etagfiledate.ModTime()))
		} else {
			err = gentemplate().Execute(w, NewMyIpDate(r, reqip, lookupips, lookuphosts, record, AnalyticsId, AnalyticsSite))
		}
		if err != nil {
			fmt.Fprintf(w, "Error template: %s\n", err)
		}
	}
}

func main() {
	llevel, _ := gotils.LogLevelString(LogLevel)
	logger = gotils.NewLogger(LogFile, llevel)
	logger.Info("Starting up")

	d := make(chan bool, 10)
	go checkDownload(GEOIPURL, GEOIPFILENAME, d)

	runtime.GOMAXPROCS(runtime.NumCPU()) // use all CPU cores
	n := runtime.NumGoroutine() + 1      // initial number of Goroutines

	// install signal handler
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGTERM)
	// Spawn request handler

	go func() {
		err := fcgi.Serve(nil, http.HandlerFunc(handler))
		if err != nil {
			logger.Info("Not in fcgi mode so not spawing handler.")
			c <- syscall.SIGTERM
			//panic(err)
		} else {
			logger.Info("Spawing handler.")
		}
	}()

	// catch signal
	_ = <-d

	// give pending requests in fcgi.Serve() some time to enter the request handler
	time.Sleep(time.Millisecond * 100)

	// wait at most 3 seconds for request handlers to finish
	//inc ase we are downloading the GeoIPFile that may take a while
	for i := 0; i < 30; i++ {
		if runtime.NumGoroutine() <= n {
			return
		}
		time.Sleep(time.Millisecond * 100)
	}

	// catch finished downloader signal
	_ = <-c
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/kaneshin/giita/giita/client"
	"github.com/kaneshin/giita/giita/request"
	"io/ioutil"
	"os"
	"path"
	"time"
)

var (
	stdout      = os.Stdout
	stderr      = os.Stderr
	stdin       = os.Stdin
	programName = os.Args[0]
	team        string
	token       string
	conf        = ".giita"
)

func init() {
	filename := path.Join(os.Getenv("HOME"), conf)
	_, err := os.Stat(filename)
	if os.IsNotExist(err) {
		fmt.Fprintf(stderr, "%s doesn't exist.\n", filename)
		os.Exit(1)
	}
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	var data map[string]interface{}
	err = json.Unmarshal(b, &data)
	if err != nil {
		panic(err)
	}
	if val, ok := data["team"].(string); ok {
		team = val
	}
	if val, ok := data["token"].(string); ok {
		token = val
	}
}

func main() {
	page := 1
	limit := 100
	var users = make(map[string]int)
	for {
		fmt.Printf("Request: page %d, per_page %d\n", page, limit)
		req := request.NewItemRequestWithPageAndLimit(team, page, limit)
		cli := client.NewClient(token)
		body, err := cli.Dispatch(req)
		if err != nil {
			fmt.Fprintln(stderr, err)
		}
		var data []map[string]interface{}
		err = json.Unmarshal(body, &data)
		if err != nil {
			fmt.Fprintln(stderr, err)
		}
		for _, d := range data {
			if user, ok := d["user"].(map[string]interface{}); ok {
				if id, ok := user["id"].(string); ok {
					users[id]++
				}
			}
		}
		if len(data) < limit {
			break
		}
		page++
		time.Sleep(10 * time.Second)
	}
	for id, val := range users {
		fmt.Printf("%s, %d\n", id, val)
	}
}
<commit_msg>Pretty format<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/kaneshin/giita/giita/client"
	"github.com/kaneshin/giita/giita/request"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"time"
)

var (
	stdout      = os.Stdout
	stderr      = os.Stderr
	stdin       = os.Stdin
	programName = os.Args[0]
	team        string
	token       string
)

const (
	conf        = ".giita"
	waitSeconds = 1
)

func init() {
	filename := path.Join(os.Getenv("HOME"), conf)
	_, err := os.Stat(filename)
	if os.IsNotExist(err) {
		fmt.Fprintf(stderr, "%s doesn't exist.\n", filename)
		os.Exit(1)
	}
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	var data map[string]interface{}
	err = json.Unmarshal(b, &data)
	if err != nil {
		panic(err)
	}
	if val, ok := data["team"].(string); ok {
		team = val
	}
	if val, ok := data["token"].(string); ok {
		token = val
	}
}

func main() {
	page := 1
	limit := 100
	var users = make(map[string]map[string]interface{})
	for {
		req := request.NewUserRequestWithPageAndLimit(team, page, limit)
		cli := client.NewClient(token)
		body, err := cli.Dispatch(req)
		if err != nil {
			panic(err)
		}
		var data []map[string]interface{}
		err = json.Unmarshal(body, &data)
		if err != nil {
			panic(err)
		}
		for _, user := range data {
			if id, ok := user["id"].(string); ok {
				info := make(map[string]interface{})
				if val, ok := user["name"].(string); ok {
					if len(val) > 0 {
						info["name"] = val
					} else {
						info["name"] = id
					}
				} else {
					info["name"] = id
				}
				if val, ok := user["profile_image_url"].(string); ok {
					info["profile_image_url"] = val
				} else {
					info["profile_image_url"] = ""
				}
				if val, ok := user["items_count"].(float64); ok {
					info["items_count"] = (int)(val)
				} else {
					info["items_count"] = 0
				}
				if val, ok := user["description"].(string); ok {
					info["description"] = val
				} else {
					info["description"] = ""
				}
				info["team_items_count"] = 0
				users[id] = info
			}
		}
		if len(data) < limit {
			break
		}
		page++
		time.Sleep(waitSeconds * time.Second)
	}
	page = 1
	for {
		req := request.NewItemRequestWithPageAndLimit(team, page, limit)
		cli := client.NewClient(token)
		body, err := cli.Dispatch(req)
		if err != nil {
			panic(err)
		}
		var data []map[string]interface{}
		err = json.Unmarshal(body, &data)
		if err != nil {
			panic(err)
		}
		for _, d := range data {
			if user, ok := d["user"].(map[string]interface{}); ok {
				if id, ok := user["id"].(string); ok {
					if val, ok := users[id]["team_items_count"].(int); ok {
						users[id]["team_items_count"] = val + 1
					}
				}
			}
		}
		if len(data) < limit {
			break
		}
		page++
		time.Sleep(waitSeconds * time.Second)
	}
	var formatString = func(v map[string]interface{}, name string) string {
		if val, ok := v[name].(string); ok {
			return strings.Replace(val, "\n", " ", -1)
		}
		return ""
	}
	fmt.Println("| User Image | Name | Description | Qiita:Team posts | Qiita posts |")
	fmt.Println("| :--------: | :--- | :---------- | :--------------- | :---------- |")
	for id, val := range users {
		imgURL := formatString(val, "profile_image_url")
		name := formatString(val, "name")
		desc := formatString(val, "description")
		titems := val["team_items_count"].(int)
		items := val["items_count"].(int)
		fmt.Printf(`|  | %s | %s | %d | %d |
`, imgURL, id, name, desc, titems, items)
	}
}

func simple() {
	page := 1
	limit := 100
	var users = make(map[string]int)
	for {
		req := request.NewItemRequestWithPageAndLimit(team, page, limit)
		cli := client.NewClient(token)
		body, err := cli.Dispatch(req)
		if err != nil {
			panic(err)
		}
		var data []map[string]interface{}
		err = json.Unmarshal(body, &data)
		if err != nil {
			panic(err)
		}
		for _, d := range data {
			if user, ok := d["user"].(map[string]interface{}); ok {
				if id, ok := user["id"].(string); ok {
					users[id]++
				}
			}
		}
		if len(data) < limit {
			break
		}
		page++
		time.Sleep(waitSeconds * time.Second)
	}
	for id, val := range users {
		fmt.Printf("%s, %d\n", id, val)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"time"

	"github.com/googlecloudplatform/gcsfuse/fs"
	"github.com/googlecloudplatform/gcsfuse/mount"
	"github.com/googlecloudplatform/gcsfuse/perms"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fsutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcscaching"
	"golang.org/x/net/context"
)

var fBucketName = flag.String("bucket", "", "Name of GCS bucket to mount.")
var fMountPoint = flag.String("mount_point", "", "File system location.")

var fMountOptions = mount.OptionFlag(
	"o",
	"Additional system-specific mount options. Be careful!")

var fUid = flag.Int64(
	"uid",
	-1,
	"If non-negative, the UID that owns all inodes. The default is the UID of "+
		"the gcsfuse process.")

var fGid = flag.Int64(
	"gid",
	-1,
	"If non-negative, the GID that owns all inodes. The default is the GID of "+
		"the gcsfuse process.")

var fFileMode = flag.Uint(
	"file_mode",
	0644,
	"Permissions bits for files. Default is 0644.")

var fDirMode = flag.Uint(
	"dir_mode",
	0755,
	"Permissions bits for directories. Default is 0755.")

var fTempDir = flag.String(
	"temp_dir", "",
	"The temporary directory in which to store local copies of GCS objects. "+
		"If empty, the system default (probably /tmp) will be used.")

var fTempDirLimit = flag.Int64(
	"temp_dir_bytes", 1<<31,
	"A desired limit on the number of bytes used in --temp_dir. May be exceeded "+
		"for dirty files that have not been flushed or closed.")

var fGCSChunkSize = flag.Uint64(
	"gcs_chunk_size", 1<<24,
	"If set to a non-zero value N, split up GCS objects into multiple chunks of "+
		"size at most N when reading, and do not read or cache unnecessary chunks.")

var fImplicitDirs = flag.Bool(
	"implicit_dirs",
	false,
	"Implicitly define directories based on their content. See "+
		"docs/semantics.md.")

var fSupportNlink = flag.Bool(
	"support_nlink",
	false,
	"Return meaningful values for nlink from fstat(2). See docs/semantics.md.")

var fStatCacheTTL = flag.Duration(
	"stat_cache_ttl",
	time.Minute,
	"How long to cache StatObject results from GCS.")

var fTypeCacheTTL = flag.Duration(
	"type_cache_ttl",
	time.Minute,
	"How long to cache name -> file/dir type mappings in directory inodes.")

func getBucketName() string {
	s := *fBucketName
	if s == "" {
		fmt.Println("You must set --bucket.")
		os.Exit(1)
	}

	return s
}

func registerSIGINTHandler(mountPoint string) {
	// Register for SIGINT.
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)

	// Start a goroutine that will unmount when the signal is received.
	go func() {
		for {
			<-signalChan
			log.Println("Received SIGINT, attempting to unmount...")

			err := fuse.Unmount(mountPoint)
			if err != nil {
				log.Printf("Failed to unmount in response to SIGINT: %v", err)
			} else {
				log.Printf("Successfully unmounted in response to SIGINT.")
				return
			}
		}
	}()
}

func getBucket() (b gcs.Bucket) {
	// Set up a GCS connection.
	log.Println("Initializing GCS connection.")
	conn, err := getConn()
	if err != nil {
		log.Fatal("Couldn't get GCS connection: ", err)
	}

	// Extract the appropriate bucket.
	b = conn.GetBucket(getBucketName())

	// Enable cached StatObject results, if appropriate.
	if *fStatCacheTTL != 0 {
		const cacheCapacity = 4096
		b = gcscaching.NewFastStatBucket(
			*fStatCacheTTL,
			gcscaching.NewStatCache(cacheCapacity),
			timeutil.RealClock(),
			b)
	}

	return
}

func main() {
	// Make logging output better.
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	// Set up flags.
	flag.Parse()

	// Check --mount_point.
	if *fMountPoint == "" {
		log.Fatalf("You must set --mount_point.")
	}

	mountPoint := *fMountPoint

	// Sanity check: make sure the temporary directory exists and is writable
	// currently. This gives a better user experience than harder to debug EIO
	// errors when reading files in the future.
	if *fTempDir != "" {
		f, err := fsutil.AnonymousFile(*fTempDir)
		f.Close()

		if err != nil {
			log.Fatalf(
				"Error writing to temporary directory (%q); are you sure it exists "+
					"with the correct permissions?",
				err.Error())
		}
	}

	// Choose UID and GID.
	uid, gid, err := perms.MyUserAndGroup()
	if err != nil {
		err = fmt.Errorf("MyUserAndGroup: %v", err)
		return
	}

	if *fUid >= 0 {
		uid = uint32(*fUid)
	}

	if *fGid >= 0 {
		gid = uint32(*fGid)
	}

	// Create a file system server.
	bucket := getBucket()
	serverCfg := &fs.ServerConfig{
		Clock:               timeutil.RealClock(),
		Bucket:              bucket,
		TempDir:             *fTempDir,
		TempDirLimit:        *fTempDirLimit,
		GCSChunkSize:        *fGCSChunkSize,
		ImplicitDirectories: *fImplicitDirs,
		SupportNlink:        *fSupportNlink,
		DirTypeCacheTTL:     *fTypeCacheTTL,
		Uid:                 uid,
		Gid:                 gid,
		FilePerms:           os.FileMode(*fFileMode),
		DirPerms:            os.FileMode(*fDirMode),
	}

	server, err := fs.NewServer(serverCfg)
	if err != nil {
		log.Fatal("fs.NewServer:", err)
	}

	// Mount the file system.
	mountCfg := &fuse.MountConfig{
		FSName:  bucket.Name(),
		Options: fMountOptions,
	}

	mountedFS, err := fuse.Mount(mountPoint, server, mountCfg)
	if err != nil {
		log.Fatal("Mount:", err)
	}

	log.Println("File system has been successfully mounted.")

	// Let the user unmount with Ctrl-C (SIGINT).
	registerSIGINTHandler(mountedFS.Dir())

	// Wait for it to be unmounted.
	if err := mountedFS.Join(context.Background()); err != nil {
		log.Fatal("MountedFileSystem.Join:", err)
	}

	log.Println("Successfully exiting.")
}
<commit_msg>Print a warning in main if the rlimit is low.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"time"

	"github.com/googlecloudplatform/gcsfuse/fs"
	"github.com/googlecloudplatform/gcsfuse/mount"
	"github.com/googlecloudplatform/gcsfuse/perms"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fsutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcscaching"
	"golang.org/x/net/context"
	"golang.org/x/sys/unix"
)

var fBucketName = flag.String("bucket", "", "Name of GCS bucket to mount.")
var fMountPoint = flag.String("mount_point", "", "File system location.")

var fMountOptions = mount.OptionFlag(
	"o",
	"Additional system-specific mount options. Be careful!")

var fUid = flag.Int64(
	"uid",
	-1,
	"If non-negative, the UID that owns all inodes. The default is the UID of "+
		"the gcsfuse process.")

var fGid = flag.Int64(
	"gid",
	-1,
	"If non-negative, the GID that owns all inodes. The default is the GID of "+
		"the gcsfuse process.")

var fFileMode = flag.Uint(
	"file_mode",
	0644,
	"Permissions bits for files. Default is 0644.")

var fDirMode = flag.Uint(
	"dir_mode",
	0755,
	"Permissions bits for directories. Default is 0755.")

var fTempDir = flag.String(
	"temp_dir", "",
	"The temporary directory in which to store local copies of GCS objects. "+
		"If empty, the system default (probably /tmp) will be used.")

var fTempDirLimit = flag.Int64(
	"temp_dir_bytes", 1<<31,
	"A desired limit on the number of bytes used in --temp_dir. May be exceeded "+
		"for dirty files that have not been flushed or closed.")

var fGCSChunkSize = flag.Uint64(
	"gcs_chunk_size", 1<<24,
	"If set to a non-zero value N, split up GCS objects into multiple chunks of "+
		"size at most N when reading, and do not read or cache unnecessary chunks.")

var fImplicitDirs = flag.Bool(
	"implicit_dirs",
	false,
	"Implicitly define directories based on their content. See "+
		"docs/semantics.md.")

var fSupportNlink = flag.Bool(
	"support_nlink",
	false,
	"Return meaningful values for nlink from fstat(2). See docs/semantics.md.")

var fStatCacheTTL = flag.Duration(
	"stat_cache_ttl",
	time.Minute,
	"How long to cache StatObject results from GCS.")

var fTypeCacheTTL = flag.Duration(
	"type_cache_ttl",
	time.Minute,
	"How long to cache name -> file/dir type mappings in directory inodes.")

func getBucketName() string {
	s := *fBucketName
	if s == "" {
		fmt.Println("You must set --bucket.")
		os.Exit(1)
	}

	return s
}

func registerSIGINTHandler(mountPoint string) {
	// Register for SIGINT.
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)

	// Start a goroutine that will unmount when the signal is received.
	go func() {
		for {
			<-signalChan
			log.Println("Received SIGINT, attempting to unmount...")

			err := fuse.Unmount(mountPoint)
			if err != nil {
				log.Printf("Failed to unmount in response to SIGINT: %v", err)
			} else {
				log.Printf("Successfully unmounted in response to SIGINT.")
				return
			}
		}
	}()
}

func getBucket() (b gcs.Bucket) {
	// Set up a GCS connection.
	log.Println("Initializing GCS connection.")
	conn, err := getConn()
	if err != nil {
		log.Fatal("Couldn't get GCS connection: ", err)
	}

	// Extract the appropriate bucket.
	b = conn.GetBucket(getBucketName())

	// Enable cached StatObject results, if appropriate.
	if *fStatCacheTTL != 0 {
		const cacheCapacity = 4096
		b = gcscaching.NewFastStatBucket(
			*fStatCacheTTL,
			gcscaching.NewStatCache(cacheCapacity),
			timeutil.RealClock(),
			b)
	}

	return
}

func main() {
	// Make logging output better.
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	// Set up flags.
	flag.Parse()

	// Check --mount_point.
	if *fMountPoint == "" {
		log.Fatalf("You must set --mount_point.")
	}

	mountPoint := *fMountPoint

	// Sanity check: make sure the temporary directory exists and is writable
	// currently. This gives a better user experience than harder to debug EIO
	// errors when reading files in the future.
	if *fTempDir != "" {
		f, err := fsutil.AnonymousFile(*fTempDir)
		f.Close()

		if err != nil {
			log.Fatalf(
				"Error writing to temporary directory (%q); are you sure it exists "+
					"with the correct permissions?",
				err.Error())
		}
	}

	// The file leaser used by the file system sizes its limit on number of
	// temporary files based on the process's rlimit. If this is too low, we'll
	// throw away cached content unnecessarily often. This is particularly a
	// problem on OS X, which has a crazy low default limit (256 as of OS X
	// 10.10.3). So print a warning if the limit is low.
	var rlimit unix.Rlimit
	if err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit); err == nil {
		const reasonableLimit = 4096

		if rlimit.Cur < reasonableLimit {
			log.Printf(
				"Warning: low file rlimit of %d will cause cached content to be "+
					"frequently evicted. Consider raising with `ulimit -n`.",
				rlimit.Cur)
		}
	}

	// Choose UID and GID.
	uid, gid, err := perms.MyUserAndGroup()
	if err != nil {
		err = fmt.Errorf("MyUserAndGroup: %v", err)
		return
	}

	if *fUid >= 0 {
		uid = uint32(*fUid)
	}

	if *fGid >= 0 {
		gid = uint32(*fGid)
	}

	// Create a file system server.
	bucket := getBucket()
	serverCfg := &fs.ServerConfig{
		Clock:               timeutil.RealClock(),
		Bucket:              bucket,
		TempDir:             *fTempDir,
		TempDirLimit:        *fTempDirLimit,
		GCSChunkSize:        *fGCSChunkSize,
		ImplicitDirectories: *fImplicitDirs,
		SupportNlink:        *fSupportNlink,
		DirTypeCacheTTL:     *fTypeCacheTTL,
		Uid:                 uid,
		Gid:                 gid,
		FilePerms:           os.FileMode(*fFileMode),
		DirPerms:            os.FileMode(*fDirMode),
	}

	server, err := fs.NewServer(serverCfg)
	if err != nil {
		log.Fatal("fs.NewServer:", err)
	}

	// Mount the file system.
	mountCfg := &fuse.MountConfig{
		FSName:  bucket.Name(),
		Options: fMountOptions,
	}

	mountedFS, err := fuse.Mount(mountPoint, server, mountCfg)
	if err != nil {
		log.Fatal("Mount:", err)
	}

	log.Println("File system has been successfully mounted.")

	// Let the user unmount with Ctrl-C (SIGINT).
	registerSIGINTHandler(mountedFS.Dir())

	// Wait for it to be unmounted.
	if err := mountedFS.Join(context.Background()); err != nil {
		log.Fatal("MountedFileSystem.Join:", err)
	}

	log.Println("Successfully exiting.")
}
<|endoftext|>