<commit_before>package migs

import (
	"crypto/md5"
	"database/sql"
	"fmt"
	"reflect"
	"runtime"
	"strings"
	"time"

	"github.com/sirupsen/logrus"
)

type Con interface {
	QueryRow(query string, args ...interface{}) *sql.Row
	Query(query string, args ...interface{}) (*sql.Rows, error)
	Prepare(query string) (*sql.Stmt, error)
	Exec(query string, args ...interface{}) (sql.Result, error)
}

const createMigrationsSql = `
  CREATE TABLE migrations (idx INTEGER PRIMARY KEY NOT NULL, md5 UUID NOT NULL, statement VARCHAR NOT NULL, created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL)
`

func New(migrations ...interface{}) *Migrations {
	return &Migrations{steps: migrations}
}

type logger interface {
	Printf(string, ...interface{})
}

type Migrations struct {
	Logger logrus.FieldLogger
	steps  []interface{}
}

func (list Migrations) Pending(tx Tx) (out []*Migration, err error) {
	all, err := list.All(tx)
	if err != nil {
		return nil, err
	}
	for _, m := range all {
		if !m.Executed {
			out = append(out, m)
		}
	}
	return out, nil
}

func (list Migrations) Execute(db *sql.DB) error {
	tx, e := db.Begin()
	if e != nil {
		return e
	}

	e = list.ExecuteTx(tx)

	if e != nil {
		tx.Rollback()
		return e
	}
	return tx.Commit()
}

type Tx interface {
	Con
	Commit() error
	Rollback() error
}

func (list Migrations) ExecuteTx(tx Tx) error {
	started := time.Now()
	if _, err := list.setup(tx); err != nil {
		return err
	}

	migrations, err := list.All(tx)
	if err != nil {
		return err
	}
	for _, m := range migrations {
		if err := m.Execute(tx); err != nil {
			return err
		}
	}
	if list.Logger != nil {
		list.Logger.Printf("migrated in %.06f", time.Since(started).Seconds())
	}
	return nil
}

func (list Migrations) loadMigrationStatus(tx Tx) (m map[int]*Migration, err error) {
	m = map[int]*Migration{}
	q := "SELECT idx, md5, statement FROM migrations ORDER BY idx"
	rows, err := tx.Query(q)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	for rows.Next() {
		s := &Migration{}
		var cs string
		if err := rows.Scan(&s.Idx, &cs, &s.Statement); err != nil {
			return nil, err
		}
		s.MD5 = strings.Replace(cs, "-", "", -1)
		if _, ok := m[s.Idx]; ok {
			return nil, fmt.Errorf("there are two migrations with idx=%d", s.Idx)
		}
		m[s.Idx] = s
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	return m, nil
}

type migrationStatus struct {
	IDX       int
	Statement string
	MD5       string
}

func (list Migrations) All(tx Tx) (out []*Migration, err error) {
	status, err := list.loadMigrationStatus(tx)
	if err != nil {
		return nil, err
	}
	for idx, i := range list.steps {
		if m, err := newMigration(idx+1, i); err != nil {
			return nil, err
		} else {
			m.Logger = list.Logger
			m.MD5 = strings.Replace(m.Statement, "-", "", -1)
			executed, ok := status[m.Idx]
			if ok {
				if m.Statement != executed.Statement {
					return nil, fmt.Errorf("MIGRATION MISMATCH:\n<<<<<<< code migration %d\n%q\n=======\n%q\n>>>>>>> db migration\n", m.Idx, m.Statement, executed.Statement)
				}
				m.Executed = true
			}
			out = append(out, m)
		}
	}
	return out, nil
}

func newMigration(idx int, statement interface{}) (*Migration, error) {
	m := &Migration{Idx: idx}

	switch casted := statement.(type) {
	case string:
		m.Statement = casted
	case fmt.Stringer:
		m.Statement = casted.String()
	case func(Con) error:
		m.Statement = runtime.FuncForPC(reflect.ValueOf(statement).Pointer()).Name()
		m.Func = casted
	default:
		return nil, fmt.Errorf("type %T not supported", casted)
	}
	return m, nil
}

type Migration struct {
	Idx       int
	Statement string
	Func      func(Con) error
	Logger    logrus.FieldLogger
	MD5       string
	Executed  bool
}

func (list Migrations) setup(tx Tx) (sql.Result, error) {
	row := tx.QueryRow("SELECT COUNT(1) FROM pg_tables WHERE schemaname = $1 AND tablename = $2", "public", "migrations")
	var cnt int
	e := row.Scan(&cnt)
	if e != nil {
		return nil, e
	}

	if cnt == 0 {
		return tx.Exec(createMigrationsSql)
	}
	return nil, nil
}

func (m *Migration) debug(t string, dur time.Duration) {
	m.logWith(m.Logger.Debugf, t, dur)
}

func (m *Migration) log(t string, dur time.Duration) {
	m.logWith(m.Logger.Printf, t, dur)
}

func (m *Migration) logWith(logger func(string, ...interface{}), t string, dur time.Duration) {
	if m.Logger != nil {
		out := []string{}
		lines := strings.Split(strings.TrimSpace(m.Statement), "\n")
		for _, l := range lines {
			out = append(out, strings.TrimSpace(l))
		}
		msg := fmt.Sprintf("%s: migration %d %q %q", t, m.Idx, m.checksum(), strings.Join(strings.Fields(strings.Join(out, " ")), " "))
		if dur != 0 {
			msg += fmt.Sprintf(" [%.06f]", dur.Seconds())
		}
		logger(msg)
	}
}

func (m *Migration) checksum() string {
	return fmt.Sprintf("%x", md5.Sum([]byte(m.Statement)))
}

func (m *Migration) Execute(tx Tx) error {
	rows, err := tx.Query("SELECT md5, statement FROM migrations where idx = $1", m.Idx)
	if err != nil {
		return err
	}
	defer rows.Close()
	for rows.Next() {
		var cs, statement string
		if err := rows.Scan(&cs, &statement); err != nil {
			return err
		}
		cs = strings.Replace(cs, "-", "", -1)
		if statement == m.Statement {
			m.debug("SKIP", 0)
			return nil
		} else {
			return fmt.Errorf("MIGRATION MISMATCH:\n<<<<<<< code migration %d\n%q\n=======\n%q\n>>>>>>> db migration\n", m.Idx, m.Statement, statement)
		}
	}
	started := time.Now()
	if m.Func != nil {
		if err := m.Func(tx); err != nil {
			return err
		}
	} else {
		if _, err := tx.Exec(m.Statement); err != nil {
			return err
		}
	}
	_, err = tx.Exec("INSERT INTO migrations (idx, md5, statement, created_at) VALUES ($1, $2, $3, $4)", m.Idx, m.checksum(), m.Statement, time.Now().UTC().Format(time.RFC3339Nano))
	if err != nil {
		return err
	}
	m.log("EXEC", time.Since(started))
	return err
}
<commit_msg>fix logging when no logger is set<commit_after>package migs

import (
	"crypto/md5"
	"database/sql"
	"fmt"
	"reflect"
	"runtime"
	"strings"
	"time"

	"github.com/sirupsen/logrus"
)

type Con interface {
	QueryRow(query string, args ...interface{}) *sql.Row
	Query(query string, args ...interface{}) (*sql.Rows, error)
	Prepare(query string) (*sql.Stmt, error)
	Exec(query string, args ...interface{}) (sql.Result, error)
}

const createMigrationsSql = `
  CREATE TABLE migrations (idx INTEGER PRIMARY KEY NOT NULL, md5 UUID NOT NULL, statement VARCHAR NOT NULL, created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL)
`

func New(migrations ...interface{}) *Migrations {
	return &Migrations{steps: migrations}
}

type logger interface {
	Printf(string, ...interface{})
}

type Migrations struct {
	Logger logrus.FieldLogger
	steps  []interface{}
}

func (list Migrations) Pending(tx Tx) (out []*Migration, err error) {
	all, err := list.All(tx)
	if err != nil {
		return nil, err
	}
	for _, m := range all {
		if !m.Executed {
			out = append(out, m)
		}
	}
	return out, nil
}

func (list Migrations) Execute(db *sql.DB) error {
	tx, e := db.Begin()
	if e != nil {
		return e
	}

	e = list.ExecuteTx(tx)

	if e != nil {
		tx.Rollback()
		return e
	}
	return tx.Commit()
}

type Tx interface {
	Con
	Commit() error
	Rollback() error
}

func (list Migrations) ExecuteTx(tx Tx) error {
	started := time.Now()
	if _, err := list.setup(tx); err != nil {
		return err
	}

	migrations, err := list.All(tx)
	if err != nil {
		return err
	}
	for _, m := range migrations {
		if err := m.Execute(tx); err != nil {
			return err
		}
	}
	if list.Logger != nil {
		list.Logger.Printf("migrated in %.06f", time.Since(started).Seconds())
	}
	return nil
}

func (list Migrations) loadMigrationStatus(tx Tx) (m map[int]*Migration, err error) {
	m = map[int]*Migration{}
	q := "SELECT idx, md5, statement FROM migrations ORDER BY idx"
	rows, err := tx.Query(q)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	for rows.Next() {
		s := &Migration{}
		var cs string
		if err := rows.Scan(&s.Idx, &cs, &s.Statement); err != nil {
			return nil, err
		}
		s.MD5 = strings.Replace(cs, "-", "", -1)
		if _, ok := m[s.Idx]; ok {
			return nil, fmt.Errorf("there are two migrations with idx=%d", s.Idx)
		}
		m[s.Idx] = s
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	return m, nil
}

type migrationStatus struct {
	IDX       int
	Statement string
	MD5       string
}

func (list Migrations) All(tx Tx) (out []*Migration, err error) {
	status, err := list.loadMigrationStatus(tx)
	if err != nil {
		return nil, err
	}
	for idx, i := range list.steps {
		if m, err := newMigration(idx+1, i); err != nil {
			return nil, err
		} else {
			m.Logger = list.Logger
			m.MD5 = strings.Replace(m.Statement, "-", "", -1)
			executed, ok := status[m.Idx]
			if ok {
				if m.Statement != executed.Statement {
					return nil, fmt.Errorf("MIGRATION MISMATCH:\n<<<<<<< code migration %d\n%q\n=======\n%q\n>>>>>>> db migration\n", m.Idx, m.Statement, executed.Statement)
				}
				m.Executed = true
			}
			out = append(out, m)
		}
	}
	return out, nil
}

func newMigration(idx int, statement interface{}) (*Migration, error) {
	m := &Migration{Idx: idx}

	switch casted := statement.(type) {
	case string:
		m.Statement = casted
	case fmt.Stringer:
		m.Statement = casted.String()
	case func(Con) error:
		m.Statement = runtime.FuncForPC(reflect.ValueOf(statement).Pointer()).Name()
		m.Func = casted
	default:
		return nil, fmt.Errorf("type %T not supported", casted)
	}
	return m, nil
}

type Migration struct {
	Idx       int
	Statement string
	Func      func(Con) error
	Logger    logrus.FieldLogger
	MD5       string
	Executed  bool
}

func (list Migrations) setup(tx Tx) (sql.Result, error) {
	row := tx.QueryRow("SELECT COUNT(1) FROM pg_tables WHERE schemaname = $1 AND tablename = $2", "public", "migrations")
	var cnt int
	e := row.Scan(&cnt)
	if e != nil {
		return nil, e
	}

	if cnt == 0 {
		return tx.Exec(createMigrationsSql)
	}
	return nil, nil
}

func (m *Migration) debug(t string, dur time.Duration) {
	if m.Logger != nil {
		m.logWith(m.Logger.Debugf, t, dur)
	}
}

func (m *Migration) log(t string, dur time.Duration) {
	if m.Logger != nil {
		m.logWith(m.Logger.Printf, t, dur)
	}
}

func (m *Migration) logWith(logger func(string, ...interface{}), t string, dur time.Duration) {
	out := []string{}
	lines := strings.Split(strings.TrimSpace(m.Statement), "\n")
	for _, l := range lines {
		out = append(out, strings.TrimSpace(l))
	}
	msg := fmt.Sprintf("%s: migration %d %q %q", t, m.Idx, m.checksum(), strings.Join(strings.Fields(strings.Join(out, " ")), " "))
	if dur != 0 {
		msg += fmt.Sprintf(" [%.06f]", dur.Seconds())
	}
	logger(msg)
}

func (m *Migration) checksum() string {
	return fmt.Sprintf("%x", md5.Sum([]byte(m.Statement)))
}

func (m *Migration) Execute(tx Tx) error {
	rows, err := tx.Query("SELECT md5, statement FROM migrations where idx = $1", m.Idx)
	if err != nil {
		return err
	}
	defer rows.Close()
	for rows.Next() {
		var cs, statement string
		if err := rows.Scan(&cs, &statement); err != nil {
			return err
		}
		cs = strings.Replace(cs, "-", "", -1)
		if statement == m.Statement {
			m.debug("SKIP", 0)
			return nil
		} else {
			return fmt.Errorf("MIGRATION MISMATCH:\n<<<<<<< code migration %d\n%q\n=======\n%q\n>>>>>>> db migration\n", m.Idx, m.Statement, statement)
		}
	}
	started := time.Now()
	if m.Func != nil {
		if err := m.Func(tx); err != nil {
			return err
		}
	} else {
		if _, err := tx.Exec(m.Statement); err != nil {
			return err
		}
	}
	_, err = tx.Exec("INSERT INTO migrations (idx, md5, statement, created_at) VALUES ($1, $2, $3, $4)", m.Idx, m.checksum(), m.Statement, time.Now().UTC().Format(time.RFC3339Nano))
	if err != nil {
		return err
	}
	m.log("EXEC", time.Since(started))
	return err
}
<|endoftext|>


<commit_before>// HTTP/2 web server with built-in support for Lua, Markdown, GCSS, Amber and JSX.
package main

import (
	"errors"
	"fmt"
	log "github.com/sirupsen/logrus"
	"io/ioutil"
	internallog "log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/bradfitz/http2"
	"github.com/tylerb/graceful"
	bolt "github.com/xyproto/permissionbolt"
	redis "github.com/xyproto/permissions2"
	mariadb "github.com/xyproto/permissionsql"
	"github.com/xyproto/pinterface"
	"github.com/xyproto/simpleredis"
	"github.com/xyproto/unzip"
	"github.com/yuin/gopher-lua"
)

const (
	versionString = "Algernon 0.72"
	description   = "HTTP/2 Web Server"
)

var (
	// List of filenames that should be displayed instead of a directory listing
	indexFilenames = []string{"index.lua", "index.html", "index.md", "index.txt", "index.amber"}

	// For convenience. Set in the main function.
	serverHost      string
	dbName          string
	refreshDuration time.Duration
)

func newServerConfiguration(mux *http.ServeMux, http2support bool, addr string) *http.Server {
	// Server configuration
	s := &http.Server{
		Addr:    addr,
		Handler: mux,

		// The timeout values is also the maximum time it can take
		// for a complete page of Server-Sent Events (SSE).
		ReadTimeout:  10 * time.Second,
		WriteTimeout: 10 * time.Second,

		MaxHeaderBytes: 1 << 20,
	}
	if http2support {
		// Enable HTTP/2 support
		http2.ConfigureServer(s, nil)
	}
	return s
}

func main() {
	var (
		err  error
		perm pinterface.IPermissions
	)

	// Use all CPUs. Soon to be the default for Go.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Temporary directory that might be used for logging, databases or file extraction
	serverTempDir, err := ioutil.TempDir("", "algernon")
	if err != nil {
		log.Fatalln(err)
	}
	defer os.RemoveAll(serverTempDir)

	// Set several configuration variables, based on the given flags and arguments
	serverHost = handleFlags(serverTempDir)

	// Version
	if showVersion {
		fmt.Println(versionString)
		os.Exit(0)
	}

	// Console output
	fmt.Println(banner())

	// Dividing line between the banner and output from any of the configuration scripts
	if len(serverConfigurationFilenames) > 0 {
		fmt.Println("--------------------------------------- - - · ·")
	}

	// Request handlers
	mux := http.NewServeMux()

	// Read mime data from the system, if available
	initializeMime()

	// Check if the given directory really is a directory
	if !isDir(serverDir) {
		// Possibly a file
		filename := serverDir
		// Check if the file exists
		if exists(filename) {
			// Switch based on the lowercase filename extension
			switch strings.ToLower(filepath.Ext(filename)) {
			case ".md":
				// Serve the given Markdown file as a static HTTP server
				serveStaticFile(filename, defaultWebColonPort)
				return
			case ".zip":
				// It might be a compressed Algernon application
				err := unzip.Extract(filename, serverTempDir)
				if err != nil {
					log.Fatalln(err)
				}
				// Use the directory where the file was extracted as the server directory
				serverDir = serverTempDir
				// If there is only one directory there, assume it's the directory of
				// the newly extracted ZIP file.
				if filenames := getFilenames(serverDir); len(filenames) == 1 {
					fullPath := filepath.Join(serverDir, filenames[0])
					if isDir(fullPath) {
						// Use this as the server directory instead
						serverDir = fullPath
					}
				}
				// If there are server configuration files in the extracted directory, register them
				for _, filename := range serverConfigurationFilenames {
					configFilename := filepath.Join(serverDir, filename)
					if exists(configFilename) {
						serverConfigurationFilenames = append(serverConfigurationFilenames, configFilename)
					}
				}
			default:
				singleFileMode = true
			}
		} else {
			fatalExit(errors.New("File does not exist: " + filename))
		}
	}

	// Make a few changes to the defaults if we are serving a single file
	if singleFileMode {
		debugMode = true
		serveJustHTTP = true
	}

	// If Bolt is to be used and no filename is given
	if useBolt && (boltFilename == "") {
		boltFilename = defaultBoltFilename
	}

	// Use one of the databases for the permission middleware,
	// then assign a name to dbName (used for the status output)
	dbName = ""
	if boltFilename != "" {
		// New permissions middleware, using a Bolt database
		perm, err = bolt.NewWithConf(boltFilename)
		if err != nil {
			log.Errorf("Could not use Bolt as database backend: %s", err)
		} else {
			dbName = "Bolt (" + boltFilename + ")"
		}
	}
	if dbName == "" && mariadbDSN != "" {
		// New permissions middleware, using a MariaDB/MySQL database
		perm, err = mariadb.NewWithDSN(mariadbDSN, mariadbDatabase)
		if err != nil {
			log.Errorf("Could not use MariaDB/MySQL as database backend: %s", err)
		} else {
			// The connection string may contain a password, so don't include it in the dbName
			dbName = "MariaDB/MySQL"
		}
	}
	if dbName == "" && mariadbDatabase != "" {
		// Given a database, but not a host, connect to localhost
		// New permissions middleware, using a MariaDB/MySQL database
		perm, err = mariadb.NewWithConf("test:@127.0.0.1/" + mariadbDatabase)
		if err != nil {
			log.Errorf("Could not use MariaDB/MySQL as database backend: %s", err)
		} else {
			// The connection string may contain a password, so don't include it in the dbName
			dbName = "MariaDB/MySQL"
		}
	}
	if dbName == "" {
		// New permissions middleware, using a Redis database
		if err := simpleredis.TestConnectionHost(redisAddr); err != nil {
			// Only warn when not in single file mode (too verbose)
			if !singleFileMode {
				log.Errorf("Could not use Redis as database backend: %s", err)
			}
		} else {
			perm = redis.NewWithRedisConf(redisDBindex, redisAddr)
			dbName = "Redis"
		}
	}
	if dbName == "" && boltFilename == "" {
		perm, err = bolt.NewWithConf(defaultBoltFilename)
		if err != nil {
			log.Errorf("Could not use Bolt as database backend: %s", err)
		} else {
			dbName = "Bolt (" + defaultBoltFilename + ")"
		}
	}
	if dbName == "" {
		// This may typically happen if Algernon is already running
		log.Fatalln("Could not find a usable database backend.")
	}

	// Lua LState pool
	luapool := &lStatePool{saved: make([]*lua.LState, 0, 4)}
	defer luapool.Shutdown()

	// Register HTTP handler functions
	registerHandlers(mux, serverDir, perm, luapool)

	if serverLogFile != "" {
		f, err := os.OpenFile(serverLogFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			log.Error("Could not log to", serverLogFile)
			fatalExit(err)
		}
		log.SetFormatter(&log.JSONFormatter{})
		log.SetOutput(f)
	}

	// Read server configuration script, if present.
	// The scripts may change global variables.
	for _, filename := range serverConfigurationFilenames {
		if exists(filename) {
			if err := runConfiguration(filename, perm, luapool); err != nil {
				log.Error("Could not use configuration script: " + filename)
				fatalExit(err)
			}
		}
	}

	// Set the values that has not been set by flags nor scripts (and can be set by both)
	ranServerReadyFunction := finalConfiguration(serverHost)

	// Dividing line between the banner and output from any of the configuration scripts
	if ranServerReadyFunction {
		fmt.Println("--------------------------------------- - - · ·")
	}

	// If we are not keeping the logs, reduce the verboseness
	http2.VerboseLogs = (serverHTTP2log != "/dev/null")

	// Direct the logging from the http2 package elsewhere
	f, err := os.Open(serverHTTP2log)
	defer f.Close()
	if err != nil {
		// Could not open the serverHTTP2log filename, try using another filename
		f, err = os.OpenFile("http2.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
		defer f.Close()
		if err != nil {
			fatalExit(fmt.Errorf("Could not write to %s nor %s.", serverHTTP2log, "http2.log"))
		}
	}
	internallog.SetOutput(f)

	// Serve filesystem events in the background.
	// Used for reloading pages when the sources change.
	// Can also be used when serving a single file.
	if autoRefresh {
		refreshDuration, err = time.ParseDuration(eventRefresh)
		if err != nil {
			log.Warn(fmt.Sprintf("%s is an invalid duration. Using %s instead.", eventRefresh, defaultEventRefresh))
			// Ignore the error, since defaultEventRefresh is a constant and must be parseable
			refreshDuration, _ = time.ParseDuration(defaultEventRefresh)
		}
		if autoRefreshDir != "" {
			// Only watch the autoRefreshDir, recursively
			EventServer(eventAddr, defaultEventPath, autoRefreshDir, refreshDuration, "*")
		} else {
			// Watch everything in the server directory, recursively
			EventServer(eventAddr, defaultEventPath, serverDir, refreshDuration, "*")
		}
	}

	if interactiveMode {
		go REPL(perm, luapool)
	}

	// Timeout when closing down the server
	shutdownTimeout := 30 * time.Second

	// Decide which protocol to listen to
	switch {
	case productionMode:
		// Listen for both HTTPS+HTTP/2 and HTTP requests, on different ports
		go func() {
			log.Info("Serving HTTPS + HTTP/2 on " + serverHost + ":443")
			// Start serving. Shut down gracefully at exit.
			// Listen for HTTPS + HTTP/2 requests
			HTTPS2server := newServerConfiguration(mux, true, serverHost+":443")
			// Start serving. Shut down gracefully at exit.
			if err := graceful.ListenAndServeTLS(HTTPS2server, serverCert, serverKey, shutdownTimeout); err != nil {
				log.Error(err)
			}
		}()
		log.Info("Serving HTTP on " + serverHost + ":80")
		HTTPserver := newServerConfiguration(mux, false, serverHost+":80")
		if err := graceful.ListenAndServe(HTTPserver, shutdownTimeout); err != nil {
			// If we can't serve regular HTTP on port 80, give up
			fatalExit(err)
		}
	case serveJustHTTP2: // It's unusual to serve HTTP/2 withoutHTTPS
		log.Info("Serving HTTP/2 on " + serverAddr)
		// Listen for HTTP/2 requests
		HTTP2server := newServerConfiguration(mux, true, serverAddr)
		// Start serving. Shut down gracefully at exit.
		if err := graceful.ListenAndServe(HTTP2server, shutdownTimeout); err != nil {
			fatalExit(err)
		}
	case !(serveJustHTTP2 || serveJustHTTP):
		log.Info("Serving HTTPS + HTTP/2 on " + serverAddr)
		// Listen for HTTPS + HTTP/2 requests
		HTTPS2server := newServerConfiguration(mux, true, serverAddr)
		// Start serving. Shut down gracefully at exit.
		if err := graceful.ListenAndServeTLS(HTTPS2server, serverCert, serverKey, shutdownTimeout); err != nil {
			log.Error(err)
			// If HTTPS failed (perhaps the key + cert are missing), serve
			// plain HTTP instead, by falling through to the next case.
		} else {
			// Don't fall through
			break
		}
		// Serve regular HTTP instead
		fallthrough
	default:
		log.Info("Serving HTTP on " + serverAddr)
		HTTPserver := newServerConfiguration(mux, false, serverAddr)
		// Start serving. Shut down gracefully at exit.
		if err := graceful.ListenAndServe(HTTPserver, shutdownTimeout); err != nil {
			// If we can't serve regular HTTP, give up
			fatalExit(err)
		}
	}
}
<commit_msg>Not loading local conf when running from zip<commit_after>// HTTP/2 web server with built-in support for Lua, Markdown, GCSS, Amber and JSX.
package main

import (
	"errors"
	"fmt"
	log "github.com/sirupsen/logrus"
	"io/ioutil"
	internallog "log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/bradfitz/http2"
	"github.com/tylerb/graceful"
	bolt "github.com/xyproto/permissionbolt"
	redis "github.com/xyproto/permissions2"
	mariadb "github.com/xyproto/permissionsql"
	"github.com/xyproto/pinterface"
	"github.com/xyproto/simpleredis"
	"github.com/xyproto/unzip"
	"github.com/yuin/gopher-lua"
)

const (
	versionString = "Algernon 0.72"
	description   = "HTTP/2 Web Server"
)

var (
	// List of filenames that should be displayed instead of a directory listing
	indexFilenames = []string{"index.lua", "index.html", "index.md", "index.txt", "index.amber"}

	// For convenience. Set in the main function.
	serverHost      string
	dbName          string
	refreshDuration time.Duration
)

func newServerConfiguration(mux *http.ServeMux, http2support bool, addr string) *http.Server {
	// Server configuration
	s := &http.Server{
		Addr:    addr,
		Handler: mux,

		// The timeout values is also the maximum time it can take
		// for a complete page of Server-Sent Events (SSE).
		ReadTimeout:  10 * time.Second,
		WriteTimeout: 10 * time.Second,

		MaxHeaderBytes: 1 << 20,
	}
	if http2support {
		// Enable HTTP/2 support
		http2.ConfigureServer(s, nil)
	}
	return s
}

func main() {
	var (
		err  error
		perm pinterface.IPermissions
	)

	// Use all CPUs. Soon to be the default for Go.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Temporary directory that might be used for logging, databases or file extraction
	serverTempDir, err := ioutil.TempDir("", "algernon")
	if err != nil {
		log.Fatalln(err)
	}
	defer os.RemoveAll(serverTempDir)

	// Set several configuration variables, based on the given flags and arguments
	serverHost = handleFlags(serverTempDir)

	// Version
	if showVersion {
		fmt.Println(versionString)
		os.Exit(0)
	}

	// Console output
	fmt.Println(banner())

	// Dividing line between the banner and output from any of the configuration scripts
	if len(serverConfigurationFilenames) > 0 {
		fmt.Println("--------------------------------------- - - · ·")
	}

	// Request handlers
	mux := http.NewServeMux()

	// Read mime data from the system, if available
	initializeMime()

	// Check if the given directory really is a directory
	if !isDir(serverDir) {
		// Possibly a file
		filename := serverDir
		// Check if the file exists
		if exists(filename) {
			// Switch based on the lowercase filename extension
			switch strings.ToLower(filepath.Ext(filename)) {
			case ".md":
				// Serve the given Markdown file as a static HTTP server
				serveStaticFile(filename, defaultWebColonPort)
				return
			case ".zip":
				// It might be a compressed Algernon application
				err := unzip.Extract(filename, serverTempDir)
				if err != nil {
					log.Fatalln(err)
				}
				// Use the directory where the file was extracted as the server directory
				serverDir = serverTempDir
				// If there is only one directory there, assume it's the directory of
				// the newly extracted ZIP file.
				if filenames := getFilenames(serverDir); len(filenames) == 1 {
					fullPath := filepath.Join(serverDir, filenames[0])
					if isDir(fullPath) {
						// Use this as the server directory instead
						serverDir = fullPath
					}
				}
				// Disregard all configuration files from the current directory
				// (filenames without a path separator), since we are serving a ZIP file.
				for i, filename := range serverConfigurationFilenames {
					if strings.Count(filepath.ToSlash(filename), "/") == 0 {
						// Remove the filename from the slice
						serverConfigurationFilenames = append(serverConfigurationFilenames[:i], serverConfigurationFilenames[i+1:]...)
					}
				}
				// If there are server configuration files in the extracted directory, register them
				for _, filename := range serverConfigurationFilenames {
					configFilename := filepath.Join(serverDir, filename)
					if exists(configFilename) {
						serverConfigurationFilenames = append(serverConfigurationFilenames, configFilename)
					}
				}
			default:
				singleFileMode = true
			}
		} else {
			fatalExit(errors.New("File does not exist: " + filename))
		}
	}

	// Make a few changes to the defaults if we are serving a single file
	if singleFileMode {
		debugMode = true
		serveJustHTTP = true
	}

	// If Bolt is to be used and no filename is given
	if useBolt && (boltFilename == "") {
		boltFilename = defaultBoltFilename
	}

	// Use one of the databases for the permission middleware,
	// then assign a name to dbName (used for the status output)
	dbName = ""
	if boltFilename != "" {
		// New permissions middleware, using a Bolt database
		perm, err = bolt.NewWithConf(boltFilename)
		if err != nil {
			log.Errorf("Could not use Bolt as database backend: %s", err)
		} else {
			dbName = "Bolt (" + boltFilename + ")"
		}
	}
	if dbName == "" && mariadbDSN != "" {
		// New permissions middleware, using a MariaDB/MySQL database
		perm, err = mariadb.NewWithDSN(mariadbDSN, mariadbDatabase)
		if err != nil {
			log.Errorf("Could not use MariaDB/MySQL as database backend: %s", err)
		} else {
			// The connection string may contain a password, so don't include it in the dbName
			dbName = "MariaDB/MySQL"
		}
	}
	if dbName == "" && mariadbDatabase != "" {
		// Given a database, but not a host, connect to localhost
		// New permissions middleware, using a MariaDB/MySQL database
		perm, err = mariadb.NewWithConf("test:@127.0.0.1/" + mariadbDatabase)
		if err != nil {
			log.Errorf("Could not use MariaDB/MySQL as database backend: %s", err)
		} else {
			// The connection string may contain a password, so don't include it in the dbName
			dbName = "MariaDB/MySQL"
		}
	}
	if dbName == "" {
		// New permissions middleware, using a Redis database
		if err := simpleredis.TestConnectionHost(redisAddr); err != nil {
			// Only warn when not in single file mode (too verbose)
			if !singleFileMode {
				log.Errorf("Could not use Redis as database backend: %s", err)
			}
		} else {
			perm = redis.NewWithRedisConf(redisDBindex, redisAddr)
			dbName = "Redis"
		}
	}
	if dbName == "" && boltFilename == "" {
		perm, err = bolt.NewWithConf(defaultBoltFilename)
		if err != nil {
			log.Errorf("Could not use Bolt as database backend: %s", err)
		} else {
			dbName = "Bolt (" + defaultBoltFilename + ")"
		}
	}
	if dbName == "" {
		// This may typically happen if Algernon is already running
		log.Fatalln("Could not find a usable database backend.")
	}

	// Lua LState pool
	luapool := &lStatePool{saved: make([]*lua.LState, 0, 4)}
	defer luapool.Shutdown()

	// Register HTTP handler functions
	registerHandlers(mux, serverDir, perm, luapool)

	if serverLogFile != "" {
		f, err := os.OpenFile(serverLogFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
		if err != nil {
			log.Error("Could not log to", serverLogFile)
			fatalExit(err)
		}
		log.SetFormatter(&log.JSONFormatter{})
		log.SetOutput(f)
	}

	// Read server configuration script, if present.
	// The scripts may change global variables.
	for _, filename := range serverConfigurationFilenames {
		if exists(filename) {
			if err := runConfiguration(filename, perm, luapool); err != nil {
				log.Error("Could not use configuration script: " + filename)
				fatalExit(err)
			}
		}
	}

	// Set the values that has not been set by flags nor scripts (and can be set by both)
	ranServerReadyFunction := finalConfiguration(serverHost)

	// Dividing line between the banner and output from any of the configuration scripts
	if ranServerReadyFunction {
		fmt.Println("--------------------------------------- - - · ·")
	}

	// If we are not keeping the logs, reduce the verboseness
	http2.VerboseLogs = (serverHTTP2log != "/dev/null")

	// Direct the logging from the http2 package elsewhere
	f, err := os.Open(serverHTTP2log)
	defer f.Close()
	if err != nil {
		// Could not open the serverHTTP2log filename, try using another filename
		f, err = os.OpenFile("http2.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
		defer f.Close()
		if err != nil {
			fatalExit(fmt.Errorf("Could not write to %s nor %s.", serverHTTP2log, "http2.log"))
		}
	}
	internallog.SetOutput(f)

	// Serve filesystem events in the background.
	// Used for reloading pages when the sources change.
	// Can also be used when serving a single file.
	if autoRefresh {
		refreshDuration, err = time.ParseDuration(eventRefresh)
		if err != nil {
			log.Warn(fmt.Sprintf("%s is an invalid duration. Using %s instead.", eventRefresh, defaultEventRefresh))
			// Ignore the error, since defaultEventRefresh is a constant and must be parseable
			refreshDuration, _ = time.ParseDuration(defaultEventRefresh)
		}
		if autoRefreshDir != "" {
			// Only watch the autoRefreshDir, recursively
			EventServer(eventAddr, defaultEventPath, autoRefreshDir, refreshDuration, "*")
		} else {
			// Watch everything in the server directory, recursively
			EventServer(eventAddr, defaultEventPath, serverDir, refreshDuration, "*")
		}
	}

	if interactiveMode {
		go REPL(perm, luapool)
	}

	// Timeout when closing down the server
	shutdownTimeout := 30 * time.Second

	// Decide which protocol to listen to
	switch {
	case productionMode:
		// Listen for both HTTPS+HTTP/2 and HTTP requests, on different ports
		go func() {
			log.Info("Serving HTTPS + HTTP/2 on " + serverHost + ":443")
			// Start serving. Shut down gracefully at exit.
			// Listen for HTTPS + HTTP/2 requests
			HTTPS2server := newServerConfiguration(mux, true, serverHost+":443")
			// Start serving. Shut down gracefully at exit.
			if err := graceful.ListenAndServeTLS(HTTPS2server, serverCert, serverKey, shutdownTimeout); err != nil {
				log.Error(err)
			}
		}()
		log.Info("Serving HTTP on " + serverHost + ":80")
		HTTPserver := newServerConfiguration(mux, false, serverHost+":80")
		if err := graceful.ListenAndServe(HTTPserver, shutdownTimeout); err != nil {
			// If we can't serve regular HTTP on port 80, give up
			fatalExit(err)
		}
	case serveJustHTTP2: // It's unusual to serve HTTP/2 withoutHTTPS
		log.Info("Serving HTTP/2 on " + serverAddr)
		// Listen for HTTP/2 requests
		HTTP2server := newServerConfiguration(mux, true, serverAddr)
		// Start serving. Shut down gracefully at exit.
		if err := graceful.ListenAndServe(HTTP2server, shutdownTimeout); err != nil {
			fatalExit(err)
		}
	case !(serveJustHTTP2 || serveJustHTTP):
		log.Info("Serving HTTPS + HTTP/2 on " + serverAddr)
		// Listen for HTTPS + HTTP/2 requests
		HTTPS2server := newServerConfiguration(mux, true, serverAddr)
		// Start serving. Shut down gracefully at exit.
		if err := graceful.ListenAndServeTLS(HTTPS2server, serverCert, serverKey, shutdownTimeout); err != nil {
			log.Error(err)
			// If HTTPS failed (perhaps the key + cert are missing), serve
			// plain HTTP instead, by falling through to the next case.
		} else {
			// Don't fall through
			break
		}
		// Serve regular HTTP instead
		fallthrough
	default:
		log.Info("Serving HTTP on " + serverAddr)
		HTTPserver := newServerConfiguration(mux, false, serverAddr)
		// Start serving. Shut down gracefully at exit.
		if err := graceful.ListenAndServe(HTTPserver, shutdownTimeout); err != nil {
			// If we can't serve regular HTTP, give up
			fatalExit(err)
		}
	}
}
<|endoftext|>


<commit_before>package example

import (
	"encoding/json"
	"math"

	"github.com/syou6162/go-active-learning/lib/feature"
)

type LabelType int

func (lt *LabelType) MarshalBinary() ([]byte, error) {
	return json.Marshal(lt)
}

func (lt *LabelType) UnmarshalBinary(data []byte) error {
	if err := json.Unmarshal(data, &lt); err != nil {
		return err
	}
	return nil
}

const (
	POSITIVE  LabelType = 1
	NEGATIVE  LabelType = -1
	UNLABELED LabelType = 0
)

type Example struct {
	Label         LabelType `json:"Label"`
	Fv            feature.FeatureVector
	Url           string `json:"Url"`
	FinalUrl      string `json:"FinalUrl"`
	Title         string `json:"Title"`
	Description   string `json:"Description"`
	OgDescription string `json:"OgDescription"`
	OgType        string `json:"OgType"`
	Body          string `json:"Body"`
	Score         float64
	IsNew         bool
	StatusCode    int `json:"StatusCode"`
}

type Examples []*Example

func NewExample(url string, label LabelType) *Example {
	IsNew := false
	if label == UNLABELED {
		IsNew = true
	}
	return &Example{label, []string{}, url, url, "", "", "", "", "", 0.0, IsNew, 0}
}

func (example *Example) Annotate(label LabelType) {
	example.Label = label
}

func (example *Example) IsLabeled() bool {
	return example.Label != UNLABELED
}

func (slice Examples) Len() int {
	return len(slice)
}

func (slice Examples) Less(i, j int) bool {
	return math.Abs(slice[i].Score) < math.Abs(slice[j].Score)
}

func (slice Examples) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

func GetStat(examples Examples) map[string]int {
	stat := make(map[string]int)
	for _, e := range examples {
		switch e.Label {
		case POSITIVE:
			stat["positive"]++
		case NEGATIVE:
			stat["negative"]++
		case UNLABELED:
			stat["unlabeled"]++
		}
	}
	return stat
}

func ExtractFeatures(e Example) feature.FeatureVector {
	var fv feature.FeatureVector
	fv = append(fv, "BIAS")
	fv = append(fv, feature.ExtractHostFeature(e.FinalUrl))
	fv = append(fv, feature.ExtractJpnNounFeatures(feature.ExtractPath(e.FinalUrl), "URL")...)
	fv = append(fv, feature.ExtractNounFeatures(e.Title, "TITLE")...)
	fv = append(fv, feature.ExtractNounFeatures(e.Description, "DESCRIPTION")...)
	fv = append(fv, feature.ExtractNounFeatures(e.Body, "BODY")...)
	return fv
}
<commit_msg>articleかを判定できるように<commit_after>package example

import (
	"encoding/json"
	"math"
	"strings"

	"github.com/syou6162/go-active-learning/lib/feature"
)

type LabelType int

func (lt *LabelType) MarshalBinary() ([]byte, error) {
	return json.Marshal(lt)
}

func (lt *LabelType) UnmarshalBinary(data []byte) error {
	if err := json.Unmarshal(data, &lt); err != nil {
		return err
	}
	return nil
}

const (
	POSITIVE  LabelType = 1
	NEGATIVE  LabelType = -1
	UNLABELED LabelType = 0
)

type Example struct {
	Label         LabelType `json:"Label"`
	Fv            feature.FeatureVector
	Url           string `json:"Url"`
	FinalUrl      string `json:"FinalUrl"`
	Title         string `json:"Title"`
	Description   string `json:"Description"`
	OgDescription string `json:"OgDescription"`
	OgType        string `json:"OgType"`
	Body          string `json:"Body"`
	Score         float64
	IsNew         bool
	StatusCode    int `json:"StatusCode"`
}

type Examples []*Example

func NewExample(url string, label LabelType) *Example {
	IsNew := false
	if label == UNLABELED {
		IsNew = true
	}
	return &Example{label, []string{}, url, url, "", "", "", "", "", 0.0, IsNew, 0}
}

func (example *Example) Annotate(label LabelType) {
	example.Label = label
}

func (example *Example) IsLabeled() bool {
	return example.Label != UNLABELED
}

func (example *Example) isTwitterUrl() bool {
	twitterUrl := "https://twitter.com"
	return strings.Contains(example.Url, twitterUrl) || strings.Contains(example.FinalUrl, twitterUrl)
}

func (example *Example) IsArticle() bool {
	// twitterはarticleと返ってくるが除外
	return example.OgType == "article" && !example.isTwitterUrl()
}

func (slice Examples) Len() int {
	return len(slice)
}

func (slice Examples) Less(i, j int) bool {
	return math.Abs(slice[i].Score) < math.Abs(slice[j].Score)
}

func (slice Examples) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

func GetStat(examples Examples) map[string]int {
	stat := make(map[string]int)
	for _, e := range examples {
		switch e.Label {
		case POSITIVE:
			stat["positive"]++
		case NEGATIVE:
			stat["negative"]++
		case UNLABELED:
			stat["unlabeled"]++
		}
	}
	return stat
}

func ExtractFeatures(e Example) feature.FeatureVector {
	var fv feature.FeatureVector
	fv = append(fv, "BIAS")
	fv = append(fv, feature.ExtractHostFeature(e.FinalUrl))
	fv = append(fv, feature.ExtractJpnNounFeatures(feature.ExtractPath(e.FinalUrl), "URL")...)
	fv = append(fv, feature.ExtractNounFeatures(e.Title, "TITLE")...)
	fv = append(fv, feature.ExtractNounFeatures(e.Description, "DESCRIPTION")...)
	fv = append(fv, feature.ExtractNounFeatures(e.Body, "BODY")...)
	return fv
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	flags "github.com/jessevdk/go-flags"
	"github.com/monochromegane/terminal"
	"github.com/monochromegane/the_platinum_searcher/search"
	"github.com/monochromegane/the_platinum_searcher/search/option"
	"github.com/monochromegane/the_platinum_searcher/search/grep"
	"os"
	"runtime"
	"strings"
	"time"
)

const version = "1.5.2"

var opts option.Option

func init() {
	if cpu := runtime.NumCPU(); cpu == 1 {
		runtime.GOMAXPROCS(2)
	} else {
		runtime.GOMAXPROCS(cpu)
	}
}

func main() {

	opts.Color = opts.SetEnableColor
	opts.NoColor = opts.SetDisableColor
	if runtime.GOOS == "windows" && os.Getenv("ANSICON") == "" {
		opts.EnableColor = false
	} else {
		opts.EnableColor = true
	}

	parser := flags.NewParser(&opts, flags.Default)
	parser.Name = "pt"
	parser.Usage = "[OPTIONS] PATTERN [PATH]"

	args, err := parser.Parse()
	if err != nil {
		os.Exit(1)
	}

	if opts.Version {
		fmt.Printf("%s\n", version)
		os.Exit(0)
	}

	if len(args) == 0 && opts.FilesWithRegexp == "" {
		parser.WriteHelp(os.Stdout)
		os.Exit(1)
	}

	opts.SearchStream = false
	if len(args) == 1 {
		if !terminal.IsTerminal(os.Stdin) {
			opts.SearchStream = true
			opts.NoGroup = true
		}
	}

	var root = "."
	if len(args) == 2 {
		root = strings.TrimRight(args[1], "\"")
		_, err := os.Lstat(root)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s\n", err)
			os.Exit(1)
		}
	}

	opts.Proc = runtime.NumCPU()

	if !terminal.IsTerminal(os.Stdout) {
		opts.EnableColor = false
		opts.NoGroup = true
	}

	if opts.Context > 0 {
		opts.Before = opts.Context
		opts.After = opts.Context
	}

	pattern := ""
	if len(args) > 0 {
		pattern = args[0]
	}

	start := time.Now()

	searcher := search.Searcher{root, pattern, &opts}
	err = searcher.Search()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		os.Exit(1)
	}
	if opts.Stats {
		elapsed := time.Since(start)
		fmt.Printf("%d Files Searched\n", grep.FilesSearched)
		fmt.Printf("%s Elapsed\n", elapsed)
	}
}
<commit_msg>Revert "Disabled color by default on Windows, but enabled it when ANSICON sets."<commit_after>package main

import (
	"fmt"
	flags "github.com/jessevdk/go-flags"
	"github.com/monochromegane/terminal"
	"github.com/monochromegane/the_platinum_searcher/search"
	"github.com/monochromegane/the_platinum_searcher/search/option"
	"github.com/monochromegane/the_platinum_searcher/search/grep"
	"os"
	"runtime"
	"strings"
	"time"
)

const version = "1.5.2"

var opts option.Option

func init() {
	if cpu := runtime.NumCPU(); cpu == 1 {
		runtime.GOMAXPROCS(2)
	} else {
		runtime.GOMAXPROCS(cpu)
	}
}

func main() {

	opts.Color = opts.SetEnableColor
	opts.NoColor = opts.SetDisableColor
	opts.EnableColor = true

	parser := flags.NewParser(&opts, flags.Default)
	parser.Name = "pt"
	parser.Usage = "[OPTIONS] PATTERN [PATH]"

	args, err := parser.Parse()
	if err != nil {
		os.Exit(1)
	}

	if opts.Version {
		fmt.Printf("%s\n", version)
		os.Exit(0)
	}

	if len(args) == 0 && opts.FilesWithRegexp == "" {
		parser.WriteHelp(os.Stdout)
		os.Exit(1)
	}

	opts.SearchStream = false
	if len(args) == 1 {
		if !terminal.IsTerminal(os.Stdin) {
			opts.SearchStream = true
			opts.NoGroup = true
		}
	}

	var root = "."
	if len(args) == 2 {
		root = strings.TrimRight(args[1], "\"")
		_, err := os.Lstat(root)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s\n", err)
			os.Exit(1)
		}
	}

	opts.Proc = runtime.NumCPU()

	if !terminal.IsTerminal(os.Stdout) {
		opts.EnableColor = false
		opts.NoGroup = true
	}

	if opts.Context > 0 {
		opts.Before = opts.Context
		opts.After = opts.Context
	}

	pattern := ""
	if len(args) > 0 {
		pattern = args[0]
	}

	start := time.Now()

	searcher := search.Searcher{root, pattern, &opts}
	err = searcher.Search()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		os.Exit(1)
	}
	if opts.Stats {
		elapsed := time.Since(start)
		fmt.Printf("%d Files Searched\n", grep.FilesSearched)
		fmt.Printf("%s Elapsed\n", elapsed)
	}
}
<|endoftext|>


<commit_before>package example

import (
	"encoding/json"
	"math"
	"strings"

	"github.com/syou6162/go-active-learning/lib/feature"
)

type LabelType int

func (lt *LabelType) MarshalBinary() ([]byte, error) {
	return json.Marshal(lt)
}

func (lt *LabelType) UnmarshalBinary(data []byte) error {
	if err := json.Unmarshal(data, &lt); err != nil {
		return err
	}
	return nil
}

const (
	POSITIVE  LabelType = 1
	NEGATIVE  LabelType = -1
	UNLABELED LabelType = 0
)

type Example struct {
	Label         LabelType `json:"Label"`
	Fv            feature.FeatureVector
	Url           string `json:"Url"`
	FinalUrl      string `json:"FinalUrl"`
	Title         string `json:"Title"`
	Description   string `json:"Description"`
	OgDescription string `json:"OgDescription"`
	OgType        string `json:"OgType"`
	OgImage       string `json:"OgImage"`
	Body          string `json:"Body"`
	Score         float64
	IsNew         bool
	StatusCode    int    `json:"StatusCode"`
	Favicon       string `json:"Favicon"`
}

type Examples []*Example

func NewExample(url string, label LabelType) *Example {
	IsNew := false
	if label == UNLABELED {
		IsNew = true
	}
	return &Example{label, []string{}, url, url, "", "", "", "", "", "", 0.0, IsNew, 0, ""}
}

func (example *Example) Annotate(label LabelType) {
	example.Label = label
}

func (example *Example) IsLabeled() bool {
	return example.Label != UNLABELED
}

func (example *Example) isTwitterUrl() bool {
	twitterUrl := "https://twitter.com"
	return strings.Contains(example.Url, twitterUrl) || strings.Contains(example.FinalUrl, twitterUrl)
}

func (example *Example) IsArticle() bool {
	// twitterはarticleと返ってくるが除外
	return example.OgType == "article" && !example.isTwitterUrl()
}

func (slice Examples) Len() int {
	return len(slice)
}

func (slice Examples) Less(i, j int) bool {
	return math.Abs(slice[i].Score) < math.Abs(slice[j].Score)
}

func (slice Examples) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

func GetStat(examples Examples) map[string]int {
	stat := make(map[string]int)
	for _, e := range examples {
		switch e.Label {
		case POSITIVE:
			stat["positive"]++
		case NEGATIVE:
			stat["negative"]++
		case UNLABELED:
			stat["unlabeled"]++
		}
	}
	return stat
}

func ExtractFeatures(e Example) feature.FeatureVector {
	var fv feature.FeatureVector
	fv = append(fv, "BIAS")
	fv = append(fv, feature.ExtractHostFeature(e.FinalUrl))
	fv = append(fv, feature.ExtractJpnNounFeatures(feature.ExtractPath(e.FinalUrl), "URL")...)
	fv = append(fv, feature.ExtractNounFeatures(e.Title, "TITLE")...)
	fv = append(fv, feature.ExtractNounFeatures(e.Description, "DESCRIPTION")...)
	fv = append(fv, feature.ExtractNounFeatures(e.Body, "BODY")...)
	return fv
}
<commit_msg>CreatedAtとUpdatedAtを追加<commit_after>package example

import (
	"encoding/json"
	"math"
	"strings"
	"time"

	"github.com/syou6162/go-active-learning/lib/feature"
)

type LabelType int

func (lt *LabelType) MarshalBinary() ([]byte, error) {
	return json.Marshal(lt)
}

func (lt *LabelType) UnmarshalBinary(data []byte) error {
	if err := json.Unmarshal(data, &lt); err != nil {
		return err
	}
	return nil
}

const (
	POSITIVE  LabelType = 1
	NEGATIVE  LabelType = -1
	UNLABELED LabelType = 0
)

type Example struct {
	Label         LabelType `json:"Label"`
	Fv            feature.FeatureVector
	Url           string `json:"Url"`
	FinalUrl      string `json:"FinalUrl"`
	Title         string `json:"Title"`
	Description   string `json:"Description"`
	OgDescription string `json:"OgDescription"`
	OgType        string `json:"OgType"`
	OgImage       string `json:"OgImage"`
	Body          string `json:"Body"`
	Score         float64
	IsNew         bool
	StatusCode    int       `json:"StatusCode"`
	Favicon       string    `json:"Favicon"`
	CreatedAt     time.Time `json:"CreatedAt"`
	UpdatedAt     time.Time `json:"UpdatedAt"`
}

type Examples []*Example

func NewExample(url string, label LabelType) *Example {
	IsNew := false
	if label == UNLABELED {
		IsNew = true
	}
	now := time.Now()
	return &Example{
		Label:         label,
		Fv:            []string{},
		Url:           url,
		FinalUrl:      url,
		Title:         "",
		Description:   "",
		OgDescription: "",
		OgType:        "",
		OgImage:       "",
		Body:          "",
		Score:         0.0,
		IsNew:         IsNew,
		StatusCode:    0,
		Favicon:       "",
		CreatedAt:     now,
		UpdatedAt:     now,
	}
}

func (example *Example) Annotate(label LabelType) {
	example.Label = label
}

func (example *Example) IsLabeled() bool {
	return example.Label != UNLABELED
}

func (example *Example) isTwitterUrl() bool {
	twitterUrl := "https://twitter.com"
	return strings.Contains(example.Url, twitterUrl) || strings.Contains(example.FinalUrl, twitterUrl)
}

func (example *Example) IsArticle() bool {
	// twitterはarticleと返ってくるが除外
	return example.OgType == "article" && !example.isTwitterUrl()
}

func (slice Examples) Len() int {
	return len(slice)
}

func (slice Examples) Less(i, j int) bool {
	return math.Abs(slice[i].Score) < math.Abs(slice[j].Score)
}

func (slice Examples) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

func GetStat(examples Examples) map[string]int {
	stat := make(map[string]int)
	for _, e := range examples {
		switch e.Label {
		case POSITIVE:
			stat["positive"]++
		case NEGATIVE:
			stat["negative"]++
		case UNLABELED:
			stat["unlabeled"]++
		}
	}
	return stat
}

func ExtractFeatures(e Example) feature.FeatureVector {
	var fv feature.FeatureVector
	fv = append(fv, "BIAS")
	fv = append(fv, feature.ExtractHostFeature(e.FinalUrl))
	fv = append(fv, feature.ExtractJpnNounFeatures(feature.ExtractPath(e.FinalUrl), "URL")...)
	fv = append(fv, feature.ExtractNounFeatures(e.Title, "TITLE")...)
	fv = append(fv, feature.ExtractNounFeatures(e.Description, "DESCRIPTION")...)
	fv = append(fv, feature.ExtractNounFeatures(e.Body, "BODY")...)
	return fv
}
<|endoftext|>


<commit_before>package filter

import (
	"fmt"
	"testing"

	"github.com/docker/swarm/cluster"
	"github.com/samalba/dockerclient"
	"github.com/stretchr/testify/assert"
)

func makeBinding(ip, port string) map[string][]dockerclient.PortBinding {
	return map[string][]dockerclient.PortBinding{
		fmt.Sprintf("%s/tcp", port): []dockerclient.PortBinding{
			dockerclient.PortBinding{
				HostIp:   ip,
				HostPort: port,
			},
		},
	}
}

func TestPortFilterNoConflicts(t *testing.T) {
	var (
		p     = PortFilter{}
		nodes = []*cluster.Node{
			cluster.NewNode("node-1"),
			cluster.NewNode("node-2"),
			cluster.NewNode("node-3"),
		}
		result []*cluster.Node
		err    error
	)

	// Request no ports.
	config := &dockerclient.ContainerConfig{
		HostConfig: dockerclient.HostConfig{
			PortBindings: map[string][]dockerclient.PortBinding{},
		},
	}
	// Make sure we don't filter anything out.
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.Equal(t, result, nodes)

	// Request port 80.
	config = &dockerclient.ContainerConfig{
		HostConfig: dockerclient.HostConfig{
			PortBindings: makeBinding("", "80"),
		},
	}

	// Since there are no other containers in the cluster, this shouldn't
	// filter anything either.
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.Equal(t, result, nodes)

	// Add a container taking a different (4242) port.
	container := &cluster.Container{Container: dockerclient.Container{Id: "c1"}, Info: dockerclient.ContainerInfo{HostConfig: &dockerclient.HostConfig{PortBindings: makeBinding("", "4242")}}}
	assert.NoError(t, nodes[0].AddContainer(container))

	// Since no node is using port 80, there should be no filter
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.Equal(t, result, nodes)
}

func TestPortFilterSimple(t *testing.T) {
	var (
		p     = PortFilter{}
		nodes = []*cluster.Node{
			cluster.NewNode("node-1"),
			cluster.NewNode("node-2"),
			cluster.NewNode("node-3"),
		}
		result []*cluster.Node
		err    error
	)

	// Add a container taking away port 80 to nodes[0].
	container := &cluster.Container{Container: dockerclient.Container{Id: "c1"}, Info: dockerclient.ContainerInfo{HostConfig: &dockerclient.HostConfig{PortBindings: makeBinding("", "80")}}}
	assert.NoError(t, nodes[0].AddContainer(container))

	// Request port 80.
	config := &dockerclient.ContainerConfig{
		HostConfig: dockerclient.HostConfig{
			PortBindings: makeBinding("", "80"),
		},
	}

	// nodes[0] should be excluded since port 80 is taken away.
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.NotContains(t, result, nodes[0])
}

func TestPortFilterDifferentInterfaces(t *testing.T) {
	var (
		p     = PortFilter{}
		nodes = []*cluster.Node{
			cluster.NewNode("node-1"),
			cluster.NewNode("node-2"),
			cluster.NewNode("node-3"),
		}
		result []*cluster.Node
		err    error
	)

	// Add a container taking away port 80 on every interface to nodes[0].
	container := &cluster.Container{Container: dockerclient.Container{Id: "c1"}, Info: dockerclient.ContainerInfo{HostConfig: &dockerclient.HostConfig{PortBindings: makeBinding("", "80")}}}
	assert.NoError(t, nodes[0].AddContainer(container))

	// Request port 80 for the local interface.
	config := &dockerclient.ContainerConfig{
		HostConfig: dockerclient.HostConfig{
			PortBindings: makeBinding("127.0.0.1", "80"),
		},
	}

	// nodes[0] should be excluded since port 80 is taken away for every
	// interface.
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.NotContains(t, result, nodes[0])

	// Add a container taking away port 4242 on the local interface of
	// nodes[1].
	container = &cluster.Container{Container: dockerclient.Container{Id: "c1"}, Info: dockerclient.ContainerInfo{HostConfig: &dockerclient.HostConfig{PortBindings: makeBinding("127.0.0.1", "4242")}}}
	assert.NoError(t, nodes[1].AddContainer(container))

	// Request port 4242 on the same interface.
	config = &dockerclient.ContainerConfig{
		HostConfig: dockerclient.HostConfig{
			PortBindings: makeBinding("127.0.0.1", "4242"),
		},
	}
	// nodes[1] should be excluded since port 4242 is already taken on that
	// interface.
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.NotContains(t, result, nodes[1])

	// Request port 4242 on every interface.
	config = &dockerclient.ContainerConfig{
		HostConfig: dockerclient.HostConfig{
			PortBindings: makeBinding("0.0.0.0", "4242"),
		},
	}
	// nodes[1] should still be excluded since the port is not available on the same interface.
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.NotContains(t, result, nodes[1])

	// Request port 4242 on every interface using an alternative syntax.
	config = &dockerclient.ContainerConfig{
		HostConfig: dockerclient.HostConfig{
			PortBindings: makeBinding("", "4242"),
		},
	}
	// nodes[1] should still be excluded since the port is not available on the same interface.
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.NotContains(t, result, nodes[1])

	// Finally, request port 4242 on a different interface.
	config = &dockerclient.ContainerConfig{
		HostConfig: dockerclient.HostConfig{
			PortBindings: makeBinding("192.168.1.1", "4242"),
		},
	}
	// nodes[1] should be included this time since the port is available on the
	// other interface.
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.Contains(t, result, nodes[1])
}
<commit_msg>simplify code<commit_after>package filter

import (
	"fmt"
	"testing"

	"github.com/docker/swarm/cluster"
	"github.com/samalba/dockerclient"
	"github.com/stretchr/testify/assert"
)

func makeBinding(ip, port string) map[string][]dockerclient.PortBinding {
	return map[string][]dockerclient.PortBinding{
		fmt.Sprintf("%s/tcp", port): {
			{
				HostIp:   ip,
				HostPort: port,
			},
		},
	}
}

func TestPortFilterNoConflicts(t *testing.T) {
	var (
		p     = PortFilter{}
		nodes = []*cluster.Node{
			cluster.NewNode("node-1"),
			cluster.NewNode("node-2"),
			cluster.NewNode("node-3"),
		}
		result []*cluster.Node
		err    error
	)

	// Request no ports.
	config := &dockerclient.ContainerConfig{
		HostConfig: dockerclient.HostConfig{
			PortBindings: map[string][]dockerclient.PortBinding{},
		},
	}
	// Make sure we don't filter anything out.
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.Equal(t, result, nodes)

	// Request port 80.
	config = &dockerclient.ContainerConfig{
		HostConfig: dockerclient.HostConfig{
			PortBindings: makeBinding("", "80"),
		},
	}

	// Since there are no other containers in the cluster, this shouldn't
	// filter anything either.
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.Equal(t, result, nodes)

	// Add a container taking a different (4242) port.
	container := &cluster.Container{Container: dockerclient.Container{Id: "c1"}, Info: dockerclient.ContainerInfo{HostConfig: &dockerclient.HostConfig{PortBindings: makeBinding("", "4242")}}}
	assert.NoError(t, nodes[0].AddContainer(container))

	// Since no node is using port 80, there should be no filter
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.Equal(t, result, nodes)
}

func TestPortFilterSimple(t *testing.T) {
	var (
		p     = PortFilter{}
		nodes = []*cluster.Node{
			cluster.NewNode("node-1"),
			cluster.NewNode("node-2"),
			cluster.NewNode("node-3"),
		}
		result []*cluster.Node
		err    error
	)

	// Add a container taking away port 80 to nodes[0].
	container := &cluster.Container{Container: dockerclient.Container{Id: "c1"}, Info: dockerclient.ContainerInfo{HostConfig: &dockerclient.HostConfig{PortBindings: makeBinding("", "80")}}}
	assert.NoError(t, nodes[0].AddContainer(container))

	// Request port 80.
	config := &dockerclient.ContainerConfig{
		HostConfig: dockerclient.HostConfig{
			PortBindings: makeBinding("", "80"),
		},
	}

	// nodes[0] should be excluded since port 80 is taken away.
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.NotContains(t, result, nodes[0])
}

func TestPortFilterDifferentInterfaces(t *testing.T) {
	var (
		p     = PortFilter{}
		nodes = []*cluster.Node{
			cluster.NewNode("node-1"),
			cluster.NewNode("node-2"),
			cluster.NewNode("node-3"),
		}
		result []*cluster.Node
		err    error
	)

	// Add a container taking away port 80 on every interface to nodes[0].
	container := &cluster.Container{Container: dockerclient.Container{Id: "c1"}, Info: dockerclient.ContainerInfo{HostConfig: &dockerclient.HostConfig{PortBindings: makeBinding("", "80")}}}
	assert.NoError(t, nodes[0].AddContainer(container))

	// Request port 80 for the local interface.
	config := &dockerclient.ContainerConfig{
		HostConfig: dockerclient.HostConfig{
			PortBindings: makeBinding("127.0.0.1", "80"),
		},
	}

	// nodes[0] should be excluded since port 80 is taken away for every
	// interface.
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.NotContains(t, result, nodes[0])

	// Add a container taking away port 4242 on the local interface of
	// nodes[1].
	container = &cluster.Container{Container: dockerclient.Container{Id: "c1"}, Info: dockerclient.ContainerInfo{HostConfig: &dockerclient.HostConfig{PortBindings: makeBinding("127.0.0.1", "4242")}}}
	assert.NoError(t, nodes[1].AddContainer(container))

	// Request port 4242 on the same interface.
	config = &dockerclient.ContainerConfig{
		HostConfig: dockerclient.HostConfig{
			PortBindings: makeBinding("127.0.0.1", "4242"),
		},
	}
	// nodes[1] should be excluded since port 4242 is already taken on that
	// interface.
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.NotContains(t, result, nodes[1])

	// Request port 4242 on every interface.
	config = &dockerclient.ContainerConfig{
		HostConfig: dockerclient.HostConfig{
			PortBindings: makeBinding("0.0.0.0", "4242"),
		},
	}
	// nodes[1] should still be excluded since the port is not available on the same interface.
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.NotContains(t, result, nodes[1])

	// Request port 4242 on every interface using an alternative syntax.
	config = &dockerclient.ContainerConfig{
		HostConfig: dockerclient.HostConfig{
			PortBindings: makeBinding("", "4242"),
		},
	}
	// nodes[1] should still be excluded since the port is not available on the same interface.
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.NotContains(t, result, nodes[1])

	// Finally, request port 4242 on a different interface.
	config = &dockerclient.ContainerConfig{
		HostConfig: dockerclient.HostConfig{
			PortBindings: makeBinding("192.168.1.1", "4242"),
		},
	}
	// nodes[1] should be included this time since the port is available on the
	// other interface.
	result, err = p.Filter(config, nodes)
	assert.NoError(t, err)
	assert.Contains(t, result, nodes[1])
}
<|endoftext|>


<commit_before>package rabbit

import (
	"fmt"
	"os"
	"sync"
	"time"

	log "github.com/mondough/slog"
	uuid "github.com/nu7hatch/gouuid"
	"github.com/streadway/amqp"
	"golang.org/x/net/context"
	"gopkg.in/tomb.v2"

	"github.com/mondough/terrors"
	"github.com/mondough/typhon/message"
	"github.com/mondough/typhon/transport"
)

const (
	DirectReplyQueue = "amq.rabbitmq.reply-to"

	connectTimeout  = 30 * time.Second
	chanSendTimeout = 10 * time.Second
	respondTimeout  = 10 * time.Second
)

var (
	ErrCouldntConnect   = terrors.InternalService("", "Could not connect to RabbitMQ", nil)
	ErrDeliveriesClosed = terrors.InternalService("", "Delivery channel closed", nil)
	ErrNoReplyTo        = terrors.BadRequest("", "Request does not have appropriate X-Rabbit-ReplyTo header", nil)
)

type rabbitTransport struct {
	tomb          *tomb.Tomb
	connM         sync.RWMutex                       // protects conn + connReady
	conn          *RabbitConnection                  // underlying connection
	connReady     chan struct{}                      // swapped along with conn (reconnecting)
	replyQueue    string                             // message reply queue name
	inflightReqs  map[string]chan<- message.Response // correlation id: response chan
	inflightReqsM sync.Mutex                         // protects inflightReqs
	listeners     map[string]*tomb.Tomb              // service name: tomb
	listenersM    sync.RWMutex                       // protects listeners
	runOnce       sync.Once                          // kicks off the run loop
}

// run starts the asynchronous run-loop connecting to RabbitMQ
func (t *rabbitTransport) run() {
	ctx := context.Background()
	initConn := func() *RabbitConnection {
		conn := NewRabbitConnection()
		t.connM.Lock()
		defer t.connM.Unlock()
		t.conn = conn
		select {
		case <-t.connReady:
			// Only swap connReady if it's already closed
			t.connReady = make(chan struct{})
		default:
		}
		return conn
	}
	conn := initConn()

	t.tomb.Go(func() error {
		defer func() {
			t.killListeners()
			conn.Close()
			log.Info(ctx, "[Typhon:RabbitTransport] Dead; connection closed")
		}()

	runLoop:
		for {
			log.Info(ctx, "[Typhon:RabbitTransport] Run loop connecting…")
			select {
			case <-t.tomb.Dying():
				return nil

			case <-conn.Init():
				log.Info(ctx, "[Typhon:RabbitTransport] Run loop connected")
				t.listenReplies()

				select {
				case <-t.tomb.Dying():
					// Do not loop again
					return nil
				default:
					conn.Close()
					conn = initConn()
					continue runLoop
				}

			case <-time.After(connectTimeout):
				log.Critical(ctx, "[Typhon:RabbitTransport] Run loop timed out after %v waiting to connect",
					connectTimeout)
				return ErrCouldntConnect
			}
		}
	})
}

// deliveryChan returns the name of a delivery channel for a given service
func (t *rabbitTransport) deliveryChan(serviceName string) string {
	return serviceName
}

func (t *rabbitTransport) Tomb() *tomb.Tomb {
	return t.tomb
}

func (t *rabbitTransport) connection() *RabbitConnection {
	t.runOnce.Do(t.run)
	t.connM.RLock()
	defer t.connM.RUnlock()
	return t.conn
}

func (t *rabbitTransport) Ready() <-chan struct{} {
	t.runOnce.Do(t.run)
	t.connM.RLock()
	defer t.connM.RUnlock()
	return t.connReady
}

func (t *rabbitTransport) killListeners() {
	t.listenersM.RLock()
	ts := make([]*tomb.Tomb, 0, len(t.listeners))
	for _, t := range t.listeners {
		t.Killf("Listeners killed")
		ts = append(ts, t)
	}
	t.listenersM.RUnlock()
	for _, t := range ts {
		t.Wait()
	}
}

func (t *rabbitTransport) StopListening(serviceName string) bool {
	t.listenersM.RLock()
	tm, ok := t.listeners[t.deliveryChan(serviceName)]
	if ok {
		tm.Killf("Stopped listening")
	}
	t.listenersM.RUnlock()
	if ok {
		tm.Wait()
		return true
	}
	return false
}

func (t *rabbitTransport) Listen(serviceName string, rc chan<- message.Request) error {
	ctx := context.Background()
	tm := &tomb.Tomb{}
	cn := t.deliveryChan(serviceName)
	t.listenersM.Lock()
	if _, ok := t.listeners[cn]; ok {
		t.listenersM.Unlock()
		return transport.ErrAlreadyListening
	}
	t.listeners[cn] = tm
	t.listenersM.Unlock()

	// Used to convey a connection error to the caller (we block until listening has begun)
	errChan := make(chan error)

	tm.Go(func() error {
		timeout := time.NewTimer(connectTimeout)
		defer func() {
			timeout.Stop()
			t.listenersM.Lock()
			defer t.listenersM.Unlock()
			delete(t.listeners, cn)
			close(rc)
			close(errChan)
			log.Debug(ctx, "[Typhon:RabbitTransport] Listener %s stopped", cn)
		}()

		select {
		case <-t.tomb.Dying():
			return nil
		case <-tm.Dying():
			return nil
		case <-timeout.C:
			errChan <- transport.ErrTimeout
			return nil
		case <-t.Ready():
		}

		deliveryChan, rabbitChannel, err := t.connection().Consume(cn)
		if err != nil {
			log.Warn(ctx, "[Typhon:RabbitTransport] Failed to consume from %s: %v", cn, err)
			errChan <- err
			return nil
		}
		defer rabbitChannel.Close()
		errChan <- nil
		log.Info(ctx, "[Typhon:RabbitTransport] Listening on %s…", cn)

		for {
			select {
			case <-t.tomb.Dying():
				return nil

			case <-tm.Dying():
				return nil

			case delivery, ok := <-deliveryChan:
				if !ok {
					log.Warn(ctx, "[Typhon:RabbitTransport] Delivery channel closed; stopping listener %s", cn)
					return nil
				}
				go t.handleReqDelivery(delivery, rc)
			}
		}
	})
	return <-errChan
}

func (t *rabbitTransport) logId(delivery amqp.Delivery) string {
	return fmt.Sprintf("%s[%s]", delivery.RoutingKey, delivery.CorrelationId)
}

func (t *rabbitTransport) Respond(req message.Request, rsp message.Response) error {
	headers := rsp.Headers()
	headers["Content-Encoding"] = "response"

	timeout := time.NewTimer(respondTimeout)
	defer timeout.Stop()
	select {
	case <-t.Ready():
		timeout.Stop()
	case <-t.tomb.Dying():
		return tomb.ErrDying
	case <-timeout.C:
		return transport.ErrTimeout
	}

	return t.connection().Publish("", req.Headers()["X-Rabbit-ReplyTo"], amqp.Publishing{
		CorrelationId: rsp.Id(),
		Timestamp:     time.Now().UTC(),
		Body:          rsp.Payload(),
		Headers:       headersToTable(headers),
	})
}

func (t *rabbitTransport) Send(req message.Request, _timeout time.Duration) (message.Response, error) {
	ctx := context.Background()
	id := req.Id()
	if id == "" {
		_uuid, err := uuid.NewV4()
		if err != nil {
			log.Error(ctx, "[Typhon:RabbitTransport] Failed to generate request uuid: %v", err)
			return nil, err
		}
		req.SetId(_uuid.String())
	}

	rspQueue := req.Id()
	defer func() {
		t.inflightReqsM.Lock()
		delete(t.inflightReqs, rspQueue)
		t.inflightReqsM.Unlock()
	}()
	rspChan := make(chan message.Response, 1)
	t.inflightReqsM.Lock()
	t.inflightReqs[rspQueue] = rspChan
	t.inflightReqsM.Unlock()

	timeout := time.NewTimer(_timeout)
	defer timeout.Stop()

	headers := req.Headers()
	headers["Content-Encoding"] = "request"
	headers["Service"] = req.Service()
	headers["Endpoint"] = req.Endpoint()

	select {
	case <-t.Ready():
	case <-timeout.C:
		log.Warn(ctx, "[Typhon:RabbitTransport] Timed out after %v waiting for ready", _timeout)
		return nil, transport.ErrTimeout
	}

	if err := t.connection().Publish(Exchange, req.Service(), amqp.Publishing{
		CorrelationId: req.Id(),
		Timestamp:     time.Now().UTC(),
		Body:          req.Payload(),
		ReplyTo:       t.replyQueue,
		Headers:       headersToTable(headers),
	}); err != nil {
		log.Error(ctx, "[Typhon:RabbitTransport] Failed to publish: %v", err)
		return nil, err
	}

	select {
	case rsp := <-rspChan:
		return rsp, nil
	case <-timeout.C:
		log.Warn(ctx, "[Typhon:RabbitTransport] Timed out after %v waiting for response to %s", _timeout, req.Id())
		return nil, transport.ErrTimeout
	}
}

func (t *rabbitTransport) listenReplies() error {
	ctx := context.Background()
	if err := t.connection().Channel.DeclareReplyQueue(t.replyQueue); err != nil {
		log.Critical(ctx, "[Typhon:RabbitTransport] Failed to declare reply queue %s: %v", t.replyQueue, err)
		os.Exit(1)
		return err
	}

	deliveries, err := t.connection().Channel.ConsumeQueue(t.replyQueue)
	if err != nil {
		log.Critical(ctx, "[Typhon:RabbitTransport] Failed to consume from reply queue %s: %v", t.replyQueue, err)
		os.Exit(1)
		return err
	}

	log.Debug(ctx, "[Typhon:RabbitTransport] Listening for replies on %s…", t.replyQueue)
	t.connM.RLock()
	readyC := t.connReady
	t.connM.RUnlock()
	select {
	case <-readyC:
		// Make sure not to close the channel if it's already closed
	default:
		close(readyC)
	}

	for {
		select {
		case delivery, ok := <-deliveries:
			if !ok {
				log.Warn(ctx, "[Typhon:RabbitTransport] Delivery channel %s closed", t.replyQueue)
				return ErrDeliveriesClosed
			}
			go t.handleRspDelivery(delivery)

		case <-t.tomb.Dying():
			log.Info(ctx, "[Typhon:RabbitTransport] Reply listener terminating (tomb death)")
			return tomb.ErrDying
		}
	}
}

func (t *rabbitTransport) deliveryToMessage(delivery amqp.Delivery, msg message.Message) {
	msg.SetId(delivery.CorrelationId)
	msg.SetHeaders(tableToHeaders(delivery.Headers))
	msg.SetHeader("X-Rabbit-ReplyTo", delivery.ReplyTo)
	msg.SetPayload(delivery.Body)
	if req, ok := msg.(message.Request); ok {
		switch service := delivery.Headers["Service"].(type) {
		case string:
			req.SetService(service)
		}
		switch endpoint := delivery.Headers["Endpoint"].(type) {
		case string:
			req.SetEndpoint(endpoint)
		}
	}
}

func (t *rabbitTransport) handleReqDelivery(delivery amqp.Delivery, reqChan chan<- message.Request) {
	ctx := context.Background()
	logId := t.logId(delivery)
	enc, _ := delivery.Headers["Content-Encoding"].(string)
	switch enc {
	case "request":
		req := message.NewRequest()
		t.deliveryToMessage(delivery, req)

		timeout := time.NewTimer(chanSendTimeout)
		defer timeout.Stop()
		select {
		case reqChan <- req:
		case <-timeout.C:
			log.Error(ctx, "[Typhon:RabbitTransport] Could not deliver request %s after %s: receiving channel is full",
				logId, chanSendTimeout.String())
		}

	default:
		log.Debug(ctx, "[Typhon:RabbitTransport] Cannot handle Content-Encoding \"%s\" as request for %s", enc, logId)
	}
}

func (t *rabbitTransport) handleRspDelivery(delivery amqp.Delivery) {
	ctx := context.Background()
	logId := t.logId(delivery)

	enc, _ := delivery.Headers["Content-Encoding"].(string)
	switch enc {
	case "response":
		rsp := message.NewResponse()
		t.deliveryToMessage(delivery, rsp)

		t.inflightReqsM.Lock()
		rspChan, ok := t.inflightReqs[rsp.Id()]
		delete(t.inflightReqs, rsp.Id())
		t.inflightReqsM.Unlock()
		if !ok {
			log.Warn(ctx, "[Typhon:RabbitTransport] Could not match response %s to channel", logId)
			return
		}

		timeout := time.NewTimer(chanSendTimeout)
		defer timeout.Stop()
		select {
		case rspChan <- rsp:
		case <-timeout.C:
			log.Error(ctx, "[Typhon:RabbitTransport] Could not deliver response %s after %v: receiving channel is full",
				logId, chanSendTimeout)
		}

	default:
		log.Error(ctx, "[Typhon:RabbitTransport] Cannot handle Content-Encoding \"%s\" as response for %s", enc, logId)
	}
}

func NewTransport() transport.Transport {
	return &rabbitTransport{
		tomb:         new(tomb.Tomb),
		inflightReqs: make(map[string]chan<- message.Response),
		listeners:    make(map[string]*tomb.Tomb),
		connReady:    make(chan struct{}),
		replyQueue:   DirectReplyQueue}
}
<commit_msg>Up connect timeout<commit_after>package rabbit

import (
	"fmt"
	"os"
	"sync"
	"time"

	log "github.com/mondough/slog"
	uuid "github.com/nu7hatch/gouuid"
	"github.com/streadway/amqp"
	"golang.org/x/net/context"
	"gopkg.in/tomb.v2"

	"github.com/mondough/terrors"
	"github.com/mondough/typhon/message"
	"github.com/mondough/typhon/transport"
)

const (
	DirectReplyQueue = "amq.rabbitmq.reply-to"
	connectTimeout   = 60 * time.Second
	chanSendTimeout  = 10 * time.Second
	respondTimeout   = 10 * time.Second
)

var (
	ErrCouldntConnect   = terrors.InternalService("", "Could not connect to RabbitMQ", nil)
	ErrDeliveriesClosed = terrors.InternalService("", "Delivery channel closed", nil)
	ErrNoReplyTo        = terrors.BadRequest("", "Request does not have appropriate X-Rabbit-ReplyTo header", nil)
)

type rabbitTransport struct {
	tomb          *tomb.Tomb
	connM         sync.RWMutex                       // protects conn + connReady
	conn          *RabbitConnection                  // underlying connection
	connReady     chan struct{}                      // swapped along with conn (reconnecting)
	replyQueue    string                             // message reply queue name
	inflightReqs  map[string]chan<- message.Response // correlation id: response chan
	inflightReqsM sync.Mutex                         // protects inflightReqs
	listeners     map[string]*tomb.Tomb              // service name: tomb
	listenersM    sync.RWMutex                       // protects listeners
	runOnce       sync.Once                          // kicks off the run loop
}

// run starts the asynchronous run-loop connecting to RabbitMQ
func (t *rabbitTransport) run() {
	ctx := context.Background()
	initConn := func() *RabbitConnection {
		conn := NewRabbitConnection()
		t.connM.Lock()
		defer t.connM.Unlock()
		t.conn = conn
		select {
		case <-t.connReady:
			// Only swap connReady if it's already closed
			t.connReady = make(chan struct{})
		default:
		}
		return conn
	}
	conn := initConn()

	t.tomb.Go(func() error {
		defer func() {
			t.killListeners()
			conn.Close()
			log.Info(ctx, "[Typhon:RabbitTransport] Dead; connection closed")
		}()

	runLoop:
		for {
			log.Info(ctx, "[Typhon:RabbitTransport] Run loop connecting…")
			select {
			case <-t.tomb.Dying():
				return nil

			case <-conn.Init():
				log.Info(ctx, "[Typhon:RabbitTransport] Run loop connected")
				t.listenReplies()

				select {
				case <-t.tomb.Dying():
					// Do not loop again
					return nil
				default:
					conn.Close()
					conn = initConn()
					continue runLoop
				}

			case <-time.After(connectTimeout):
				log.Critical(ctx, "[Typhon:RabbitTransport] Run loop timed out after %v waiting to connect",
					connectTimeout)
				return ErrCouldntConnect
			}
		}
	})
}

// deliveryChan returns the name of a delivery channel for a given service
func (t *rabbitTransport) deliveryChan(serviceName string) string {
	return serviceName
}

func (t *rabbitTransport) Tomb() *tomb.Tomb {
	return t.tomb
}

func (t *rabbitTransport) connection() *RabbitConnection {
	t.runOnce.Do(t.run)
	t.connM.RLock()
	defer t.connM.RUnlock()
	return t.conn
}

func (t *rabbitTransport) Ready() <-chan struct{} {
	t.runOnce.Do(t.run)
	t.connM.RLock()
	defer t.connM.RUnlock()
	return t.connReady
}

func (t *rabbitTransport) killListeners() {
	t.listenersM.RLock()
	ts := make([]*tomb.Tomb, 0, len(t.listeners))
	for _, t := range t.listeners {
		t.Killf("Listeners killed")
		ts = append(ts, t)
	}
	t.listenersM.RUnlock()
	for _, t := range ts {
		t.Wait()
	}
}

func (t *rabbitTransport) StopListening(serviceName string) bool {
	t.listenersM.RLock()
	tm, ok := t.listeners[t.deliveryChan(serviceName)]
	if ok {
		tm.Killf("Stopped listening")
	}
	t.listenersM.RUnlock()
	if ok {
		tm.Wait()
		return true
	}
	return false
}

func (t *rabbitTransport) Listen(serviceName string, rc chan<- message.Request) error {
	ctx := context.Background()
	tm := &tomb.Tomb{}
	cn := t.deliveryChan(serviceName)
	t.listenersM.Lock()
	if _, ok := t.listeners[cn]; ok {
		t.listenersM.Unlock()
		return transport.ErrAlreadyListening
	}
	t.listeners[cn] = tm
	t.listenersM.Unlock()

	// Used to convey a connection error to the caller (we block until listening has begun)
	errChan := make(chan error)

	tm.Go(func() error {
		timeout := time.NewTimer(connectTimeout)
		defer func() {
			timeout.Stop()
			t.listenersM.Lock()
			defer t.listenersM.Unlock()
			delete(t.listeners, cn)
			close(rc)
			close(errChan)
			log.Debug(ctx, "[Typhon:RabbitTransport] Listener %s stopped", cn)
		}()

		select {
		case <-t.tomb.Dying():
			return nil
		case <-tm.Dying():
			return nil
		case <-timeout.C:
			errChan <- transport.ErrTimeout
			return nil
		case <-t.Ready():
		}

		deliveryChan, rabbitChannel, err := t.connection().Consume(cn)
		if err != nil {
			log.Warn(ctx, "[Typhon:RabbitTransport] Failed to consume from %s: %v", cn, err)
			errChan <- err
			return nil
		}
		defer rabbitChannel.Close()
		errChan <- nil
		log.Info(ctx, "[Typhon:RabbitTransport] Listening on %s…", cn)

		for {
			select {
			case <-t.tomb.Dying():
				return nil

			case <-tm.Dying():
				return nil

			case delivery, ok := <-deliveryChan:
				if !ok {
					log.Warn(ctx, "[Typhon:RabbitTransport] Delivery channel closed; stopping listener %s", cn)
					return nil
				}
				go t.handleReqDelivery(delivery, rc)
			}
		}
	})
	return <-errChan
}

func (t *rabbitTransport) logId(delivery amqp.Delivery) string {
	return fmt.Sprintf("%s[%s]", delivery.RoutingKey, delivery.CorrelationId)
}

func (t *rabbitTransport) Respond(req message.Request, rsp message.Response) error {
	headers := rsp.Headers()
	headers["Content-Encoding"] = "response"

	timeout := time.NewTimer(respondTimeout)
	defer timeout.Stop()
	select {
	case <-t.Ready():
		timeout.Stop()
	case <-t.tomb.Dying():
		return tomb.ErrDying
	case <-timeout.C:
		return transport.ErrTimeout
	}

	return t.connection().Publish("", req.Headers()["X-Rabbit-ReplyTo"], amqp.Publishing{
		CorrelationId: rsp.Id(),
		Timestamp:     time.Now().UTC(),
		Body:          rsp.Payload(),
		Headers:       headersToTable(headers),
	})
}

func (t *rabbitTransport) Send(req message.Request, _timeout time.Duration) (message.Response, error) {
	ctx := context.Background()
	id := req.Id()
	if id == "" {
		_uuid, err := uuid.NewV4()
		if err != nil {
			log.Error(ctx, "[Typhon:RabbitTransport] Failed to generate request uuid: %v", err)
			return nil, err
		}
		req.SetId(_uuid.String())
	}

	rspQueue := req.Id()
	defer func() {
		t.inflightReqsM.Lock()
		delete(t.inflightReqs, rspQueue)
		t.inflightReqsM.Unlock()
	}()
	rspChan := make(chan message.Response, 1)
	t.inflightReqsM.Lock()
	t.inflightReqs[rspQueue] = rspChan
	t.inflightReqsM.Unlock()

	timeout := time.NewTimer(_timeout)
	defer timeout.Stop()

	headers := req.Headers()
	headers["Content-Encoding"] = "request"
	headers["Service"] = req.Service()
	headers["Endpoint"] = req.Endpoint()

	select {
	case <-t.Ready():
	case <-timeout.C:
		log.Warn(ctx, "[Typhon:RabbitTransport] Timed out after %v waiting for ready", _timeout)
		return nil, transport.ErrTimeout
	}

	if err := t.connection().Publish(Exchange, req.Service(), amqp.Publishing{
		CorrelationId: req.Id(),
		Timestamp:     time.Now().UTC(),
		Body:          req.Payload(),
		ReplyTo:       t.replyQueue,
		Headers:       headersToTable(headers),
	}); err != nil {
		log.Error(ctx, "[Typhon:RabbitTransport] Failed to publish: %v", err)
		return nil, err
	}

	select {
	case rsp := <-rspChan:
		return rsp, nil
	case <-timeout.C:
		log.Warn(ctx, "[Typhon:RabbitTransport] Timed out after %v waiting for response to %s", _timeout, req.Id())
		return nil, transport.ErrTimeout
	}
}

func (t *rabbitTransport) listenReplies() error {
	ctx := context.Background()
	if err := t.connection().Channel.DeclareReplyQueue(t.replyQueue); err != nil {
		log.Critical(ctx, "[Typhon:RabbitTransport] Failed to declare reply queue %s: %v", t.replyQueue, err)
		os.Exit(1)
		return err
	}

	deliveries, err := t.connection().Channel.ConsumeQueue(t.replyQueue)
	if err != nil {
		log.Critical(ctx, "[Typhon:RabbitTransport] Failed to consume from reply queue %s: %v", t.replyQueue, err)
		os.Exit(1)
		return err
	}

	log.Debug(ctx, "[Typhon:RabbitTransport] Listening for replies on %s…", t.replyQueue)
	t.connM.RLock()
	readyC := t.connReady
	t.connM.RUnlock()
	select {
	case <-readyC:
		// Make sure not to close the channel if it's already closed
	default:
		close(readyC)
	}

	for {
		select {
		case delivery, ok := <-deliveries:
			if !ok {
				log.Warn(ctx, "[Typhon:RabbitTransport] Delivery channel %s closed", t.replyQueue)
				return ErrDeliveriesClosed
			}
			go t.handleRspDelivery(delivery)

		case <-t.tomb.Dying():
			log.Info(ctx, "[Typhon:RabbitTransport] Reply listener terminating (tomb death)")
			return tomb.ErrDying
		}
	}
}

func (t *rabbitTransport) deliveryToMessage(delivery amqp.Delivery, msg message.Message) {
	msg.SetId(delivery.CorrelationId)
	msg.SetHeaders(tableToHeaders(delivery.Headers))
	msg.SetHeader("X-Rabbit-ReplyTo", delivery.ReplyTo)
	msg.SetPayload(delivery.Body)
	if req, ok := msg.(message.Request); ok {
		switch service := delivery.Headers["Service"].(type) {
		case string:
			req.SetService(service)
		}
		switch endpoint := delivery.Headers["Endpoint"].(type) {
		case string:
			req.SetEndpoint(endpoint)
		}
	}
}

func (t *rabbitTransport) handleReqDelivery(delivery amqp.Delivery, reqChan chan<- message.Request) {
	ctx := context.Background()
	logId := t.logId(delivery)
	enc, _ := delivery.Headers["Content-Encoding"].(string)
	switch enc {
	case "request":
		req := message.NewRequest()
		t.deliveryToMessage(delivery, req)

		timeout := time.NewTimer(chanSendTimeout)
		defer timeout.Stop()
		select {
		case reqChan <- req:
		case <-timeout.C:
			log.Error(ctx, "[Typhon:RabbitTransport] Could not deliver request %s after %s: receiving channel is full",
				logId, chanSendTimeout.String())
		}

	default:
		log.Debug(ctx, "[Typhon:RabbitTransport] Cannot handle Content-Encoding \"%s\" as request for %s", enc, logId)
	}
}

func (t *rabbitTransport) handleRspDelivery(delivery amqp.Delivery) {
	ctx := context.Background()
	logId := t.logId(delivery)

	enc, _ := delivery.Headers["Content-Encoding"].(string)
	switch enc {
	case "response":
		rsp := message.NewResponse()
		t.deliveryToMessage(delivery, rsp)

		t.inflightReqsM.Lock()
		rspChan, ok := t.inflightReqs[rsp.Id()]
		delete(t.inflightReqs, rsp.Id())
		t.inflightReqsM.Unlock()
		if !ok {
			log.Warn(ctx, "[Typhon:RabbitTransport] Could not match response %s to channel", logId)
			return
		}

		timeout := time.NewTimer(chanSendTimeout)
		defer timeout.Stop()
		select {
		case rspChan <- rsp:
		case <-timeout.C:
			log.Error(ctx, "[Typhon:RabbitTransport] Could not deliver response %s after %v: receiving channel is full",
				logId, chanSendTimeout)
		}

	default:
		log.Error(ctx, "[Typhon:RabbitTransport] Cannot handle Content-Encoding \"%s\" as response for %s", enc, logId)
	}
}

func NewTransport() transport.Transport {
	return &rabbitTransport{
		tomb:         new(tomb.Tomb),
		inflightReqs: make(map[string]chan<- message.Response),
		listeners:    make(map[string]*tomb.Tomb),
		connReady:    make(chan struct{}),
		replyQueue:   DirectReplyQueue}
}
<|endoftext|>


<commit_before>package search

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"

	iopt "github.com/algolia/algoliasearch-client-go/algolia/internal/opt"
	"github.com/algolia/algoliasearch-client-go/algolia/opt"
	"github.com/algolia/algoliasearch-client-go/algolia/transport"
)

// GenerateSecuredAPIKey generates a public API key intended to restrict access
// to certain records. This new key is built upon the existing key named
// `apiKey` and the following options:
//
//   - opt.Filters (string of filters to apply automatically on search queries)
//   - opt.Referers (string slice of allowed referers)
//   - opt.RestrictIndices (comma-separated string list of the indices to restrict)
//   - opt.RestrictSources (string of the allowed IPv4 network)
//   - opt.UserToken (string identifier generally used to rate-limit users per IP)
//   - opt.ValidUntil (timestamp of the expiration date)
//
// More details here:
// https://www.algolia.com/doc/api-reference/api-methods/generate-secured-api-key/?language=python#parameters
func GenerateSecuredAPIKey(apiKey string, opts ...interface{}) (string, error) {
	h := hmac.New(sha256.New, []byte(apiKey))

	message := transport.URLEncode(newSecuredAPIKeyParams(opts...))
	_, err := h.Write([]byte(message))
	if err != nil {
		return "", err
	}

	checksum := hex.EncodeToString(h.Sum(nil))
	key := base64.StdEncoding.EncodeToString([]byte(checksum + message))

	return key, nil
}

type securedAPIKeyParams struct {
	Referers        *opt.ReferersOption        `json:"referers,omitempty"`
	RestrictIndices *opt.RestrictIndicesOption `json:"restrictIndices,omitempty"`
	RestrictSources *opt.RestrictSourcesOption `json:"restrictSources,omitempty"`
	UserToken       *opt.UserTokenOption       `json:"userToken,omitempty"`
	ValidUntil      int64                      `json:"validUntil,omitempty"`
	QueryParams
}

func newSecuredAPIKeyParams(opts ...interface{}) securedAPIKeyParams {
	var validUntilInt int64
	if validUntil := iopt.ExtractValidUntil(opts...); validUntil != nil {
		validUntilInt = validUntil.Get().Unix()
	}
	return securedAPIKeyParams{
		Referers:        iopt.ExtractReferers(opts...),
		RestrictIndices: iopt.ExtractRestrictIndices(opts...),
		RestrictSources: iopt.ExtractRestrictSources(opts...),
		UserToken:       iopt.ExtractUserToken(opts...),
		ValidUntil:      validUntilInt,
		QueryParams:     newQueryParams(opts...),
	}
}
<commit_msg>fix: make filters usable for secured API keys<commit_after>package search

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"

	iopt "github.com/algolia/algoliasearch-client-go/algolia/internal/opt"
	"github.com/algolia/algoliasearch-client-go/algolia/opt"
	"github.com/algolia/algoliasearch-client-go/algolia/transport"
)

// GenerateSecuredAPIKey generates a public API key intended to restrict access
// to certain records. This new key is built upon the existing key named
// `apiKey` and the following options:
//
//   - opt.Filters (string of filters to apply automatically on search queries)
//   - opt.Referers (string slice of allowed referers)
//   - opt.RestrictIndices (comma-separated string list of the indices to restrict)
//   - opt.RestrictSources (string of the allowed IPv4 network)
//   - opt.UserToken (string identifier generally used to rate-limit users per IP)
//   - opt.ValidUntil (timestamp of the expiration date)
//
// More details here:
// https://www.algolia.com/doc/api-reference/api-methods/generate-secured-api-key/?language=python#parameters
func GenerateSecuredAPIKey(apiKey string, opts ...interface{}) (string, error) {
	h := hmac.New(sha256.New, []byte(apiKey))

	message := transport.URLEncode(newSecuredAPIKeyParams(opts...))
	_, err := h.Write([]byte(message))
	if err != nil {
		return "", err
	}

	checksum := hex.EncodeToString(h.Sum(nil))
	key := base64.StdEncoding.EncodeToString([]byte(checksum + message))

	return key, nil
}

type securedAPIKeyParams struct {
	Filters         *opt.FiltersOption         `json:"filters,omitempty"`
	Referers        *opt.ReferersOption        `json:"referers,omitempty"`
	RestrictIndices *opt.RestrictIndicesOption `json:"restrictIndices,omitempty"`
	RestrictSources *opt.RestrictSourcesOption `json:"restrictSources,omitempty"`
	UserToken       *opt.UserTokenOption       `json:"userToken,omitempty"`
	ValidUntil      int64                      `json:"validUntil,omitempty"`
	QueryParams
}

func newSecuredAPIKeyParams(opts ...interface{}) securedAPIKeyParams {
	var validUntilInt int64
	if validUntil := iopt.ExtractValidUntil(opts...); validUntil != nil {
		validUntilInt = validUntil.Get().Unix()
	}
	return securedAPIKeyParams{
		Filters:         iopt.ExtractFilters(opts...),
		Referers:        iopt.ExtractReferers(opts...),
		RestrictIndices: iopt.ExtractRestrictIndices(opts...),
		RestrictSources: iopt.ExtractRestrictSources(opts...),
		UserToken:       iopt.ExtractUserToken(opts...),
		ValidUntil:      validUntilInt,
		QueryParams:     newQueryParams(opts...),
	}
}
<|endoftext|>