<commit_before>package main

import (
	"errors"
	"fmt"
	"log"
	"os"

	"github.com/stripe/sequins/blocks"
	"github.com/stripe/sequins/sequencefile"
)

var errFilesChanged = errors.New("the list of remote files changed while building")

// build prepares the version, blocking until all local partitions are ready,
// then returns it. If onlyFromManifest is true, it will only load data on local
// disk from a manifest, and fail otherwise.
func (vs *version) build(files []string) error {
	if vs.blockStore != nil {
		return nil
	}

	if len(files) == 0 {
		log.Println("Version", vs.name, "of", vs.db, "has no data. Loading it anyway.")
		return nil
	}

	var local map[int]bool
	if vs.sequins.peers != nil {
		local = vs.partitions.pickLocalPartitions()
		if len(local) == 0 {
			log.Println("All valid partitions for", vs.db, "version", vs.name,
				"are already spoken for. Consider increasing the replication level.")
			return nil
		}
	}

	log.Println("Loading", vs.db, "version", vs.name, "from",
		vs.sequins.backend.DisplayPath(vs.db, vs.name), "into local directory", vs.path)

	blockStore, err := vs.createStore(files, local)
	if err != nil {
		return err
	}

	// Verify that the list of files stayed the same.
	newFiles, err := vs.sequins.backend.ListFiles(vs.db, vs.name)
	if err != nil {
		return err
	} else {
		if vs.compareFileSets(files, newFiles) {
			return errFilesChanged
		}
	}

	vs.blockStoreLock.Lock()
	defer vs.blockStoreLock.Unlock()
	vs.blockStore = blockStore
	if vs.partitions != nil {
		vs.partitions.updateLocalPartitions(local)
	}

	return nil
}

func (vs *version) compareFileSets(oldFiles, newFiles []string) bool {
	setOld := make(map[string]bool, len(oldFiles))
	setNew := make(map[string]bool, len(newFiles))
	different := false

	if len(oldFiles) != len(newFiles) {
		log.Printf("Number of files under %s changed (%d vs %d)",
			vs.sequins.backend.DisplayPath(vs.db, vs.name), len(oldFiles), len(newFiles))
		different = true
	}

	for _, f := range oldFiles {
		setOld[f] = true
	}

	for _, f := range newFiles {
		setNew[f] = true
		if !setOld[f] {
			log.Println("New file:", vs.sequins.backend.DisplayPath(vs.db, vs.name, f))
			different = true
		}
	}

	for _, f := range oldFiles {
		if !setNew[f] {
			log.Println("Missing file:", vs.sequins.backend.DisplayPath(vs.db, vs.name, f))
			different = true
		}
	}

	return different
}

// TODO: parallelize files

func (vs *version) createStore(files []string, partitions map[int]bool) (*blocks.BlockStore, error) {
	if _, err := os.Stat(vs.path); err == nil {
		log.Println("Clearing local directory", vs.path)
	}

	os.RemoveAll(vs.path)
	err := os.MkdirAll(vs.path, 0755|os.ModeDir)
	if err != nil {
		return nil, fmt.Errorf("error creating local storage directory: %s", err)
	}

	blockStore := blocks.New(vs.path, len(files), partitions,
		vs.sequins.config.Storage.Compression, vs.sequins.config.Storage.BlockSize)
	for _, file := range files {
		err := vs.addFile(blockStore, file)
		if err != nil {
			return nil, err
		}
	}

	blockStore.Save()
	return blockStore, nil
}

func (vs *version) addFile(bs *blocks.BlockStore, file string) error {
	disp := vs.sequins.backend.DisplayPath(vs.db, vs.name, file)
	log.Println("Reading records from", disp)

	stream, err := vs.sequins.backend.Open(vs.db, vs.name, file)
	if err != nil {
		return fmt.Errorf("reading %s: %s", disp, err)
	}

	sf := sequencefile.New(stream)
	err = sf.ReadHeader()
	if err != nil {
		return fmt.Errorf("reading header from %s: %s", disp, err)
	}

	err = bs.AddFile(sf, vs.sequins.config.ThrottleLoads.Duration)
	if err == blocks.ErrWrongPartition {
		log.Println("Skipping", disp, "because it contains no relevant partitions")
	} else if err != nil {
		return fmt.Errorf("reading %s: %s", disp, err)
	}

	return nil
}
<commit_msg>If files have changed, remove the new blockstore<commit_after>package main

import (
	"errors"
	"fmt"
	"log"
	"os"

	"github.com/stripe/sequins/blocks"
	"github.com/stripe/sequins/sequencefile"
)

var errFilesChanged = errors.New("the list of remote files changed while building")

// build prepares the version, blocking until all local partitions are ready,
// then returns it. If onlyFromManifest is true, it will only load data on local
// disk from a manifest, and fail otherwise.
func (vs *version) build(files []string) error {
	if vs.blockStore != nil {
		return nil
	}

	if len(files) == 0 {
		log.Println("Version", vs.name, "of", vs.db, "has no data. Loading it anyway.")
		return nil
	}

	var local map[int]bool
	if vs.sequins.peers != nil {
		local = vs.partitions.pickLocalPartitions()
		if len(local) == 0 {
			log.Println("All valid partitions for", vs.db, "version", vs.name,
				"are already spoken for. Consider increasing the replication level.")
			return nil
		}
	}

	log.Println("Loading", vs.db, "version", vs.name, "from",
		vs.sequins.backend.DisplayPath(vs.db, vs.name), "into local directory", vs.path)

	blockStore, err := vs.createStore(files, local)
	if err != nil {
		return err
	}

	// Verify that the list of files stayed the same. If files do not match,
	// discard the new blockstore in order to prevent the manifest from updating.
	newFiles, err := vs.sequins.backend.ListFiles(vs.db, vs.name)
	if err != nil {
		return err
	} else {
		if vs.compareFileSets(files, newFiles) {
			blockStore.Close()
			blockStore.Delete()
			return errFilesChanged
		}
	}

	vs.blockStoreLock.Lock()
	defer vs.blockStoreLock.Unlock()
	vs.blockStore = blockStore
	if vs.partitions != nil {
		vs.partitions.updateLocalPartitions(local)
	}

	return nil
}

func (vs *version) compareFileSets(oldFiles, newFiles []string) bool {
	setOld := make(map[string]bool, len(oldFiles))
	setNew := make(map[string]bool, len(newFiles))
	different := false

	if len(oldFiles) != len(newFiles) {
		log.Printf("Number of files under %s changed (%d vs %d)",
			vs.sequins.backend.DisplayPath(vs.db, vs.name), len(oldFiles), len(newFiles))
		different = true
	}

	for _, f := range oldFiles {
		setOld[f] = true
	}

	for _, f := range newFiles {
		setNew[f] = true
		if !setOld[f] {
			log.Println("New file:", vs.sequins.backend.DisplayPath(vs.db, vs.name, f))
			different = true
		}
	}

	for _, f := range oldFiles {
		if !setNew[f] {
			log.Println("Missing file:", vs.sequins.backend.DisplayPath(vs.db, vs.name, f))
			different = true
		}
	}

	return different
}

// TODO: parallelize files

func (vs *version) createStore(files []string, partitions map[int]bool) (*blocks.BlockStore, error) {
	if _, err := os.Stat(vs.path); err == nil {
		log.Println("Clearing local directory", vs.path)
	}

	os.RemoveAll(vs.path)
	err := os.MkdirAll(vs.path, 0755|os.ModeDir)
	if err != nil {
		return nil, fmt.Errorf("error creating local storage directory: %s", err)
	}

	blockStore := blocks.New(vs.path, len(files), partitions,
		vs.sequins.config.Storage.Compression, vs.sequins.config.Storage.BlockSize)
	for _, file := range files {
		err := vs.addFile(blockStore, file)
		if err != nil {
			return nil, err
		}
	}

	blockStore.Save()
	return blockStore, nil
}

func (vs *version) addFile(bs *blocks.BlockStore, file string) error {
	disp := vs.sequins.backend.DisplayPath(vs.db, vs.name, file)
	log.Println("Reading records from", disp)

	stream, err := vs.sequins.backend.Open(vs.db, vs.name, file)
	if err != nil {
		return fmt.Errorf("reading %s: %s", disp, err)
	}

	sf := sequencefile.New(stream)
	err = sf.ReadHeader()
	if err != nil {
		return fmt.Errorf("reading header from %s: %s", disp, err)
	}

	err = bs.AddFile(sf, vs.sequins.config.ThrottleLoads.Duration)
	if err == blocks.ErrWrongPartition {
		log.Println("Skipping", disp, "because it contains no relevant partitions")
	} else if err != nil {
		return fmt.Errorf("reading %s: %s", disp, err)
	}

	return nil
}
<|endoftext|>


<commit_before>/*
 * Build management stuff. This is used to build modules.
 */

package main

import (
	"fmt"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"
)

// Build is the result of a build.
type Build struct {
	Module   ModuleConfig
	Previous bool
	Success  bool
	Skipped  bool
	Output   string
}

// Builds is a list of builds of the configuration.
type Builds []Build

func (build Build) String() string {
	if build.Skipped {
		return "SKIPPED"
	}
	if build.Success {
		return "SUCCESS"
	}
	return "FAILURE"
}

// Tell if should send an email:
// - config: email configuration.
// Return a bool that tells if should send email.
func (build Build) SendEmail(config EmailConfig) bool {
	if build.Skipped {
		return false
	}
	if config.Once {
		return build.Success != build.Previous
	}
	return !build.Success || (build.Success && config.Success)
}

// BuildModule is called to build a module, that is:
// - get the repository clone.
// - run command to build the module.
// If build command returns 0 (as of Unix standard), the build is a success, else
// this is a failure.
func BuildModule(module ModuleConfig, directory string) Build {
	moduleDir := path.Join(directory, module.Name)
	// go in build directory
	currentDir, err := os.Getwd()
	defer os.Chdir(currentDir)
	err = os.Chdir(directory)
	if err != nil {
		return Build{
			Module:  module,
			Success: false,
			Skipped: false,
			Output:  err.Error(),
		}
	}
	// delete module directory if it already exists
	if _, err := os.Stat(moduleDir); err == nil {
		os.RemoveAll(moduleDir)
	}
	// get the module
	output, err := CloneRepo(module)
	if err != nil {
		return Build{
			Module:  module,
			Success: false,
			Skipped: false,
			Output:  string(output),
		}
	}
	defer os.RemoveAll(moduleDir)
	os.Chdir(moduleDir)
	// run the build command
	cmd := exec.Command("bash", "-c", module.Command)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return Build{
			Module:  module,
			Success: false,
			Skipped: false,
			Output:  strings.TrimSpace(string(out)),
		}
	}
	return Build{
		Module:  module,
		Success: true,
		Skipped: false,
		Output:  string(out),
	}
}

// BuildModules builds the list of modules in the configuration (in the exact same
// order).
func BuildModules(config Config) Builds {
	builds := make(Builds, len(config.Modules))
	modulesInfo := LoadModulesInfo(config.Status)
	for _, module := range config.Modules {
		fmt.Printf("Building '%s'... ", module.Name)
		start := time.Now()
		repoHash := GetRepoHash(module)
		var build Build
		if modulesInfo[module.Name].RepoHash == "" {
			build = BuildModule(module, config.Directory)
			build.Previous = true
		} else if modulesInfo[module.Name].RepoHash != repoHash {
			build = BuildModule(module, config.Directory)
			build.Previous = modulesInfo[module.Name].BuildOK
		} else {
			build = Build{
				Module:   module,
				Previous: modulesInfo[module.Name].BuildOK,
				Success:  modulesInfo[module.Name].BuildOK,
				Skipped:  true,
				Output:   "",
			}
		}
		duration := time.Since(start)
		fmt.Println(build.String())
		SendEmail(build, start, duration, config.Email)
		if build.Success {
			modulesInfo[module.Name] = ModuleInfo{
				RepoHash: repoHash,
				BuildOK:  true,
			}
		} else {
			modulesInfo[module.Name] = ModuleInfo{
				RepoHash: repoHash,
				BuildOK:  false,
			}
		}
	}
	SaveModulesInfo(modulesInfo, config.Status)
	return builds
}
<commit_msg>Fixed golint warning<commit_after>/*
 * Build management stuff. This is used to build modules.
 */

package main

import (
	"fmt"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"
)

// Build is the result of a build.
type Build struct {
	Module   ModuleConfig
	Previous bool
	Success  bool
	Skipped  bool
	Output   string
}

// Builds is a list of builds of the configuration.
type Builds []Build

func (build Build) String() string {
	if build.Skipped {
		return "SKIPPED"
	}
	if build.Success {
		return "SUCCESS"
	}
	return "FAILURE"
}

// SendMail tells if should send an email:
// - config: email configuration.
// Return a bool that tells if should send email.
func (build Build) SendEmail(config EmailConfig) bool {
	if build.Skipped {
		return false
	}
	if config.Once {
		return build.Success != build.Previous
	}
	return !build.Success || (build.Success && config.Success)
}

// BuildModule is called to build a module, that is:
// - get the repository clone.
// - run command to build the module.
// If build command returns 0 (as of Unix standard), the build is a success, else
// this is a failure.
func BuildModule(module ModuleConfig, directory string) Build {
	moduleDir := path.Join(directory, module.Name)
	// go in build directory
	currentDir, err := os.Getwd()
	defer os.Chdir(currentDir)
	err = os.Chdir(directory)
	if err != nil {
		return Build{
			Module:  module,
			Success: false,
			Skipped: false,
			Output:  err.Error(),
		}
	}
	// delete module directory if it already exists
	if _, err := os.Stat(moduleDir); err == nil {
		os.RemoveAll(moduleDir)
	}
	// get the module
	output, err := CloneRepo(module)
	if err != nil {
		return Build{
			Module:  module,
			Success: false,
			Skipped: false,
			Output:  string(output),
		}
	}
	defer os.RemoveAll(moduleDir)
	os.Chdir(moduleDir)
	// run the build command
	cmd := exec.Command("bash", "-c", module.Command)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return Build{
			Module:  module,
			Success: false,
			Skipped: false,
			Output:  strings.TrimSpace(string(out)),
		}
	}
	return Build{
		Module:  module,
		Success: true,
		Skipped: false,
		Output:  string(out),
	}
}

// BuildModules builds the list of modules in the configuration (in the exact same
// order).
func BuildModules(config Config) Builds {
	builds := make(Builds, len(config.Modules))
	modulesInfo := LoadModulesInfo(config.Status)
	for _, module := range config.Modules {
		fmt.Printf("Building '%s'... ", module.Name)
		start := time.Now()
		repoHash := GetRepoHash(module)
		var build Build
		if modulesInfo[module.Name].RepoHash == "" {
			build = BuildModule(module, config.Directory)
			build.Previous = true
		} else if modulesInfo[module.Name].RepoHash != repoHash {
			build = BuildModule(module, config.Directory)
			build.Previous = modulesInfo[module.Name].BuildOK
		} else {
			build = Build{
				Module:   module,
				Previous: modulesInfo[module.Name].BuildOK,
				Success:  modulesInfo[module.Name].BuildOK,
				Skipped:  true,
				Output:   "",
			}
		}
		duration := time.Since(start)
		fmt.Println(build.String())
		SendEmail(build, start, duration, config.Email)
		if build.Success {
			modulesInfo[module.Name] = ModuleInfo{
				RepoHash: repoHash,
				BuildOK:  true,
			}
		} else {
			modulesInfo[module.Name] = ModuleInfo{
				RepoHash: repoHash,
				BuildOK:  false,
			}
		}
	}
	SaveModulesInfo(modulesInfo, config.Status)
	return builds
}
<|endoftext|>


<commit_before>// Package v1 implements the first version of the Ripple API.
package v1

import (
	"database/sql"
	"strconv"
	"strings"
	"time"

	"git.zxq.co/ripple/rippleapi/common"
)

type userData struct {
	common.ResponseBase
	ID             int       `json:"id"`
	Username       string    `json:"username"`
	UsernameAKA    string    `json:"username_aka"`
	RegisteredOn   time.Time `json:"registered_on"`
	Rank           int       `json:"rank"`
	LatestActivity time.Time `json:"latest_activity"`
	Country        string    `json:"country"`
}

// UsersGET is the API handler for GET /users
func UsersGET(md common.MethodData) common.CodeMessager {
	shouldRet, whereClause, param := whereClauseUser(md, "users")
	if shouldRet != nil {
		return *shouldRet
	}

	query := `
SELECT users.id, users.username, register_datetime, rank,
	latest_activity, users_stats.username_aka,
	users_stats.country, users_stats.show_country
FROM users
LEFT JOIN users_stats
ON users.id=users_stats.id
WHERE ` + whereClause + ` AND users.allowed='1'
LIMIT 1`
	return userPuts(md, md.DB.QueryRow(query, param))
}

func userPuts(md common.MethodData, row *sql.Row) common.CodeMessager {
	var err error
	var user userData

	var (
		registeredOn   int64
		latestActivity int64
		showCountry    bool
	)
	err = row.Scan(&user.ID, &user.Username, &registeredOn, &user.Rank, &latestActivity, &user.UsernameAKA, &user.Country, &showCountry)
	switch {
	case err == sql.ErrNoRows:
		return common.SimpleResponse(404, "No such user was found!")
	case err != nil:
		md.Err(err)
		return Err500
	}

	user.RegisteredOn = time.Unix(registeredOn, 0)
	user.LatestActivity = time.Unix(latestActivity, 0)

	user.Country = genCountry(md, user.ID, showCountry, user.Country)

	user.Code = 200
	return user
}

func badgesToArray(badges string) []int {
	var end []int
	badgesSl := strings.Split(badges, ",")
	for _, badge := range badgesSl {
		if badge != "" && badge != "0" {
			// We are ignoring errors because who really gives a shit if something's gone wrong on our end in this
			// particular thing, we can just silently ignore this.
			nb, err := strconv.Atoi(badge)
			if err == nil && nb != 0 {
				end = append(end, nb)
			}
		}
	}
	return end
}

func genCountry(md common.MethodData, uid int, showCountry bool, country string) string {
	// If the user wants to stay anonymous, don't show their country.
	// This can be overriden if we have the ReadConfidential privilege and the user we are accessing is the token owner.
	if showCountry || (md.User.Privileges.HasPrivilegeReadConfidential() && uid == md.ID()) {
		return country
	}
	return "XX"
}

// UserSelfGET is a shortcut for /users/id/self. (/users/self)
func UserSelfGET(md common.MethodData) common.CodeMessager {
	md.C.Request.URL.RawQuery = "id=self&" + md.C.Request.URL.RawQuery
	return UsersGET(md)
}

type whatIDResponse struct {
	common.ResponseBase
	ID int `json:"id"`
}

// UserWhatsTheIDGET is an API request that only returns an user's ID.
func UserWhatsTheIDGET(md common.MethodData) common.CodeMessager {
	var (
		r       whatIDResponse
		allowed int
	)
	err := md.DB.QueryRow("SELECT id, allowed FROM users WHERE username = ? LIMIT 1", md.C.Query("name")).Scan(&r.ID, &allowed)
	if err != nil || (allowed != 1 && !md.User.Privileges.HasPrivilegeViewUserAdvanced()) {
		return common.SimpleResponse(404, "That user could not be found!")
	}
	r.Code = 200
	return r
}

type modeData struct {
	RankedScore           uint64  `json:"ranked_score"`
	TotalScore            uint64  `json:"total_score"`
	PlayCount             int     `json:"playcount"`
	ReplaysWatched        int     `json:"replays_watched"`
	TotalHits             int     `json:"total_hits"`
	Level                 float64 `json:"level"`
	Accuracy              float64 `json:"accuracy"`
	PP                    int     `json:"pp"`
	GlobalLeaderboardRank int     `json:"global_leaderboard_rank"`
}
type userFullResponse struct {
	common.ResponseBase
	userData
	STD           modeData `json:"std"`
	Taiko         modeData `json:"taiko"`
	CTB           modeData `json:"ctb"`
	Mania         modeData `json:"mania"`
	PlayStyle     int      `json:"play_style"`
	FavouriteMode int      `json:"favourite_mode"`
	Badges        []int    `json:"badges"`
}

// UserFullGET gets all of an user's information, with one exception: their userpage.
func UserFullGET(md common.MethodData) common.CodeMessager {
	shouldRet, whereClause, param := whereClauseUser(md, "users")
	if shouldRet != nil {
		return *shouldRet
	}

	// Hellest query I've ever done.
	query := `
SELECT
	users.id, users.username, users.register_datetime, users.rank, users.latest_activity,

	users_stats.username_aka, users_stats.badges_shown, users_stats.country, users_stats.show_country,
	users_stats.play_style, users_stats.favourite_mode,

	users_stats.ranked_score_std, users_stats.total_score_std, users_stats.playcount_std,
	users_stats.replays_watched_std, users_stats.total_hits_std, users_stats.level_std,
	users_stats.avg_accuracy_std, users_stats.pp_std, leaderboard_std.position as std_position,

	users_stats.ranked_score_taiko, users_stats.total_score_taiko, users_stats.playcount_taiko,
	users_stats.replays_watched_taiko, users_stats.total_hits_taiko, users_stats.level_taiko,
	users_stats.avg_accuracy_taiko, users_stats.pp_taiko, leaderboard_taiko.position as taiko_position,

	users_stats.ranked_score_ctb, users_stats.total_score_ctb, users_stats.playcount_ctb,
	users_stats.replays_watched_ctb, users_stats.total_hits_ctb, users_stats.level_ctb,
	users_stats.avg_accuracy_ctb, users_stats.pp_ctb, leaderboard_ctb.position as ctb_position,

	users_stats.ranked_score_mania, users_stats.total_score_mania, users_stats.playcount_mania,
	users_stats.replays_watched_mania, users_stats.total_hits_mania, users_stats.level_mania,
	users_stats.avg_accuracy_mania, users_stats.pp_mania, leaderboard_mania.position as mania_position

FROM users
LEFT JOIN users_stats
ON users.id=users_stats.id
LEFT JOIN leaderboard_std
ON users.id=leaderboard_std.user
LEFT JOIN leaderboard_taiko
ON users.id=leaderboard_taiko.user
LEFT JOIN leaderboard_ctb
ON users.id=leaderboard_ctb.user
LEFT JOIN leaderboard_mania
ON users.id=leaderboard_mania.user
WHERE ` + whereClause + ` AND users.allowed = '1'
LIMIT 1
`
	// Fuck.
	r := userFullResponse{}
	var (
		badges         string
		country        string
		showCountry    bool
		registeredOn   int64
		latestActivity int64
	)
	err := md.DB.QueryRow(query, param).Scan(
		&r.ID, &r.Username, &registeredOn, &r.Rank, &latestActivity,

		&r.UsernameAKA, &badges, &country, &showCountry,
		&r.PlayStyle, &r.FavouriteMode,

		&r.STD.RankedScore, &r.STD.TotalScore, &r.STD.PlayCount,
		&r.STD.ReplaysWatched, &r.STD.TotalHits, &r.STD.Level,
		&r.STD.Accuracy, &r.STD.PP, &r.STD.GlobalLeaderboardRank,

		&r.Taiko.RankedScore, &r.Taiko.TotalScore, &r.Taiko.PlayCount,
		&r.Taiko.ReplaysWatched, &r.Taiko.TotalHits, &r.Taiko.Level,
		&r.Taiko.Accuracy, &r.Taiko.PP, &r.Taiko.GlobalLeaderboardRank,

		&r.CTB.RankedScore, &r.CTB.TotalScore, &r.CTB.PlayCount,
		&r.CTB.ReplaysWatched, &r.CTB.TotalHits, &r.CTB.Level,
		&r.CTB.Accuracy, &r.CTB.PP, &r.CTB.GlobalLeaderboardRank,

		&r.Mania.RankedScore, &r.Mania.TotalScore, &r.Mania.PlayCount,
		&r.Mania.ReplaysWatched, &r.Mania.TotalHits, &r.Mania.Level,
		&r.Mania.Accuracy, &r.Mania.PP, &r.Mania.GlobalLeaderboardRank,
	)
	switch {
	case err == sql.ErrNoRows:
		return common.SimpleResponse(404, "That user could not be found!")
	case err != nil:
		md.Err(err)
		return Err500
	}

	r.Country = genCountry(md, r.ID, showCountry, country)
	r.Badges = badgesToArray(badges)

	r.RegisteredOn = time.Unix(registeredOn, 0)
	r.LatestActivity = time.Unix(latestActivity, 0)

	r.Code = 200
	return r
}

type userpageResponse struct {
	common.ResponseBase
	Userpage string `json:"userpage"`
}

// UserUserpageGET gets an user's userpage, as in the customisable thing.
func UserUserpageGET(md common.MethodData) common.CodeMessager {
	shouldRet, whereClause, param := whereClauseUser(md, "users_stats")
	if shouldRet != nil {
		return *shouldRet
	}
	var r userpageResponse
	err := md.DB.QueryRow("SELECT userpage_content FROM users_stats WHERE "+whereClause+" LIMIT 1", param).Scan(&r.Userpage)
	switch {
	case err == sql.ErrNoRows:
		return common.SimpleResponse(404, "No such user!")
	case err != nil:
		md.Err(err)
		return Err500
	}
	r.Code = 200
	return r
}

func whereClauseUser(md common.MethodData, tableName string) (*common.CodeMessager, string, interface{}) {
	switch {
	case md.C.Query("id") == "self":
		return nil, tableName + ".id = ?", md.ID()
	case md.C.Query("id") != "":
		id, err := strconv.Atoi(md.C.Query("id"))
		if err != nil {
			a := common.SimpleResponse(400, "please pass a valid user ID")
			return &a, "", nil
		}
		return nil, tableName + ".id = ?", id
	case md.C.Query("name") != "":
		return nil, tableName + ".username = ?", md.C.Query("name")
	}
	a := common.SimpleResponse(400, "you need to pass either querystring parameters name or id")
	return &a, "", nil
}

type userLookupResponse struct {
	common.ResponseBase
	Users []lookupUser `json:"users"`
}
type lookupUser struct {
	ID       int    `json:"id"`
	Username string `json:"username"`
}

// UserLookupGET does a quick lookup of users beginning with the passed
// querystring value name.
func UserLookupGET(md common.MethodData) common.CodeMessager {
	name := strings.NewReplacer(
		"%", "\\%",
		"_", "\\_",
		"\\", "\\\\",
	).Replace(md.C.Query("name"))
	if name == "" {
		return common.SimpleResponse(400, "please provide an username to start searching")
	}
	name += "%"
	rows, err := md.DB.Query("SELECT users.id, users.username FROM users WHERE username LIKE ? AND allowed = '1' LIMIT 25", name)
	if err != nil {
		md.Err(err)
		return Err500
	}

	var r userLookupResponse

	for rows.Next() {
		var l lookupUser
		err := rows.Scan(&l.ID, &l.Username)
		if err != nil {
			continue // can't be bothered to handle properly
		}
		r.Users = append(r.Users, l)
	}

	r.Code = 200
	return r
}
<commit_msg>Use precise level in /api/v1/users/full<commit_after>// Package v1 implements the first version of the Ripple API.
package v1

import (
	"database/sql"
	"strconv"
	"strings"
	"time"

	"git.zxq.co/ripple/ocl"
	"git.zxq.co/ripple/rippleapi/common"
)

type userData struct {
	common.ResponseBase
	ID             int       `json:"id"`
	Username       string    `json:"username"`
	UsernameAKA    string    `json:"username_aka"`
	RegisteredOn   time.Time `json:"registered_on"`
	Rank           int       `json:"rank"`
	LatestActivity time.Time `json:"latest_activity"`
	Country        string    `json:"country"`
}

// UsersGET is the API handler for GET /users
func UsersGET(md common.MethodData) common.CodeMessager {
	shouldRet, whereClause, param := whereClauseUser(md, "users")
	if shouldRet != nil {
		return *shouldRet
	}

	query := `
SELECT users.id, users.username, register_datetime, rank,
	latest_activity, users_stats.username_aka,
	users_stats.country, users_stats.show_country
FROM users
LEFT JOIN users_stats
ON users.id=users_stats.id
WHERE ` + whereClause + ` AND users.allowed='1'
LIMIT 1`
	return userPuts(md, md.DB.QueryRow(query, param))
}

func userPuts(md common.MethodData, row *sql.Row) common.CodeMessager {
	var err error
	var user userData

	var (
		registeredOn   int64
		latestActivity int64
		showCountry    bool
	)
	err = row.Scan(&user.ID, &user.Username, &registeredOn, &user.Rank, &latestActivity, &user.UsernameAKA, &user.Country, &showCountry)
	switch {
	case err == sql.ErrNoRows:
		return common.SimpleResponse(404, "No such user was found!")
	case err != nil:
		md.Err(err)
		return Err500
	}

	user.RegisteredOn = time.Unix(registeredOn, 0)
	user.LatestActivity = time.Unix(latestActivity, 0)

	user.Country = genCountry(md, user.ID, showCountry, user.Country)

	user.Code = 200
	return user
}

func badgesToArray(badges string) []int {
	var end []int
	badgesSl := strings.Split(badges, ",")
	for _, badge := range badgesSl {
		if badge != "" && badge != "0" {
			// We are ignoring errors because who really gives a shit if something's gone wrong on our end in this
			// particular thing, we can just silently ignore this.
			nb, err := strconv.Atoi(badge)
			if err == nil && nb != 0 {
				end = append(end, nb)
			}
		}
	}
	return end
}

func genCountry(md common.MethodData, uid int, showCountry bool, country string) string {
	// If the user wants to stay anonymous, don't show their country.
	// This can be overriden if we have the ReadConfidential privilege and the user we are accessing is the token owner.
	if showCountry || (md.User.Privileges.HasPrivilegeReadConfidential() && uid == md.ID()) {
		return country
	}
	return "XX"
}

// UserSelfGET is a shortcut for /users/id/self. (/users/self)
func UserSelfGET(md common.MethodData) common.CodeMessager {
	md.C.Request.URL.RawQuery = "id=self&" + md.C.Request.URL.RawQuery
	return UsersGET(md)
}

type whatIDResponse struct {
	common.ResponseBase
	ID int `json:"id"`
}

// UserWhatsTheIDGET is an API request that only returns an user's ID.
func UserWhatsTheIDGET(md common.MethodData) common.CodeMessager {
	var (
		r       whatIDResponse
		allowed int
	)
	err := md.DB.QueryRow("SELECT id, allowed FROM users WHERE username = ? LIMIT 1", md.C.Query("name")).Scan(&r.ID, &allowed)
	if err != nil || (allowed != 1 && !md.User.Privileges.HasPrivilegeViewUserAdvanced()) {
		return common.SimpleResponse(404, "That user could not be found!")
	}
	r.Code = 200
	return r
}

type modeData struct {
	RankedScore           uint64  `json:"ranked_score"`
	TotalScore            uint64  `json:"total_score"`
	PlayCount             int     `json:"playcount"`
	ReplaysWatched        int     `json:"replays_watched"`
	TotalHits             int     `json:"total_hits"`
	Level                 float64 `json:"level"`
	Accuracy              float64 `json:"accuracy"`
	PP                    int     `json:"pp"`
	GlobalLeaderboardRank int     `json:"global_leaderboard_rank"`
}
type userFullResponse struct {
	common.ResponseBase
	userData
	STD           modeData `json:"std"`
	Taiko         modeData `json:"taiko"`
	CTB           modeData `json:"ctb"`
	Mania         modeData `json:"mania"`
	PlayStyle     int      `json:"play_style"`
	FavouriteMode int      `json:"favourite_mode"`
	Badges        []int    `json:"badges"`
}

// UserFullGET gets all of an user's information, with one exception: their userpage.
func UserFullGET(md common.MethodData) common.CodeMessager {
	shouldRet, whereClause, param := whereClauseUser(md, "users")
	if shouldRet != nil {
		return *shouldRet
	}

	// Hellest query I've ever done.
	query := `
SELECT
	users.id, users.username, users.register_datetime, users.rank, users.latest_activity,

	users_stats.username_aka, users_stats.badges_shown, users_stats.country, users_stats.show_country,
	users_stats.play_style, users_stats.favourite_mode,

	users_stats.ranked_score_std, users_stats.total_score_std, users_stats.playcount_std,
	users_stats.replays_watched_std, users_stats.total_hits_std,
	users_stats.avg_accuracy_std, users_stats.pp_std, leaderboard_std.position as std_position,

	users_stats.ranked_score_taiko, users_stats.total_score_taiko, users_stats.playcount_taiko,
	users_stats.replays_watched_taiko, users_stats.total_hits_taiko,
	users_stats.avg_accuracy_taiko, users_stats.pp_taiko, leaderboard_taiko.position as taiko_position,

	users_stats.ranked_score_ctb, users_stats.total_score_ctb, users_stats.playcount_ctb,
	users_stats.replays_watched_ctb, users_stats.total_hits_ctb,
	users_stats.avg_accuracy_ctb, users_stats.pp_ctb, leaderboard_ctb.position as ctb_position,

	users_stats.ranked_score_mania, users_stats.total_score_mania, users_stats.playcount_mania,
	users_stats.replays_watched_mania, users_stats.total_hits_mania,
	users_stats.avg_accuracy_mania, users_stats.pp_mania, leaderboard_mania.position as mania_position

FROM users
LEFT JOIN users_stats
ON users.id=users_stats.id
LEFT JOIN leaderboard_std
ON users.id=leaderboard_std.user
LEFT JOIN leaderboard_taiko
ON users.id=leaderboard_taiko.user
LEFT JOIN leaderboard_ctb
ON users.id=leaderboard_ctb.user
LEFT JOIN leaderboard_mania
ON users.id=leaderboard_mania.user
WHERE ` + whereClause + ` AND users.allowed = '1'
LIMIT 1
`
	// Fuck.
	r := userFullResponse{}
	var (
		badges         string
		country        string
		showCountry    bool
		registeredOn   int64
		latestActivity int64
	)
	err := md.DB.QueryRow(query, param).Scan(
		&r.ID, &r.Username, &registeredOn, &r.Rank, &latestActivity,

		&r.UsernameAKA, &badges, &country, &showCountry,
		&r.PlayStyle, &r.FavouriteMode,

		&r.STD.RankedScore, &r.STD.TotalScore, &r.STD.PlayCount,
		&r.STD.ReplaysWatched, &r.STD.TotalHits,
		&r.STD.Accuracy, &r.STD.PP, &r.STD.GlobalLeaderboardRank,

		&r.Taiko.RankedScore, &r.Taiko.TotalScore, &r.Taiko.PlayCount,
		&r.Taiko.ReplaysWatched, &r.Taiko.TotalHits,
		&r.Taiko.Accuracy, &r.Taiko.PP, &r.Taiko.GlobalLeaderboardRank,

		&r.CTB.RankedScore, &r.CTB.TotalScore, &r.CTB.PlayCount,
		&r.CTB.ReplaysWatched, &r.CTB.TotalHits,
		&r.CTB.Accuracy, &r.CTB.PP, &r.CTB.GlobalLeaderboardRank,

		&r.Mania.RankedScore, &r.Mania.TotalScore, &r.Mania.PlayCount,
		&r.Mania.ReplaysWatched, &r.Mania.TotalHits,
		&r.Mania.Accuracy, &r.Mania.PP, &r.Mania.GlobalLeaderboardRank,
	)
	switch {
	case err == sql.ErrNoRows:
		return common.SimpleResponse(404, "That user could not be found!")
	case err != nil:
		md.Err(err)
		return Err500
	}

	r.Country = genCountry(md, r.ID, showCountry, country)
	r.Badges = badgesToArray(badges)

	r.RegisteredOn = time.Unix(registeredOn, 0)
	r.LatestActivity = time.Unix(latestActivity, 0)

	for _, m := range []*modeData{&r.STD, &r.Taiko, &r.CTB, &r.Mania} {
		m.Level = ocl.GetLevelPrecise(int64(m.TotalScore))
	}

	r.Code = 200
	return r
}

type userpageResponse struct {
	common.ResponseBase
	Userpage string `json:"userpage"`
}

// UserUserpageGET gets an user's userpage, as in the customisable thing.
func UserUserpageGET(md common.MethodData) common.CodeMessager {
	shouldRet, whereClause, param := whereClauseUser(md, "users_stats")
	if shouldRet != nil {
		return *shouldRet
	}
	var r userpageResponse
	err := md.DB.QueryRow("SELECT userpage_content FROM users_stats WHERE "+whereClause+" LIMIT 1", param).Scan(&r.Userpage)
	switch {
	case err == sql.ErrNoRows:
		return common.SimpleResponse(404, "No such user!")
	case err != nil:
		md.Err(err)
		return Err500
	}
	r.Code = 200
	return r
}

func whereClauseUser(md common.MethodData, tableName string) (*common.CodeMessager, string, interface{}) {
	switch {
	case md.C.Query("id") == "self":
		return nil, tableName + ".id = ?", md.ID()
	case md.C.Query("id") != "":
		id, err := strconv.Atoi(md.C.Query("id"))
		if err != nil {
			a := common.SimpleResponse(400, "please pass a valid user ID")
			return &a, "", nil
		}
		return nil, tableName + ".id = ?", id
	case md.C.Query("name") != "":
		return nil, tableName + ".username = ?", md.C.Query("name")
	}
	a := common.SimpleResponse(400, "you need to pass either querystring parameters name or id")
	return &a, "", nil
}

type userLookupResponse struct {
	common.ResponseBase
	Users []lookupUser `json:"users"`
}
type lookupUser struct {
	ID       int    `json:"id"`
	Username string `json:"username"`
}

// UserLookupGET does a quick lookup of users beginning with the passed
// querystring value name.
func UserLookupGET(md common.MethodData) common.CodeMessager {
	name := strings.NewReplacer(
		"%", "\\%",
		"_", "\\_",
		"\\", "\\\\",
	).Replace(md.C.Query("name"))
	if name == "" {
		return common.SimpleResponse(400, "please provide an username to start searching")
	}
	name += "%"
	rows, err := md.DB.Query("SELECT users.id, users.username FROM users WHERE username LIKE ? AND allowed = '1' LIMIT 25", name)
	if err != nil {
		md.Err(err)
		return Err500
	}

	var r userLookupResponse

	for rows.Next() {
		var l lookupUser
		err := rows.Scan(&l.ID, &l.Username)
		if err != nil {
			continue // can't be bothered to handle properly
		}
		r.Users = append(r.Users, l)
	}

	r.Code = 200
	return r
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/barnacs/compy/proxy"
	tc "github.com/barnacs/compy/transcoder"
	"log"
	"os"
	"os/signal"
	"sync/atomic"
)

var (
	host  = flag.String("host", ":9999", "<host:port>")
	cert  = flag.String("cert", "", "proxy cert path")
	key   = flag.String("key", "", "proxy cert key path")
	ca    = flag.String("ca", "", "CA path")
	caKey = flag.String("cakey", "", "CA key path")

	jpeg   = flag.Int("jpeg", 50, "jpeg quality (1-100, 0 to disable)")
	gif    = flag.Bool("gif", true, "transcode gifs into static images")
	png    = flag.Bool("png", true, "transcode png")
	minify = flag.Bool("minify", false, "minify css/html/js - WARNING: tends to break the web")
)

func main() {
	flag.Parse()

	p := proxy.New()

	if (*ca == "") != (*caKey == "") {
		log.Fatalln("must specify both CA certificate and key")
	}

	if (*cert == "") != (*key == "") {
		log.Fatalln("must specify both certificate and key")
	}

	if *ca != "" {
		if err := p.EnableMitm(*ca, *caKey); err != nil {
			fmt.Println("not using mitm:", err)
		}
	}

	if *jpeg != 0 {
		p.AddTranscoder("image/jpeg", tc.NewJpeg(*jpeg))
	}
	if *gif {
		p.AddTranscoder("image/gif", &tc.Gif{})
	}
	if *png {
		p.AddTranscoder("image/png", &tc.Png{})
	}

	var ttc proxy.Transcoder
	if *minify {
		ttc = &tc.Gzip{tc.NewMinifier(), false}
	} else {
		ttc = &tc.Gzip{&tc.Identity{}, true}
	}

	p.AddTranscoder("text/css", ttc)
	p.AddTranscoder("text/html", ttc)
	p.AddTranscoder("text/javascript", ttc)
	p.AddTranscoder("application/javascript", ttc)
	p.AddTranscoder("application/x-javascript", ttc)

	c := make(chan os.Signal, 2)
	signal.Notify(c, os.Interrupt)
	go func() {
		for _ = range c {
			read := atomic.LoadUint64(&p.ReadCount)
			written := atomic.LoadUint64(&p.WriteCount)
			log.Printf("compy exiting, total transcoded: %d -> %d (%3.1f%%)",
				read, written, float64(written)/float64(read)*100)
			os.Exit(0)
		}
	}()

	log.Printf("compy listening on %s", *host)

	var err error
	if *cert != "" {
		err = p.StartTLS(*host, *cert, *key)
	} else {
		err = p.Start(*host)
	}
	log.Fatalln(err)
}
<commit_msg>Simplify for loop<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/barnacs/compy/proxy"
	tc "github.com/barnacs/compy/transcoder"
	"log"
	"os"
	"os/signal"
	"sync/atomic"
)

var (
	host  = flag.String("host", ":9999", "<host:port>")
	cert  = flag.String("cert", "", "proxy cert path")
	key   = flag.String("key", "", "proxy cert key path")
	ca    = flag.String("ca", "", "CA path")
	caKey = flag.String("cakey", "", "CA key path")

	jpeg   = flag.Int("jpeg", 50, "jpeg quality (1-100, 0 to disable)")
	gif    = flag.Bool("gif", true, "transcode gifs into static images")
	png    = flag.Bool("png", true, "transcode png")
	minify = flag.Bool("minify", false, "minify css/html/js - WARNING: tends to break the web")
)

func main() {
	flag.Parse()

	p := proxy.New()

	if (*ca == "") != (*caKey == "") {
		log.Fatalln("must specify both CA certificate and key")
	}

	if (*cert == "") != (*key == "") {
		log.Fatalln("must specify both certificate and key")
	}

	if *ca != "" {
		if err := p.EnableMitm(*ca, *caKey); err != nil {
			fmt.Println("not using mitm:", err)
		}
	}

	if *jpeg != 0 {
		p.AddTranscoder("image/jpeg", tc.NewJpeg(*jpeg))
	}
	if *gif {
		p.AddTranscoder("image/gif", &tc.Gif{})
	}
	if *png {
		p.AddTranscoder("image/png", &tc.Png{})
	}

	var ttc proxy.Transcoder
	if *minify {
		ttc = &tc.Gzip{tc.NewMinifier(), false}
	} else {
		ttc = &tc.Gzip{&tc.Identity{}, true}
	}

	p.AddTranscoder("text/css", ttc)
	p.AddTranscoder("text/html", ttc)
	p.AddTranscoder("text/javascript", ttc)
	p.AddTranscoder("application/javascript", ttc)
	p.AddTranscoder("application/x-javascript", ttc)

	c := make(chan os.Signal, 2)
	signal.Notify(c, os.Interrupt)
	go func() {
		for range c {
			read := atomic.LoadUint64(&p.ReadCount)
			written := atomic.LoadUint64(&p.WriteCount)
			log.Printf("compy exiting, total transcoded: %d -> %d (%3.1f%%)",
				read, written, float64(written)/float64(read)*100)
			os.Exit(0)
		}
	}()

	log.Printf("compy listening on %s", *host)

	var err error
	if *cert != "" {
		err = p.StartTLS(*host, *cert, *key)
	} else {
		err = p.Start(*host)
	}
	log.Fatalln(err)
}
<|endoftext|>


<commit_before>package emitter

import (
	"fmt"
	"net"
	"net/http"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/metric"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

type PrometheusEmitter struct {
	buildsStarted     prometheus.Counter
	buildsFinished    prometheus.Counter
	buildsSucceeded   prometheus.Counter
	buildsErrored     prometheus.Counter
	buildsFailed      prometheus.Counter
	buildsAborted     prometheus.Counter
	buildsFinishedVec *prometheus.CounterVec
	buildDurationsVec *prometheus.HistogramVec

	workerContainers *prometheus.GaugeVec
	workerVolumes    *prometheus.GaugeVec

	httpRequestsDuration *prometheus.HistogramVec

	schedulingFullDuration        *prometheus.GaugeVec
	schedulingLoadingDuration     *prometheus.GaugeVec
	schedulingJobDurationTotalVec *prometheus.CounterVec
	schedulingJobTotalVec         *prometheus.CounterVec

	dbQueriesTotal prometheus.Counter
	dbConnections  *prometheus.GaugeVec
}

type PrometheusConfig struct {
	BindIP   string `long:"prometheus-bind-ip" description:"IP to listen on to expose Prometheus metrics."`
	BindPort string `long:"prometheus-bind-port" description:"Port to listen on to expose Prometheus metrics."`
}

func init() {
	metric.RegisterEmitter(&PrometheusConfig{})
}

func (config *PrometheusConfig) Description() string { return "Prometheus" }
func (config *PrometheusConfig) IsConfigured() bool {
	return config.BindPort != "" && config.BindIP != ""
}
func (config *PrometheusConfig) bind() string {
	return fmt.Sprintf("%s:%s", config.BindIP, config.BindPort)
}

func (config *PrometheusConfig) NewEmitter() (metric.Emitter, error) {
	// build metrics
	buildsStarted := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "started_total",
		Help:      "Total number of Concourse builds started.",
	})
	prometheus.MustRegister(buildsStarted)

	buildsFinished := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "finished_total",
		Help:      "Total number of Concourse builds finished.",
	})
	prometheus.MustRegister(buildsFinished)

	buildsSucceeded := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "succeeded_total",
		Help:      "Total number of Concourse builds succeeded.",
	})
	prometheus.MustRegister(buildsSucceeded)

	buildsErrored := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "errored_total",
		Help:      "Total number of Concourse builds errored.",
	})
	prometheus.MustRegister(buildsErrored)

	buildsFailed := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "failed_total",
		Help:      "Total number of Concourse builds failed.",
	})
	prometheus.MustRegister(buildsFailed)

	buildsAborted := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "aborted_total",
		Help:      "Total number of Concourse builds aborted.",
	})
	prometheus.MustRegister(buildsAborted)

	buildsFinishedVec := prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "concourse",
			Subsystem: "builds",
			Name:      "finished",
			Help:      "Count of builds finished across various dimensions.",
		},
		[]string{"team", "pipeline", "status"},
	)
	prometheus.MustRegister(buildsFinishedVec)
	buildDurationsVec := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "concourse",
			Subsystem: "builds",
			Name:      "duration_seconds",
			Help:      "Build time in seconds",
			Buckets:   []float64{1, 60, 180, 300, 600, 900, 1200, 1800, 2700, 3600, 7200, 18000, 36000},
		},
		[]string{"team", "pipeline"},
	)
	prometheus.MustRegister(buildDurationsVec)

	// worker metrics
	workerContainers := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "workers",
			Name:      "containers",
			Help:      "Number of containers per worker",
		},
		[]string{"worker"},
	)
	prometheus.MustRegister(workerContainers)

	workerVolumes := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "workers",
			Name:      "volumes",
			Help:      "Number of volumes per worker",
		},
		[]string{"worker"},
	)
	prometheus.MustRegister(workerVolumes)

	// http metrics
	httpRequestsDuration := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "concourse",
			Subsystem: "http_responses",
			Name:      "duration_seconds",
			Help:      "Response time in seconds",
		},
		[]string{"method", "route"},
	)
	prometheus.MustRegister(httpRequestsDuration)

	// scheduling metrics
	schedulingFullDuration := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "scheduling",
			Name:      "full_duration_seconds",
			Help:      "Last time taken to schedule an entire pipeline.",
		},
		[]string{"pipeline"},
	)
	prometheus.MustRegister(schedulingFullDuration)

	schedulingLoadingDuration := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "scheduling",
			Name:      "loading_duration_seconds",
			Help:      "Last time taken to load version information from the database for a pipeline.",
		},
		[]string{"pipeline"},
	)
	prometheus.MustRegister(schedulingLoadingDuration)

	schedulingJobDurationTotalVec := prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "concourse",
			Subsystem: "scheduling",
			Name:      "job_duration_seconds_total",
			Help:      "Total amount of time spend calculating the set of valid input versions for a job.",
		},
		[]string{"pipeline", "job"},
	)
	schedulingJobTotalVec := prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "concourse",
			Subsystem: "scheduling",
			Name:      "job_total",
			Help:      "Number of times the job was scheduled",
		},
		[]string{"pipeline", "job"},
	)

	prometheus.MustRegister(schedulingJobDurationTotalVec)
	prometheus.MustRegister(schedulingJobTotalVec)

	dbQueriesTotal := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "db",
		Name:      "queries_total",
		Help:      "Total number of database Concourse database queries",
	})
	prometheus.MustRegister(dbQueriesTotal)

	dbConnections := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "db",
			Name:      "connections",
			Help:      "Current number of concourse database connections",
		},
		[]string{"dbname"},
	)
	prometheus.MustRegister(dbConnections)

	listener, err := net.Listen("tcp", config.bind())
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, promhttp.Handler())

	return &PrometheusEmitter{
		buildsStarted:     buildsStarted,
		buildsFinished:    buildsFinished,
		buildsFinishedVec: buildsFinishedVec,
		buildDurationsVec: buildDurationsVec,
		buildsSucceeded:   buildsSucceeded,
		buildsErrored:     buildsErrored,
		buildsFailed:      buildsFailed,
		buildsAborted:     buildsAborted,

		workerContainers: workerContainers,
		workerVolumes:    workerVolumes,

		httpRequestsDuration: httpRequestsDuration,

		schedulingFullDuration:        schedulingFullDuration,
		schedulingLoadingDuration:     schedulingLoadingDuration,
		schedulingJobDurationTotalVec: schedulingJobDurationTotalVec,
		schedulingJobTotalVec:         schedulingJobTotalVec,

		dbQueriesTotal: dbQueriesTotal,
		dbConnections:  dbConnections,
	}, nil
}

// Emit processes incoming metrics.
// In order to provide idiomatic Prometheus metrics, we'll have to convert the various
// Event types (differentiated by the less-than-ideal string Name field) into different
// Prometheus metrics.
func (emitter *PrometheusEmitter) Emit(logger lager.Logger, event metric.Event) {
	switch event.Name {
	case "build started":
		emitter.buildsStarted.Inc()
	case "build finished":
		emitter.buildFinishedMetrics(logger, event)
	case "worker containers":
		emitter.workerContainersMetric(logger, event)
	case "worker volumes":
		emitter.workerVolumesMetric(logger, event)
	case "http response time":
		emitter.httpResponseTimeMetrics(logger, event)
	case "scheduling: full duration (ms)":
		emitter.schedulingMetrics(logger, event)
	case "scheduling: loading versions duration (ms)":
		emitter.schedulingMetrics(logger, event)
	case "scheduling: job duration (ms)":
		emitter.schedulingMetrics(logger, event)
	case "database queries":
		emitter.databaseMetrics(logger, event)
	case "database connections":
		emitter.databaseMetrics(logger, event)
	default:
		// unless we have a specific metric, we do nothing
	}
}

func (emitter *PrometheusEmitter) buildFinishedMetrics(logger lager.Logger, event metric.Event) {
	// concourse_builds_finished_total
	emitter.buildsFinished.Inc()

	// concourse_builds_finished
	team, exists := event.Attributes["team_name"]
	if !exists {
		logger.Error("failed-to-find-team-name-in-event", fmt.Errorf("expected team_name to exist in event.Attributes"))
	}

	pipeline, exists := event.Attributes["pipeline"]
	if !exists {
		logger.Error("failed-to-find-pipeline-in-event", fmt.Errorf("expected pipeline to exist in event.Attributes"))
	}

	buildStatus, exists := event.Attributes["build_status"]
	if !exists {
		logger.Error("failed-to-find-build_status-in-event", fmt.Errorf("expected build_status to exist in event.Attributes"))
	}
	emitter.buildsFinishedVec.WithLabelValues(team, pipeline, buildStatus).Inc()

	// concourse_builds_(aborted|succeeded|failed|errored)_total
	switch buildStatus {
	case string(db.BuildStatusAborted):
		// concourse_builds_aborted_total
		emitter.buildsAborted.Inc()
	case string(db.BuildStatusSucceeded):
		// concourse_builds_succeeded_total
		emitter.buildsSucceeded.Inc()
	case string(db.BuildStatusFailed):
		// concourse_builds_failed_total
		emitter.buildsFailed.Inc()
	case string(db.BuildStatusErrored):
		// concourse_builds_errored_total
		emitter.buildsErrored.Inc()
	}

	// concourse_builds_duration_seconds
	duration, ok := event.Value.(float64)
	if !ok {
		logger.Error("build-finished-event-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}
	// seconds are the standard prometheus base unit for time
	duration = duration / 1000
	emitter.buildDurationsVec.WithLabelValues(team, pipeline).Observe(duration)
}

func (emitter *PrometheusEmitter) workerContainersMetric(logger lager.Logger, event metric.Event) {
	worker, exists := event.Attributes["worker"]
	if !exists {
		logger.Error("failed-to-find-worker-in-event", fmt.Errorf("expected worker to exist in event.Attributes"))
	}

	containers, ok := event.Value.(int)
	if !ok {
		logger.Error("worker-volumes-event-value-type-mismatch", fmt.Errorf("expected event.Value to be an int"))
	}

	emitter.workerContainers.WithLabelValues(worker).Set(float64(containers))
}

func (emitter *PrometheusEmitter) workerVolumesMetric(logger lager.Logger, event metric.Event) {
	worker, exists := event.Attributes["worker"]
	if !exists {
		logger.Error("failed-to-find-worker-in-event", fmt.Errorf("expected worker to exist in event.Attributes"))
	}

	volumes, ok := event.Value.(int)
	if !ok {
		logger.Error("worker-volumes-event-value-type-mismatch", fmt.Errorf("expected event.Value to be an int"))
	}

	emitter.workerVolumes.WithLabelValues(worker).Set(float64(volumes))
}

func (emitter *PrometheusEmitter) httpResponseTimeMetrics(logger lager.Logger, event metric.Event) {
	route, exists := event.Attributes["route"]
	if !exists {
		logger.Error("failed-to-find-route-in-event", fmt.Errorf("expected method to exist in event.Attributes"))
	}

	method, exists := event.Attributes["method"]
	if !exists {
		logger.Error("failed-to-find-method-in-event", fmt.Errorf("expected method to exist in event.Attributes"))
	}

	responseTime, ok := event.Value.(float64)
	if !ok {
		logger.Error("http-response-time-event-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}

	emitter.httpRequestsDuration.WithLabelValues(method, route).Observe(responseTime / 1000)
}

func (emitter *PrometheusEmitter) schedulingMetrics(logger lager.Logger, event metric.Event) {
	pipeline, exists := event.Attributes["pipeline"]
	if !exists {
		logger.Error("failed-to-find-pipeline-in-event", fmt.Errorf("expected pipeline to exist in event.Attributes"))
	}

	duration, ok := event.Value.(float64)
	if !ok {
		logger.Error("scheduling-full-duration-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}

	switch event.Name {
	case "scheduling: full duration (ms)":
		// concourse_scheduling_full_duration_seconds
		emitter.schedulingFullDuration.WithLabelValues(pipeline).Set(duration / 1000)
	case "scheduling: loading versions duration (ms)":
		// concourse_scheduling_loading_duration_seconds
		emitter.schedulingLoadingDuration.WithLabelValues(pipeline).Set(duration / 1000)
	case "scheduling: job duration (ms)":
		job, exists := event.Attributes["job"]
		if !exists {
			logger.Error("failed-to-find-job-in-event", fmt.Errorf("expected job to exist in event.Attributes"))
		}
		// concourse_scheduling_job_duration_seconds_total
		emitter.schedulingJobDurationTotalVec.WithLabelValues(pipeline, job).Add(duration / 1000)
		// concourse_scheduling_job_total
		emitter.schedulingJobTotalVec.WithLabelValues(pipeline, job).Inc()
	default:
	}
}

func (emitter *PrometheusEmitter) databaseMetrics(logger lager.Logger, event metric.Event) {
	value, ok := event.Value.(int)
	if !ok {
		logger.Error("db-value-type-mismatch", fmt.Errorf("expected event.Value to be a int"))
	}
	switch event.Name {
	case "database queries":
		emitter.dbQueriesTotal.Add(float64(value))
	case "database connections":
		connectionName, exists := event.Attributes["ConnectionName"]
		if !exists {
			logger.Error("failed-to-connection-name-in-event", fmt.Errorf("expected ConnectionName to exist in event.Attributes"))
		}
		emitter.dbConnections.WithLabelValues(connectionName).Set(float64(value))
	default:
	}

}
<commit_msg>remove job scheduling metric<commit_after>package emitter

import (
	"fmt"
	"net"
	"net/http"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/metric"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

type PrometheusEmitter struct {
	buildsStarted     prometheus.Counter
	buildsFinished    prometheus.Counter
	buildsSucceeded   prometheus.Counter
	buildsErrored     prometheus.Counter
	buildsFailed      prometheus.Counter
	buildsAborted     prometheus.Counter
	buildsFinishedVec *prometheus.CounterVec
	buildDurationsVec *prometheus.HistogramVec

	workerContainers *prometheus.GaugeVec
	workerVolumes    *prometheus.GaugeVec

	httpRequestsDuration *prometheus.HistogramVec

	schedulingFullDuration    *prometheus.GaugeVec
	schedulingLoadingDuration *prometheus.GaugeVec

	dbQueriesTotal prometheus.Counter
	dbConnections  *prometheus.GaugeVec
}

type PrometheusConfig struct {
	BindIP   string `long:"prometheus-bind-ip" description:"IP to listen on to expose Prometheus metrics."`
	BindPort string `long:"prometheus-bind-port" description:"Port to listen on to expose Prometheus metrics."`
}

func init() {
	metric.RegisterEmitter(&PrometheusConfig{})
}

func (config *PrometheusConfig) Description() string { return "Prometheus" }
func (config *PrometheusConfig) IsConfigured() bool {
	return config.BindPort != "" && config.BindIP != ""
}
func (config *PrometheusConfig) bind() string {
	return fmt.Sprintf("%s:%s", config.BindIP, config.BindPort)
}

func (config *PrometheusConfig) NewEmitter() (metric.Emitter, error) {
	// build metrics
	buildsStarted := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "started_total",
		Help:      "Total number of Concourse builds started.",
	})
	prometheus.MustRegister(buildsStarted)

	buildsFinished := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "finished_total",
		Help:      "Total number of Concourse builds finished.",
	})
	prometheus.MustRegister(buildsFinished)

	buildsSucceeded := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "succeeded_total",
		Help:      "Total number of Concourse builds succeeded.",
	})
	prometheus.MustRegister(buildsSucceeded)

	buildsErrored := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "errored_total",
		Help:      "Total number of Concourse builds errored.",
	})
	prometheus.MustRegister(buildsErrored)

	buildsFailed := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "failed_total",
		Help:      "Total number of Concourse builds failed.",
	})
	prometheus.MustRegister(buildsFailed)

	buildsAborted := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "aborted_total",
		Help:      "Total number of Concourse builds aborted.",
	})
	prometheus.MustRegister(buildsAborted)

	buildsFinishedVec := prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "concourse",
			Subsystem: "builds",
			Name:      "finished",
			Help:      "Count of builds finished across various dimensions.",
		},
		[]string{"team", "pipeline", "status"},
	)
	prometheus.MustRegister(buildsFinishedVec)
	buildDurationsVec := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "concourse",
			Subsystem: "builds",
			Name:      "duration_seconds",
			Help:      "Build time in seconds",
			Buckets:   []float64{1, 60, 180, 300, 600, 900, 1200, 1800, 2700, 3600, 7200, 18000, 36000},
		},
		[]string{"team", "pipeline"},
	)
	prometheus.MustRegister(buildDurationsVec)

	// worker metrics
	workerContainers := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "workers",
			Name:      "containers",
			Help:      "Number of containers per worker",
		},
		[]string{"worker"},
	)
	prometheus.MustRegister(workerContainers)

	workerVolumes := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "workers",
			Name:      "volumes",
			Help:      "Number of volumes per worker",
		},
		[]string{"worker"},
	)
	prometheus.MustRegister(workerVolumes)

	// http metrics
	httpRequestsDuration := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "concourse",
			Subsystem: "http_responses",
			Name:      "duration_seconds",
			Help:      "Response time in seconds",
		},
		[]string{"method", "route"},
	)
	prometheus.MustRegister(httpRequestsDuration)

	// scheduling metrics
	schedulingFullDuration := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "scheduling",
			Name:      "full_duration_seconds",
			Help:      "Last time taken to schedule an entire pipeline.",
		},
		[]string{"pipeline"},
	)
	prometheus.MustRegister(schedulingFullDuration)

	schedulingLoadingDuration := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "scheduling",
			Name:      "loading_duration_seconds",
			Help:      "Last time taken to load version information from the database for a pipeline.",
		},
		[]string{"pipeline"},
	)
	prometheus.MustRegister(schedulingLoadingDuration)

	dbQueriesTotal := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "db",
		Name:      "queries_total",
		Help:      "Total number of database Concourse database queries",
	})
	prometheus.MustRegister(dbQueriesTotal)

	dbConnections := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "db",
			Name:      "connections",
			Help:      "Current number of concourse database connections",
		},
		[]string{"dbname"},
	)
	prometheus.MustRegister(dbConnections)

	listener, err := net.Listen("tcp", config.bind())
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, promhttp.Handler())

	return &PrometheusEmitter{
		buildsStarted:     buildsStarted,
		buildsFinished:    buildsFinished,
		buildsFinishedVec: buildsFinishedVec,
		buildDurationsVec: buildDurationsVec,
		buildsSucceeded:   buildsSucceeded,
		buildsErrored:     buildsErrored,
		buildsFailed:      buildsFailed,
		buildsAborted:     buildsAborted,

		workerContainers: workerContainers,
		workerVolumes:    workerVolumes,

		httpRequestsDuration: httpRequestsDuration,

		schedulingFullDuration:    schedulingFullDuration,
		schedulingLoadingDuration: schedulingLoadingDuration,

		dbQueriesTotal: dbQueriesTotal,
		dbConnections:  dbConnections,
	}, nil
}

// Emit processes incoming metrics.
// In order to provide idiomatic Prometheus metrics, we'll have to convert the various
// Event types (differentiated by the less-than-ideal string Name field) into different
// Prometheus metrics.
func (emitter *PrometheusEmitter) Emit(logger lager.Logger, event metric.Event) {
	switch event.Name {
	case "build started":
		emitter.buildsStarted.Inc()
	case "build finished":
		emitter.buildFinishedMetrics(logger, event)
	case "worker containers":
		emitter.workerContainersMetric(logger, event)
	case "worker volumes":
		emitter.workerVolumesMetric(logger, event)
	case "http response time":
		emitter.httpResponseTimeMetrics(logger, event)
	case "scheduling: full duration (ms)":
		emitter.schedulingMetrics(logger, event)
	case "scheduling: loading versions duration (ms)":
		emitter.schedulingMetrics(logger, event)
	case "scheduling: job duration (ms)":
		emitter.schedulingMetrics(logger, event)
	case "database queries":
		emitter.databaseMetrics(logger, event)
	case "database connections":
		emitter.databaseMetrics(logger, event)
	default:
		// unless we have a specific metric, we do nothing
	}
}

func (emitter *PrometheusEmitter) buildFinishedMetrics(logger lager.Logger, event metric.Event) {
	// concourse_builds_finished_total
	emitter.buildsFinished.Inc()

	// concourse_builds_finished
	team, exists := event.Attributes["team_name"]
	if !exists {
		logger.Error("failed-to-find-team-name-in-event", fmt.Errorf("expected team_name to exist in event.Attributes"))
	}

	pipeline, exists := event.Attributes["pipeline"]
	if !exists {
		logger.Error("failed-to-find-pipeline-in-event", fmt.Errorf("expected pipeline to exist in event.Attributes"))
	}

	buildStatus, exists := event.Attributes["build_status"]
	if !exists {
		logger.Error("failed-to-find-build_status-in-event", fmt.Errorf("expected build_status to exist in event.Attributes"))
	}
	emitter.buildsFinishedVec.WithLabelValues(team, pipeline, buildStatus).Inc()

	// concourse_builds_(aborted|succeeded|failed|errored)_total
	switch buildStatus {
	case string(db.BuildStatusAborted):
		// concourse_builds_aborted_total
		emitter.buildsAborted.Inc()
	case string(db.BuildStatusSucceeded):
		// concourse_builds_succeeded_total
		emitter.buildsSucceeded.Inc()
	case string(db.BuildStatusFailed):
		// concourse_builds_failed_total
		emitter.buildsFailed.Inc()
	case string(db.BuildStatusErrored):
		// concourse_builds_errored_total
		emitter.buildsErrored.Inc()
	}

	// concourse_builds_duration_seconds
	duration, ok := event.Value.(float64)
	if !ok {
		logger.Error("build-finished-event-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}
	// seconds are the standard prometheus base unit for time
	duration = duration / 1000
	emitter.buildDurationsVec.WithLabelValues(team, pipeline).Observe(duration)
}

func (emitter *PrometheusEmitter) workerContainersMetric(logger lager.Logger, event metric.Event) {
	worker, exists := event.Attributes["worker"]
	if !exists {
		logger.Error("failed-to-find-worker-in-event", fmt.Errorf("expected worker to exist in event.Attributes"))
	}

	containers, ok := event.Value.(int)
	if !ok {
		logger.Error("worker-volumes-event-value-type-mismatch", fmt.Errorf("expected event.Value to be an int"))
	}

	emitter.workerContainers.WithLabelValues(worker).Set(float64(containers))
}

func (emitter *PrometheusEmitter) workerVolumesMetric(logger lager.Logger, event metric.Event) {
	worker, exists := event.Attributes["worker"]
	if !exists {
		logger.Error("failed-to-find-worker-in-event", fmt.Errorf("expected worker to exist in event.Attributes"))
	}

	volumes, ok := event.Value.(int)
	if !ok {
		logger.Error("worker-volumes-event-value-type-mismatch", fmt.Errorf("expected event.Value to be an int"))
	}

	emitter.workerVolumes.WithLabelValues(worker).Set(float64(volumes))
}

func (emitter *PrometheusEmitter) httpResponseTimeMetrics(logger lager.Logger, event metric.Event) {
	route, exists := event.Attributes["route"]
	if !exists {
		logger.Error("failed-to-find-route-in-event", fmt.Errorf("expected method to exist in event.Attributes"))
	}

	method, exists := event.Attributes["method"]
	if !exists {
		logger.Error("failed-to-find-method-in-event", fmt.Errorf("expected method to exist in event.Attributes"))
	}

	responseTime, ok := event.Value.(float64)
	if !ok {
		logger.Error("http-response-time-event-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}

	emitter.httpRequestsDuration.WithLabelValues(method, route).Observe(responseTime / 1000)
}

func (emitter *PrometheusEmitter) schedulingMetrics(logger lager.Logger, event metric.Event) {
	pipeline, exists := event.Attributes["pipeline"]
	if !exists {
		logger.Error("failed-to-find-pipeline-in-event", fmt.Errorf("expected pipeline to exist in event.Attributes"))
	}

	duration, ok := event.Value.(float64)
	if !ok {
		logger.Error("scheduling-full-duration-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}

	switch event.Name {
	case "scheduling: full duration (ms)":
		// concourse_scheduling_full_duration_seconds
		emitter.schedulingFullDuration.WithLabelValues(pipeline).Set(duration / 1000)
	case "scheduling: loading versions duration (ms)":
		// concourse_scheduling_loading_duration_seconds
		emitter.schedulingLoadingDuration.WithLabelValues(pipeline).Set(duration / 1000)
	default:
	}
}

func (emitter *PrometheusEmitter) databaseMetrics(logger lager.Logger, event metric.Event) {
	value, ok := event.Value.(int)
	if !ok {
		logger.Error("db-value-type-mismatch", fmt.Errorf("expected event.Value to be a int"))
	}
	switch event.Name {
	case "database queries":
		emitter.dbQueriesTotal.Add(float64(value))
	case "database connections":
		connectionName, exists := event.Attributes["ConnectionName"]
		if !exists {
			logger.Error("failed-to-connection-name-in-event", fmt.Errorf("expected ConnectionName to exist in event.Attributes"))
		}
		emitter.dbConnections.WithLabelValues(connectionName).Set(float64(value))
	default:
	}

}
<|endoftext|>


<commit_before>package horizon

import (
	"log"
	"net/http"

	"github.com/stellar/horizon/actions"
	"github.com/stellar/horizon/db"
	"github.com/zenazn/goji/web"
)

// Action is the "base type" for all actions in horizon.  It provides
// structs that embed it with access to the App struct.
//
// Additionally, this type is a trigger for go-codegen and causes
// the file at Action.tmpl to be instantiated for each struct that
// embeds Action.
type Action struct {
	actions.Base
	App *App
}

// Prepare sets the action's App field based upon the goji context
func (action *Action) Prepare(c web.C, w http.ResponseWriter, r *http.Request) {
	base := &action.Base
	base.Prepare(c, w, r)
	action.App = action.GojiCtx.Env["app"].(*App)
}

// GetPagingParams modifies the base GetPagingParams method to replace
// cursors that are "now" with the last seen ledger's cursor.
func (action *Action) GetPagingParams() (cursor string, order string, limit int32) {
	if action.Err != nil {
		return
	}

	cursor, order, limit = action.Base.GetPagingParams()

	if cursor == "now" {
		tid := db.TotalOrderId{
			LedgerSequence: action.App.latestLedgerState.HorizonSequence,
		}
		log.Printf("tid: %#v", tid)
		cursor = tid.String()
	}

	return
}

// GetPageQuery is a helper that returns a new db.PageQuery struct initialized
// using the results from a call to GetPagingParams()
func (action *Action) GetPageQuery() db.PageQuery {
	if action.Err != nil {
		return db.PageQuery{}
	}

	r, err := db.NewPageQuery(action.GetPagingParams())

	if err != nil {
		action.Err = err
	}

	return r
}

func (action *Action) ValidateCursorAsDefault() {
	if action.Err != nil {
		return
	}

	if action.GetString(actions.ParamCursor) == "now" {
		return
	}

	action.GetInt64(actions.ParamCursor)
}
<commit_msg>Add action log helper<commit_after>package horizon

import (
	"net/http"

	"github.com/stellar/horizon/actions"
	"github.com/stellar/horizon/db"
	"github.com/stellar/horizon/log"
	"github.com/zenazn/goji/web"
)

// Action is the "base type" for all actions in horizon.  It provides
// structs that embed it with access to the App struct.
//
// Additionally, this type is a trigger for go-codegen and causes
// the file at Action.tmpl to be instantiated for each struct that
// embeds Action.
type Action struct {
	actions.Base
	App *App
	Log *log.Entry
}

// Prepare sets the action's App field based upon the goji context
func (action *Action) Prepare(c web.C, w http.ResponseWriter, r *http.Request) {
	base := &action.Base
	base.Prepare(c, w, r)
	action.App = action.GojiCtx.Env["app"].(*App)
	action.Log = log.Ctx(action.Ctx)

}

// GetPagingParams modifies the base GetPagingParams method to replace
// cursors that are "now" with the last seen ledger's cursor.
func (action *Action) GetPagingParams() (cursor string, order string, limit int32) {
	if action.Err != nil {
		return
	}

	cursor, order, limit = action.Base.GetPagingParams()

	if cursor == "now" {
		tid := db.TotalOrderId{
			LedgerSequence: action.App.latestLedgerState.HorizonSequence,
		}
		cursor = tid.String()
	}

	return
}

// GetPageQuery is a helper that returns a new db.PageQuery struct initialized
// using the results from a call to GetPagingParams()
func (action *Action) GetPageQuery() db.PageQuery {
	if action.Err != nil {
		return db.PageQuery{}
	}

	r, err := db.NewPageQuery(action.GetPagingParams())

	if err != nil {
		action.Err = err
	}

	return r
}

func (action *Action) ValidateCursorAsDefault() {
	if action.Err != nil {
		return
	}

	if action.GetString(actions.ParamCursor) == "now" {
		return
	}

	action.GetInt64(actions.ParamCursor)
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2017 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package upstream

import (
	"encoding/binary"
	"fmt"
	"github.com/bitmark-inc/bitmarkd/announce"
	"github.com/bitmark-inc/bitmarkd/blockdigest"
	"github.com/bitmark-inc/bitmarkd/counter"
	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/bitmarkd/messagebus"
	"github.com/bitmark-inc/bitmarkd/mode"
	"github.com/bitmark-inc/bitmarkd/util"
	"github.com/bitmark-inc/bitmarkd/zmqutil"
	"github.com/bitmark-inc/logger"
	zmq "github.com/pebbe/zmq4"
	"sync"
	"time"
)

const (
	cycleInterval = 30 * time.Second
	queueSize     = 10 // 0 => synchronous queue
)

type Upstream struct {
	sync.RWMutex
	log         *logger.L
	client      *zmqutil.Client
	registered  bool
	blockHeight uint64
	shutdown    chan<- struct{}
}

// atomically incremented counter for log names
var upstreamCounter counter.Counter

func New(privateKey []byte, publicKey []byte, timeout time.Duration) (*Upstream, error) {
	client, err := zmqutil.NewClient(zmq.REQ, privateKey, publicKey, timeout)
	if nil != err {
		return nil, err
	}

	n := upstreamCounter.Increment()

	shutdown := make(chan struct{})
	u := &Upstream{
		log:         logger.New(fmt.Sprintf("upstream@%d", n)),
		client:      client,
		registered:  false,
		blockHeight: 0,
		shutdown:    shutdown,
	}
	go upstreamRunner(u, shutdown)
	return u, nil
}

func (u *Upstream) Destroy() {
	close(u.shutdown)
}

// check the current destination
//
// does not mean actually connected, as could be in a timeout and
// reconnect state
func (u *Upstream) IsConnectedTo(serverPublicKey []byte) bool {
	return u.client.IsConnectedTo(serverPublicKey)
}

// if registered the have avalid connection
func (u *Upstream) IsOK() bool {
	return u.registered
}

// if registered the have avalid connection
func (u *Upstream) ConnectedTo() *zmqutil.Connected {
	return u.client.ConnectedTo()
}

// connect (or reconnect) to a specific server
func (u *Upstream) Connect(address *util.Connection, serverPublicKey []byte) error {
	u.log.Infof("connecting to address: %s", address)
	u.log.Infof("connecting to server: %x", serverPublicKey)
	u.Lock()
	err := u.client.Connect(address, serverPublicKey, mode.ChainName())
	u.Unlock()
	return err
}

// fetch height from last polled value
func (u *Upstream) GetHeight() uint64 {
	return u.blockHeight
}

// fetch block digest
func (u *Upstream) GetBlockDigest(blockNumber uint64) (blockdigest.Digest, error) {
	parameter := make([]byte, 8)
	binary.BigEndian.PutUint64(parameter, blockNumber)

	// critical section - lock out the runner process
	u.Lock()
	var data [][]byte
	err := u.client.Send("H", parameter)
	if nil == err {
		data, err = u.client.Receive(0)
	}
	u.Unlock()

	if nil != err {
		return blockdigest.Digest{}, err
	}

	if 2 != len(data) {
		return blockdigest.Digest{}, fault.ErrInvalidPeerResponse
	}

	switch string(data[0]) {
	case "E":
		return blockdigest.Digest{}, fault.InvalidError(string(data[1]))
	case "H":
		d := blockdigest.Digest{}
		if blockdigest.Length == len(data[1]) {
			err := blockdigest.DigestFromBytes(&d, data[1])
			return d, err
		}
	default:
	}
	return blockdigest.Digest{}, fault.ErrInvalidPeerResponse
}

// fetch block data
func (u *Upstream) GetBlockData(blockNumber uint64) ([]byte, error) {
	parameter := make([]byte, 8)
	binary.BigEndian.PutUint64(parameter, blockNumber)

	// critical section - lock out the runner process
	u.Lock()
	var data [][]byte
	err := u.client.Send("B", parameter)
	if nil == err {
		data, err = u.client.Receive(0)
	}
	u.Unlock()

	if nil != err {
		return nil, err
	}

	if 2 != len(data) {
		return nil, fault.ErrInvalidPeerResponse
	}

	switch string(data[0]) {
	case "E":
		return nil, fault.InvalidError(string(data[1]))
	case "B":
		return data[1], nil
	default:
	}
	return nil, fault.ErrInvalidPeerResponse
}

func upstreamRunner(u *Upstream, shutdown <-chan struct{}) {
	log := u.log

	log.Info("starting…")

	queue := messagebus.Bus.Broadcast.Chan(queueSize)

loop:
	for {
		log.Info("waiting…")

		select {
		case <-shutdown:
			break loop

		case item := <-queue:
			log.Infof("received: %q  %x", item.Command, item.Parameters)
			if u.registered {
				u.Lock()
				err := push(u.client, u.log, &item)
				if nil != err {
					log.Errorf("push: error: %s", err)
					err := u.client.Reconnect()
					if nil != err {
						log.Errorf("push: reconnect error: %s", err)
					}
				}
				u.Unlock()
			}

		case <-time.After(cycleInterval):
			u.Lock()
			if !u.registered {
				err := register(u.client, u.log)
				if nil != err {
					log.Errorf("register: error: %s", err)
					err := u.client.Reconnect()
					if nil != err {
						log.Errorf("register: reconnect error: %s", err)
					}
					u.Unlock()
					continue loop // try again later
				}
				u.registered = true
			}

			h, err := getHeight(u.client, u.log)
			if nil == err {
				u.blockHeight = h
			} else {
				u.registered = false
				log.Errorf("getHeight: error: %s", err)
				err := u.client.Reconnect()
				if nil != err {
					log.Errorf("highestBlock: reconnect error: %s", err)
				}
			}
			u.Unlock()
		}
	}
	log.Info("shutting down…")
	u.client.Close()
	log.Info("stopped")
}

func register(client *zmqutil.Client, log *logger.L) error {

	log.Debugf("register: client: %s", client)

	err := announce.SendRegistration(client, "R")
	if nil != err {
		return err
	}

	data, err := client.Receive(0)
	if nil != err {
		return err
	}

	if len(data) < 2 {
		return fmt.Errorf("register received: %d  expected at least: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return fmt.Errorf("register error: %q", data[1])
	case "R":
		if len(data) < 6 {
			return fmt.Errorf("register response incorrect: %x", data)
		}
		chain := mode.ChainName()
		received := string(data[1])
		if received != chain {
			log.Criticalf("expected chain: %q but received: %q", chain, received)
			logger.Panicf("expected chain: %q but received: %q", chain, received)
		}

		timestamp := binary.BigEndian.Uint64(data[5])
		log.Infof("register replied: %x:  broadcasts: %x  listeners: %x  timestamp: %d", data[2], data[3], data[4], timestamp)
		announce.AddPeer(data[2], data[3], data[4], timestamp) // publicKey, broadcasts, listeners
		return nil
	default:
		return fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}

func getHeight(client *zmqutil.Client, log *logger.L) (uint64, error) {

	log.Debugf("getHeight: client: %s", client)

	err := client.Send("N")
	if nil != err {
		return 0, err
	}

	data, err := client.Receive(0)
	if nil != err {
		return 0, err
	}
	if 2 != len(data) {
		return 0, fmt.Errorf("getHeight received: %d  expected: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return 0, fmt.Errorf("rpc error response: %q", data[1])
	case "N":
		if 8 != len(data[1]) {
			return 0, fmt.Errorf("highestBlock: rpc invalid response: %q", data[1])
		}
		height := binary.BigEndian.Uint64(data[1])
		log.Infof("height: %d", height)
		return height, nil
	default:
		return 0, fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}

func push(client *zmqutil.Client, log *logger.L, item *messagebus.Message) error {

	log.Debugf("push: client: %s", client)

	err := client.Send(item.Command, item.Parameters)
	if nil != err {
		return err
	}

	data, err := client.Receive(0)
	if nil != err {
		return err
	}
	if 2 != len(data) {
		return fmt.Errorf("push received: %d  expected: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return fmt.Errorf("rpc error response: %q", data[1])
	case "A":
		return nil
	default:
		return fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}
<commit_msg>[peer] detect correct peer response<commit_after>// Copyright (c) 2014-2017 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package upstream

import (
	"encoding/binary"
	"fmt"
	"github.com/bitmark-inc/bitmarkd/announce"
	"github.com/bitmark-inc/bitmarkd/blockdigest"
	"github.com/bitmark-inc/bitmarkd/counter"
	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/bitmarkd/messagebus"
	"github.com/bitmark-inc/bitmarkd/mode"
	"github.com/bitmark-inc/bitmarkd/util"
	"github.com/bitmark-inc/bitmarkd/zmqutil"
	"github.com/bitmark-inc/logger"
	zmq "github.com/pebbe/zmq4"
	"sync"
	"time"
)

const (
	cycleInterval = 30 * time.Second
	queueSize     = 50 // 0 => synchronous queue
)

type Upstream struct {
	sync.RWMutex
	log         *logger.L
	client      *zmqutil.Client
	registered  bool
	blockHeight uint64
	shutdown    chan<- struct{}
}

// atomically incremented counter for log names
var upstreamCounter counter.Counter

func New(privateKey []byte, publicKey []byte, timeout time.Duration) (*Upstream, error) {
	client, err := zmqutil.NewClient(zmq.REQ, privateKey, publicKey, timeout)
	if nil != err {
		return nil, err
	}

	n := upstreamCounter.Increment()

	shutdown := make(chan struct{})
	u := &Upstream{
		log:         logger.New(fmt.Sprintf("upstream@%d", n)),
		client:      client,
		registered:  false,
		blockHeight: 0,
		shutdown:    shutdown,
	}
	go upstreamRunner(u, shutdown)
	return u, nil
}

func (u *Upstream) Destroy() {
	close(u.shutdown)
}

// check the current destination
//
// does not mean actually connected, as could be in a timeout and
// reconnect state
func (u *Upstream) IsConnectedTo(serverPublicKey []byte) bool {
	return u.client.IsConnectedTo(serverPublicKey)
}

// if registered the have avalid connection
func (u *Upstream) IsOK() bool {
	return u.registered
}

// if registered the have avalid connection
func (u *Upstream) ConnectedTo() *zmqutil.Connected {
	return u.client.ConnectedTo()
}

// connect (or reconnect) to a specific server
func (u *Upstream) Connect(address *util.Connection, serverPublicKey []byte) error {
	u.log.Infof("connecting to address: %s", address)
	u.log.Infof("connecting to server: %x", serverPublicKey)
	u.Lock()
	err := u.client.Connect(address, serverPublicKey, mode.ChainName())
	u.Unlock()
	return err
}

// fetch height from last polled value
func (u *Upstream) GetHeight() uint64 {
	return u.blockHeight
}

// fetch block digest
func (u *Upstream) GetBlockDigest(blockNumber uint64) (blockdigest.Digest, error) {
	parameter := make([]byte, 8)
	binary.BigEndian.PutUint64(parameter, blockNumber)

	// critical section - lock out the runner process
	u.Lock()
	var data [][]byte
	err := u.client.Send("H", parameter)
	if nil == err {
		data, err = u.client.Receive(0)
	}
	u.Unlock()

	if nil != err {
		return blockdigest.Digest{}, err
	}

	if 2 != len(data) {
		return blockdigest.Digest{}, fault.ErrInvalidPeerResponse
	}

	switch string(data[0]) {
	case "E":
		return blockdigest.Digest{}, fault.InvalidError(string(data[1]))
	case "H":
		d := blockdigest.Digest{}
		if blockdigest.Length == len(data[1]) {
			err := blockdigest.DigestFromBytes(&d, data[1])
			return d, err
		}
	default:
	}
	return blockdigest.Digest{}, fault.ErrInvalidPeerResponse
}

// fetch block data
func (u *Upstream) GetBlockData(blockNumber uint64) ([]byte, error) {
	parameter := make([]byte, 8)
	binary.BigEndian.PutUint64(parameter, blockNumber)

	// critical section - lock out the runner process
	u.Lock()
	var data [][]byte
	err := u.client.Send("B", parameter)
	if nil == err {
		data, err = u.client.Receive(0)
	}
	u.Unlock()

	if nil != err {
		return nil, err
	}

	if 2 != len(data) {
		return nil, fault.ErrInvalidPeerResponse
	}

	switch string(data[0]) {
	case "E":
		return nil, fault.InvalidError(string(data[1]))
	case "B":
		return data[1], nil
	default:
	}
	return nil, fault.ErrInvalidPeerResponse
}

func upstreamRunner(u *Upstream, shutdown <-chan struct{}) {
	log := u.log

	log.Info("starting…")

	queue := messagebus.Bus.Broadcast.Chan(queueSize)

loop:
	for {
		log.Info("waiting…")

		select {
		case <-shutdown:
			break loop

		case item := <-queue:
			log.Infof("from queue: %q  %x", item.Command, item.Parameters)
			if u.registered {
				u.Lock()
				err := push(u.client, u.log, &item)
				if nil != err {
					log.Errorf("push: error: %s", err)
					err := u.client.Reconnect()
					if nil != err {
						log.Errorf("push: reconnect error: %s", err)
					}
				}
				u.Unlock()
			}

		case <-time.After(cycleInterval):
			u.Lock()
			if !u.registered {
				err := register(u.client, u.log)
				if fault.ErrNotConnected == err {
					log.Infof("register: %s", err)
					u.Unlock()
					continue loop // try again late
				} else if nil != err {
					log.Errorf("register: error: %s", err)
					err := u.client.Reconnect()
					if nil != err {
						log.Errorf("register: reconnect error: %s", err)
					}
					u.Unlock()
					continue loop // try again later
				}
				u.registered = true
			}

			h, err := getHeight(u.client, u.log)
			if nil == err {
				u.blockHeight = h
			} else {
				u.registered = false
				log.Errorf("getHeight: error: %s", err)
				err := u.client.Reconnect()
				if nil != err {
					log.Errorf("highestBlock: reconnect error: %s", err)
				}
			}
			u.Unlock()
		}
	}
	log.Info("shutting down…")
	u.client.Close()
	log.Info("stopped")
}

func register(client *zmqutil.Client, log *logger.L) error {

	log.Debugf("register: client: %s", client)

	err := announce.SendRegistration(client, "R")
	if nil != err {
		return err
	}

	data, err := client.Receive(0)
	if nil != err {
		return err
	}

	if len(data) < 2 {
		return fmt.Errorf("register received: %d  expected at least: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return fmt.Errorf("register error: %q", data[1])
	case "R":
		if len(data) < 6 {
			return fmt.Errorf("register response incorrect: %x", data)
		}
		chain := mode.ChainName()
		received := string(data[1])
		if received != chain {
			log.Criticalf("expected chain: %q but received: %q", chain, received)
			logger.Panicf("expected chain: %q but received: %q", chain, received)
		}

		timestamp := binary.BigEndian.Uint64(data[5])
		log.Infof("register replied: %x:  broadcasts: %x  listeners: %x  timestamp: %d", data[2], data[3], data[4], timestamp)
		announce.AddPeer(data[2], data[3], data[4], timestamp) // publicKey, broadcasts, listeners
		return nil
	default:
		return fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}

func getHeight(client *zmqutil.Client, log *logger.L) (uint64, error) {

	log.Debugf("getHeight: client: %s", client)

	err := client.Send("N")
	if nil != err {
		return 0, err
	}

	data, err := client.Receive(0)
	if nil != err {
		return 0, err
	}
	if 2 != len(data) {
		return 0, fmt.Errorf("getHeight received: %d  expected: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return 0, fmt.Errorf("rpc error response: %q", data[1])
	case "N":
		if 8 != len(data[1]) {
			return 0, fmt.Errorf("highestBlock: rpc invalid response: %q", data[1])
		}
		height := binary.BigEndian.Uint64(data[1])
		log.Infof("height: %d", height)
		return height, nil
	default:
		return 0, fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}

func push(client *zmqutil.Client, log *logger.L, item *messagebus.Message) error {

	log.Debugf("push: client: %s  %q %x", client, item.Command, item.Parameters)

	err := client.Send(item.Command, item.Parameters)
	if nil != err {
		log.Errorf("push: %s send error: %s", client, err)
		return err
	}

	data, err := client.Receive(0)
	if nil != err {
		log.Errorf("push: %s receive error: %s", client, err)
		return err
	}
	if 2 != len(data) {
		return fmt.Errorf("push received: %d  expected: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return fmt.Errorf("rpc error response: %q", data[1])
	case item.Command:
		log.Debugf("push: client: %s complete: %q", client, data[1])
		return nil
	default:
		return fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}
<|endoftext|>


<commit_before>package qemu

import (
	"fmt"
	"log"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/template/interpolate"
)

// stepRun runs the virtual machine
type stepRun struct {
	BootDrive string
	Message   string
}

type qemuArgsTemplateData struct {
	HTTPIP      string
	HTTPPort    uint
	HTTPDir     string
	OutputDir   string
	Name        string
	SSHHostPort uint
}

func (s *stepRun) Run(state multistep.StateBag) multistep.StepAction {
	driver := state.Get("driver").(Driver)
	ui := state.Get("ui").(packer.Ui)

	ui.Say(s.Message)

	command, err := getCommandArgs(s.BootDrive, state)
	if err != nil {
		err := fmt.Errorf("Error processing QemuArggs: %s", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	if err := driver.Qemu(command...); err != nil {
		err := fmt.Errorf("Error launching VM: %s", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	return multistep.ActionContinue
}

func (s *stepRun) Cleanup(state multistep.StateBag) {
	driver := state.Get("driver").(Driver)
	ui := state.Get("ui").(packer.Ui)

	if err := driver.Stop(); err != nil {
		ui.Error(fmt.Sprintf("Error shutting down VM: %s", err))
	}
}

func getCommandArgs(bootDrive string, state multistep.StateBag) ([]string, error) {
	config := state.Get("config").(*Config)
	isoPath := state.Get("iso_path").(string)
	vncPort := state.Get("vnc_port").(uint)
	sshHostPort := state.Get("sshHostPort").(uint)
	ui := state.Get("ui").(packer.Ui)
	driver := state.Get("driver").(Driver)

	vnc := fmt.Sprintf("0.0.0.0:%d", vncPort-5900)
	vmName := config.VMName
	imgPath := filepath.Join(config.OutputDir, vmName)

	defaultArgs := make(map[string]interface{})
	var deviceArgs []string
	var driveArgs []string

	defaultArgs["-name"] = vmName
	defaultArgs["-machine"] = fmt.Sprintf("type=%s", config.MachineType)
	defaultArgs["-netdev"] = fmt.Sprintf("user,id=user.0,hostfwd=tcp::%v-:%d", sshHostPort, config.Comm.Port())

	qemuVersion, err := driver.Version()
	if err != nil {
		return nil, err
	}
	parts := strings.Split(qemuVersion, ".")
	qemuMajor, err := strconv.Atoi(parts[0])
	if err != nil {
		return nil, err
	}
	if qemuMajor >= 2 {
		if config.DiskInterface == "virtio-scsi" {
			deviceArgs = append(deviceArgs, "virtio-scsi-pci,id=scsi0", "scsi-hd,bus=scsi0.0,drive=drive0")
			driveArgs = append(driveArgs, fmt.Sprintf("if=none,file=%s,id=drive0,cache=%s,discard=%s", imgPath, config.DiskCache, config.DiskDiscard))
		} else {
			driveArgs = append(driveArgs, fmt.Sprintf("file=%s,if=%s,cache=%s,discard=%s", imgPath, config.DiskInterface, config.DiskCache, config.DiskDiscard))
		}
	} else {
		driveArgs["-drive"] = fmt.Sprintf("file=%s,if=%s,cache=%s", imgPath, config.DiskInterface, config.DiskCache)
	}
	deviceArgs = append(deviceArgs, fmt.Sprintf("%s,netdev=user.0", config.NetDevice))

	if config.Headless == true {
		ui.Message("WARNING: The VM will be started in headless mode, as configured.\n" +
			"In headless mode, errors during the boot sequence or OS setup\n" +
			"won't be easily visible. Use at your own discretion.")
	} else {
		if qemuMajor >= 2 {
			defaultArgs["-display"] = "sdl"
		} else {
			ui.Message("WARNING: The version of qemu  on your host doesn't support display mode.\n" +
				"The display parameter will be ignored.")
		}
	}

	defaultArgs["-device"] = deviceArgs
	defaultArgs["-drive"] = driveArgs

	if !config.DiskImage {
		defaultArgs["-cdrom"] = isoPath
	}
	defaultArgs["-boot"] = bootDrive
	defaultArgs["-m"] = "512M"
	defaultArgs["-vnc"] = vnc

	// Append the accelerator to the machine type if it is specified
	if config.Accelerator != "none" {
		defaultArgs["-machine"] = fmt.Sprintf("%s,accel=%s", defaultArgs["-machine"], config.Accelerator)
	} else {
		ui.Message("WARNING: The VM will be started with no hardware acceleration.\n" +
			"The installation may take considerably longer to finish.\n")
	}

	// Determine if we have a floppy disk to attach
	if floppyPathRaw, ok := state.GetOk("floppy_path"); ok {
		defaultArgs["-fda"] = floppyPathRaw.(string)
	} else {
		log.Println("Qemu Builder has no floppy files, not attaching a floppy.")
	}

	inArgs := make(map[string][]string)
	if len(config.QemuArgs) > 0 {
		ui.Say("Overriding defaults Qemu arguments with QemuArgs...")

		httpPort := state.Get("http_port").(uint)
		ctx := config.ctx
		ctx.Data = qemuArgsTemplateData{
			"10.0.2.2",
			httpPort,
			config.HTTPDir,
			config.OutputDir,
			config.VMName,
			sshHostPort,
		}
		newQemuArgs, err := processArgs(config.QemuArgs, &ctx)
		if err != nil {
			return nil, err
		}

		// because qemu supports multiple appearances of the same
		// switch, just different values, each key in the args hash
		// will have an array of string values
		for _, qemuArgs := range newQemuArgs {
			key := qemuArgs[0]
			val := strings.Join(qemuArgs[1:], "")
			if _, ok := inArgs[key]; !ok {
				inArgs[key] = make([]string, 0)
			}
			if len(val) > 0 {
				inArgs[key] = append(inArgs[key], val)
			}
		}
	}

	// get any remaining missing default args from the default settings
	for key := range defaultArgs {
		if _, ok := inArgs[key]; !ok {
			arg := make([]string, 1)
			switch defaultArgs[key].(type) {
			case string:
				arg[0] = defaultArgs[key].(string)
			case []string:
				arg = defaultArgs[key].([]string)
			}
			inArgs[key] = arg
		}
	}

	// Flatten to array of strings
	outArgs := make([]string, 0)
	for key, values := range inArgs {
		if len(values) > 0 {
			for idx := range values {
				outArgs = append(outArgs, key, values[idx])
			}
		} else {
			outArgs = append(outArgs, key)
		}
	}

	return outArgs, nil
}

func processArgs(args [][]string, ctx *interpolate.Context) ([][]string, error) {
	var err error

	if args == nil {
		return make([][]string, 0), err
	}

	newArgs := make([][]string, len(args))
	for argsIdx, rowArgs := range args {
		parms := make([]string, len(rowArgs))
		newArgs[argsIdx] = parms
		for i, parm := range rowArgs {
			parms[i], err = interpolate.Render(parm, ctx)
			if err != nil {
				return nil, err
			}
		}
	}

	return newArgs, err
}
<commit_msg>Revert "close #3192"<commit_after>package qemu

import (
	"fmt"
	"log"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/template/interpolate"
)

// stepRun runs the virtual machine
type stepRun struct {
	BootDrive string
	Message   string
}

type qemuArgsTemplateData struct {
	HTTPIP      string
	HTTPPort    uint
	HTTPDir     string
	OutputDir   string
	Name        string
	SSHHostPort uint
}

func (s *stepRun) Run(state multistep.StateBag) multistep.StepAction {
	driver := state.Get("driver").(Driver)
	ui := state.Get("ui").(packer.Ui)

	ui.Say(s.Message)

	command, err := getCommandArgs(s.BootDrive, state)
	if err != nil {
		err := fmt.Errorf("Error processing QemuArggs: %s", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	if err := driver.Qemu(command...); err != nil {
		err := fmt.Errorf("Error launching VM: %s", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	return multistep.ActionContinue
}

func (s *stepRun) Cleanup(state multistep.StateBag) {
	driver := state.Get("driver").(Driver)
	ui := state.Get("ui").(packer.Ui)

	if err := driver.Stop(); err != nil {
		ui.Error(fmt.Sprintf("Error shutting down VM: %s", err))
	}
}

func getCommandArgs(bootDrive string, state multistep.StateBag) ([]string, error) {
	config := state.Get("config").(*Config)
	isoPath := state.Get("iso_path").(string)
	vncPort := state.Get("vnc_port").(uint)
	sshHostPort := state.Get("sshHostPort").(uint)
	ui := state.Get("ui").(packer.Ui)
	driver := state.Get("driver").(Driver)

	vnc := fmt.Sprintf("0.0.0.0:%d", vncPort-5900)
	vmName := config.VMName
	imgPath := filepath.Join(config.OutputDir, vmName)

	defaultArgs := make(map[string]interface{})
	var deviceArgs []string
	var driveArgs []string

	defaultArgs["-name"] = vmName
	defaultArgs["-machine"] = fmt.Sprintf("type=%s", config.MachineType)
	defaultArgs["-netdev"] = fmt.Sprintf("user,id=user.0,hostfwd=tcp::%v-:%d", sshHostPort, config.Comm.Port())

	qemuVersion, err := driver.Version()
	if err != nil {
		return nil, err
	}
	parts := strings.Split(qemuVersion, ".")
	qemuMajor, err := strconv.Atoi(parts[0])
	if err != nil {
		return nil, err
	}
	if qemuMajor >= 2 {
		if config.DiskInterface == "virtio-scsi" {
			deviceArgs = append(deviceArgs, "virtio-scsi-pci,id=scsi0", "scsi-hd,bus=scsi0.0,drive=drive0")
			driveArgs = append(driveArgs, fmt.Sprintf("if=none,file=%s,id=drive0,cache=%s,discard=%s", imgPath, config.DiskCache, config.DiskDiscard))
		} else {
			driveArgs = append(driveArgs, fmt.Sprintf("file=%s,if=%s,cache=%s,discard=%s", imgPath, config.DiskInterface, config.DiskCache, config.DiskDiscard))
		}
	} else {
		defaultArgs["-drive"] = fmt.Sprintf("file=%s,if=%s,cache=%s", imgPath, config.DiskInterface, config.DiskCache)
	}
	deviceArgs = append(deviceArgs, fmt.Sprintf("%s,netdev=user.0", config.NetDevice))

	if config.Headless == true {
		ui.Message("WARNING: The VM will be started in headless mode, as configured.\n" +
			"In headless mode, errors during the boot sequence or OS setup\n" +
			"won't be easily visible. Use at your own discretion.")
	} else {
		if qemuMajor >= 2 {
			defaultArgs["-display"] = "sdl"
		} else {
			ui.Message("WARNING: The version of qemu  on your host doesn't support display mode.\n" +
				"The display parameter will be ignored.")
		}
	}

	defaultArgs["-device"] = deviceArgs
	defaultArgs["-drive"] = driveArgs

	if !config.DiskImage {
		defaultArgs["-cdrom"] = isoPath
	}
	defaultArgs["-boot"] = bootDrive
	defaultArgs["-m"] = "512M"
	defaultArgs["-vnc"] = vnc

	// Append the accelerator to the machine type if it is specified
	if config.Accelerator != "none" {
		defaultArgs["-machine"] = fmt.Sprintf("%s,accel=%s", defaultArgs["-machine"], config.Accelerator)
	} else {
		ui.Message("WARNING: The VM will be started with no hardware acceleration.\n" +
			"The installation may take considerably longer to finish.\n")
	}

	// Determine if we have a floppy disk to attach
	if floppyPathRaw, ok := state.GetOk("floppy_path"); ok {
		defaultArgs["-fda"] = floppyPathRaw.(string)
	} else {
		log.Println("Qemu Builder has no floppy files, not attaching a floppy.")
	}

	inArgs := make(map[string][]string)
	if len(config.QemuArgs) > 0 {
		ui.Say("Overriding defaults Qemu arguments with QemuArgs...")

		httpPort := state.Get("http_port").(uint)
		ctx := config.ctx
		ctx.Data = qemuArgsTemplateData{
			"10.0.2.2",
			httpPort,
			config.HTTPDir,
			config.OutputDir,
			config.VMName,
			sshHostPort,
		}
		newQemuArgs, err := processArgs(config.QemuArgs, &ctx)
		if err != nil {
			return nil, err
		}

		// because qemu supports multiple appearances of the same
		// switch, just different values, each key in the args hash
		// will have an array of string values
		for _, qemuArgs := range newQemuArgs {
			key := qemuArgs[0]
			val := strings.Join(qemuArgs[1:], "")
			if _, ok := inArgs[key]; !ok {
				inArgs[key] = make([]string, 0)
			}
			if len(val) > 0 {
				inArgs[key] = append(inArgs[key], val)
			}
		}
	}

	// get any remaining missing default args from the default settings
	for key := range defaultArgs {
		if _, ok := inArgs[key]; !ok {
			arg := make([]string, 1)
			switch defaultArgs[key].(type) {
			case string:
				arg[0] = defaultArgs[key].(string)
			case []string:
				arg = defaultArgs[key].([]string)
			}
			inArgs[key] = arg
		}
	}

	// Flatten to array of strings
	outArgs := make([]string, 0)
	for key, values := range inArgs {
		if len(values) > 0 {
			for idx := range values {
				outArgs = append(outArgs, key, values[idx])
			}
		} else {
			outArgs = append(outArgs, key)
		}
	}

	return outArgs, nil
}

func processArgs(args [][]string, ctx *interpolate.Context) ([][]string, error) {
	var err error

	if args == nil {
		return make([][]string, 0), err
	}

	newArgs := make([][]string, len(args))
	for argsIdx, rowArgs := range args {
		parms := make([]string, len(rowArgs))
		newArgs[argsIdx] = parms
		for i, parm := range rowArgs {
			parms[i], err = interpolate.Render(parm, ctx)
			if err != nil {
				return nil, err
			}
		}
	}

	return newArgs, err
}
<|endoftext|>