<commit_before>package api

import (
	"encoding/hex"
	"fmt"
	"net/http"

	"github.com/rivine/rivine/build"
	"github.com/rivine/rivine/crypto"
	"github.com/rivine/rivine/encoding"
	"github.com/rivine/rivine/modules"
	"github.com/rivine/rivine/types"

	"github.com/julienschmidt/httprouter"
)

type (
	// ExplorerBlock is a block with some extra information such as the id and
	// height. This information is provided for programs that may not be
	// complex enough to compute the ID on their own.
	ExplorerBlock struct {
		MinerPayoutIDs []types.CoinOutputID  `json:"minerpayoutids"`
		Transactions   []ExplorerTransaction `json:"transactions"`
		RawBlock       types.Block           `json:"rawblock"`
		HexBlock       string                `json:"hexblock"`

		modules.BlockFacts
	}

	// ExplorerTransaction is a transcation with some extra information such as
	// the parent block. This information is provided for programs that may not
	// be complex enough to compute the extra information on their own.
	ExplorerTransaction struct {
		ID             types.TransactionID `json:"id"`
		Height         types.BlockHeight   `json:"height"`
		Parent         types.BlockID       `json:"parent"`
		RawTransaction types.Transaction   `json:"rawtransaction"`
		HexTransaction string              `json:"hextransaction"`

		CoinInputOutputs       []types.CoinOutput         `json:"coininputoutputs"` // the outputs being spent
		CoinOutputIDs          []types.CoinOutputID       `json:"coinoutputids"`
		BlockStakeInputOutputs []types.BlockStakeOutput   `json:"blockstakeinputoutputs"` // the outputs being spent
		BlockStakeOutputIDs    []types.BlockStakeOutputID `json:"blockstakeoutputids"`
	}

	// ExplorerGET is the object returned as a response to a GET request to
	// /explorer.
	ExplorerGET struct {
		modules.BlockFacts
	}

	// ExplorerBlockGET is the object returned by a GET request to
	// /explorer/block.
	ExplorerBlockGET struct {
		Block ExplorerBlock `json:"block"`
	}

	// ExplorerHashGET is the object returned as a response to a GET request to
	// /explorer/hash. The HashType will indicate whether the hash corresponds
	// to a block id, a transaction id, a siacoin output id, a file contract
	// id, or a siafund output id. In the case of a block id, 'Block' will be
	// filled out and all the rest of the fields will be blank. In the case of
	// a transaction id, 'Transaction' will be filled out and all the rest of
	// the fields will be blank. For everything else, 'Transactions' and
	// 'Blocks' will/may be filled out and everything else will be blank.
	ExplorerHashGET struct {
		HashType     string                `json:"hashtype"`
		Block        ExplorerBlock         `json:"block"`
		Blocks       []ExplorerBlock       `json:"blocks"`
		Transaction  ExplorerTransaction   `json:"transaction"`
		Transactions []ExplorerTransaction `json:"transactions"`
		Unconfirmed  bool                  `json:"unconfirmed"`
	}
)

// buildExplorerTransaction takes a transaction and the height + id of the
// block it appears in an uses that to build an explorer transaction.
func (api *API) buildExplorerTransaction(height types.BlockHeight, parent types.BlockID, txn types.Transaction) (et ExplorerTransaction) {
	// Get the header information for the transaction.
	et.ID = txn.ID()
	et.Height = height
	et.Parent = parent
	et.RawTransaction = txn
	et.HexTransaction = hex.EncodeToString(encoding.Marshal(txn))

	// Add the siacoin outputs that correspond with each siacoin input.
	for _, sci := range txn.CoinInputs {
		sco, exists := api.explorer.CoinOutput(sci.ParentID)
		if build.DEBUG && !exists {
			panic("could not find corresponding coin output")
		}
		et.CoinInputOutputs = append(et.CoinInputOutputs, sco)
	}

	for i := range txn.CoinOutputs {
		et.CoinOutputIDs = append(et.CoinOutputIDs, txn.CoinOutputID(uint64(i)))
	}

	// Add the siafund outputs that correspond to each siacoin input.
	for _, sci := range txn.BlockStakeInputs {
		sco, exists := api.explorer.BlockStakeOutput(sci.ParentID)
		if build.DEBUG && !exists {
			panic("could not find corresponding blockstake output")
		}
		et.BlockStakeInputOutputs = append(et.BlockStakeInputOutputs, sco)
	}

	for i := range txn.BlockStakeOutputs {
		et.BlockStakeOutputIDs = append(et.BlockStakeOutputIDs, txn.BlockStakeOutputID(uint64(i)))
	}

	return et
}

// buildExplorerBlock takes a block and its height and uses it to construct an
// explorer block.
func (api *API) buildExplorerBlock(height types.BlockHeight, block types.Block) ExplorerBlock {
	var mpoids []types.CoinOutputID
	for i := range block.MinerPayouts {
		mpoids = append(mpoids, block.MinerPayoutID(uint64(i)))
	}

	var etxns []ExplorerTransaction
	for _, txn := range block.Transactions {
		etxns = append(etxns, api.buildExplorerTransaction(height, block.ID(), txn))
	}

	facts, exists := api.explorer.BlockFacts(height)
	if build.DEBUG && !exists {
		panic("incorrect request to buildExplorerBlock - block does not exist")
	}

	return ExplorerBlock{
		MinerPayoutIDs: mpoids,
		Transactions:   etxns,
		RawBlock:       block,
		HexBlock:       hex.EncodeToString(encoding.Marshal(block)),

		BlockFacts: facts,
	}
}

// explorerHandler handles API calls to /explorer/blocks/:height.
func (api *API) explorerBlocksHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
	// Parse the height that's being requested.
	var height types.BlockHeight
	_, err := fmt.Sscan(ps.ByName("height"), &height)
	if err != nil {
		WriteError(w, Error{err.Error()}, http.StatusBadRequest)
		return
	}

	// Fetch and return the explorer block.
	block, exists := api.cs.BlockAtHeight(height)
	if !exists {
		WriteError(w, Error{"no block found at input height in call to /explorer/block"}, http.StatusBadRequest)
		return
	}
	WriteJSON(w, ExplorerBlockGET{
		Block: api.buildExplorerBlock(height, block),
	})
}

// buildTransactionSet returns the blocks and transactions that are associated
// with a set of transaction ids.
func (api *API) buildTransactionSet(txids []types.TransactionID) (txns []ExplorerTransaction, blocks []ExplorerBlock) {
	for _, txid := range txids {
		// Get the block containing the transaction - in the case of miner
		// payouts, the block might be the transaction.
		block, height, exists := api.explorer.Transaction(txid)
		if !exists && build.DEBUG {
			panic("explorer pointing to nonexistent txn")
		}

		// Check if the block is the transaction.
		if types.TransactionID(block.ID()) == txid {
			blocks = append(blocks, api.buildExplorerBlock(height, block))
		} else {
			// Find the transaction within the block with the correct id.
			for _, t := range block.Transactions {
				if t.ID() == txid {
					txns = append(txns, api.buildExplorerTransaction(height, block.ID(), t))
					break
				}
			}
		}
	}
	return txns, blocks
}

// explorerHashHandler handles GET requests to /explorer/hash/:hash.
func (api *API) explorerHashHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
	// Scan the hash as a hash. If that fails, try scanning the hash as an
	// address.
	hash, err := scanHash(ps.ByName("hash"))
	if err != nil {
		addr, err := scanAddress(ps.ByName("hash"))
		if err != nil {
			WriteError(w, Error{err.Error()}, http.StatusBadRequest)
			return
		}

		// Try the hash as an unlock hash. Unlock hash is checked last because
		// unlock hashes do not have collision-free guarantees. Someone can create
		// an unlock hash that collides with another object id. They will not be
		// able to use the unlock hash, but they can disrupt the explorer. This is
		// handled by checking the unlock hash last. Anyone intentionally creating
		// a colliding unlock hash (such a collision can only happen if done
		// intentionally) will be unable to find their unlock hash in the
		// blockchain through the explorer hash lookup.
		txids := api.explorer.UnlockHash(addr)
		if len(txids) != 0 {
			txns, blocks := api.buildTransactionSet(txids)
			WriteJSON(w, ExplorerHashGET{
				HashType:     "unlockhash",
				Blocks:       blocks,
				Transactions: txns,
			})
			return
		}

		// Hash not found, return an error.
		WriteError(w, Error{"unrecognized hash used as input to /explorer/hash"}, http.StatusBadRequest)
		return
	}

	// TODO: lookups on the zero hash are too expensive to allow. Need a
	// better way to handle this case.
	if hash == (crypto.Hash{}) {
		WriteError(w, Error{"can't lookup the empty unlock hash"}, http.StatusBadRequest)
		return
	}

	// Try the hash as a block id.
	block, height, exists := api.explorer.Block(types.BlockID(hash))
	if exists {
		WriteJSON(w, ExplorerHashGET{
			HashType: "blockid",
			Block:    api.buildExplorerBlock(height, block),
		})
		return
	}

	// Try the hash as a transaction id.
	block, height, exists = api.explorer.Transaction(types.TransactionID(hash))
	if exists {
		var txn types.Transaction
		for _, t := range block.Transactions {
			if t.ID() == types.TransactionID(hash) {
				txn = t
			}
		}
		WriteJSON(w, ExplorerHashGET{
			HashType:    HashTypeTransactionIDStr,
			Transaction: api.buildExplorerTransaction(height, block.ID(), txn),
		})
		return
	}

	// Try the hash as a siacoin output id.
	txids := api.explorer.CoinOutputID(types.CoinOutputID(hash))
	if len(txids) != 0 {
		txns, blocks := api.buildTransactionSet(txids)
		WriteJSON(w, ExplorerHashGET{
			HashType:     HashTypeCoinOutputIDStr,
			Blocks:       blocks,
			Transactions: txns,
		})
		return
	}

	// Try the hash as a siafund output id.
	txids = api.explorer.BlockStakeOutputID(types.BlockStakeOutputID(hash))
	if len(txids) != 0 {
		txns, blocks := api.buildTransactionSet(txids)
		WriteJSON(w, ExplorerHashGET{
			HashType:     HashTypeBlockStakeOutputIDStr,
			Blocks:       blocks,
			Transactions: txns,
		})
		return
	}

	// if the transaction pool is available, try to use it
	if api.tpool != nil {
		// Try the hash as a transactionID in the transaction pool
		txn, err := api.tpool.Transaction(types.TransactionID(hash))
		if err == nil {
			WriteJSON(w, ExplorerHashGET{
				HashType:    HashTypeTransactionIDStr,
				Transaction: api.buildExplorerTransaction(0, types.BlockID{}, txn),
				Unconfirmed: true,
			})
			return
		}
		if err != modules.ErrTransactionNotFound {
			WriteError(w, Error{
				"error during call to /explorer/hash: failed to get txn from transaction pool: " + err.Error()},
				http.StatusInternalServerError)
		}
	}

	// Hash not found, return an error.
	WriteError(w, Error{"unrecognized hash used as input to /explorer/hash"}, http.StatusBadRequest)
}

// hash type string constants
const (
	HashTypeTransactionIDStr      = "transactionid"
	HashTypeCoinOutputIDStr       = "coinoutputid"
	HashTypeBlockStakeOutputIDStr = "blockstakeoutputid"
)

// explorerHandler handles API calls to /explorer
func (api *API) explorerHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
	facts := api.explorer.LatestBlockFacts()
	WriteJSON(w, ExplorerGET{
		BlockFacts: facts,
	})
}

func (api *API) constantsHandler(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) {
	WriteJSON(w, api.explorer.Constants())
}

func (api *API) historyStatsHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
	var history types.BlockHeight
	// GET request so the only place the vars can be is the queryparams
	q := req.URL.Query()
	_, err := fmt.Sscan(q.Get("history"), &history)
	if err != nil {
		WriteError(w, Error{err.Error()}, http.StatusBadRequest)
		return
	}
	stats, err := api.explorer.HistoryStats(history)
	if err != nil {
		WriteError(w, Error{err.Error()}, http.StatusBadRequest)
		return
	}
	WriteJSON(w, stats)
}

func (api *API) rangeStatsHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
	var start, end types.BlockHeight
	// GET request so the only place the vars can be is the queryparams
	q := req.URL.Query()
	_, err := fmt.Sscan(q.Get("start"), &start)
	if err != nil {
		WriteError(w, Error{err.Error()}, http.StatusBadRequest)
		return
	}
	_, err = fmt.Sscan(q.Get("end"), &end)
	if err != nil {
		WriteError(w, Error{err.Error()}, http.StatusBadRequest)
		return
	}
	stats, err := api.explorer.RangeStats(start, end)
	if err != nil {
		WriteError(w, Error{err.Error()}, http.StatusBadRequest)
		return
	}
	WriteJSON(w, stats)
}
<commit_msg>add unlockhashes to explorer transactions<commit_after>package api

import (
	"encoding/hex"
	"fmt"
	"net/http"

	"github.com/rivine/rivine/build"
	"github.com/rivine/rivine/crypto"
	"github.com/rivine/rivine/encoding"
	"github.com/rivine/rivine/modules"
	"github.com/rivine/rivine/types"

	"github.com/julienschmidt/httprouter"
)

type (
	// ExplorerBlock is a block with some extra information such as the id and
	// height. This information is provided for programs that may not be
	// complex enough to compute the ID on their own.
	ExplorerBlock struct {
		MinerPayoutIDs []types.CoinOutputID  `json:"minerpayoutids"`
		Transactions   []ExplorerTransaction `json:"transactions"`
		RawBlock       types.Block           `json:"rawblock"`
		HexBlock       string                `json:"hexblock"`

		modules.BlockFacts
	}

	// ExplorerTransaction is a transcation with some extra information such as
	// the parent block. This information is provided for programs that may not
	// be complex enough to compute the extra information on their own.
	ExplorerTransaction struct {
		ID             types.TransactionID `json:"id"`
		Height         types.BlockHeight   `json:"height"`
		Parent         types.BlockID       `json:"parent"`
		RawTransaction types.Transaction   `json:"rawtransaction"`
		HexTransaction string              `json:"hextransaction"`

		CoinInputOutputs             []ExplorerCoinOutput       `json:"coininputoutputs"` // the outputs being spent
		CoinOutputIDs                []types.CoinOutputID       `json:"coinoutputids"`
		CoinOutputUnlockHashes       []types.UnlockHash         `json:"coinoutputunlockhashes"`
		BlockStakeInputOutputs       []ExplorerBlockStakeOutput `json:"blockstakeinputoutputs"` // the outputs being spent
		BlockStakeOutputIDs          []types.BlockStakeOutputID `json:"blockstakeoutputids"`
		BlockStakeOutputUnlockHashes []types.UnlockHash         `json:"blockstakeunlockhashes"`
	}

	// ExplorerCoinOutput is the same a regular types.CoinOutput,
	// but with the addition of the pre-computed UnlockHash of its condition.
	ExplorerCoinOutput struct {
		types.CoinOutput
		UnlockHash types.UnlockHash `json:"unlockhash"`
	}

	// ExplorerBlockStakeOutput is the same a regular types.BlockStakeOutput,
	// but with the addition of the pre-computed UnlockHash of its condition.
	ExplorerBlockStakeOutput struct {
		types.BlockStakeOutput
		UnlockHash types.UnlockHash `json:"unlockhash"`
	}

	// ExplorerGET is the object returned as a response to a GET request to
	// /explorer.
	ExplorerGET struct {
		modules.BlockFacts
	}

	// ExplorerBlockGET is the object returned by a GET request to
	// /explorer/block.
	ExplorerBlockGET struct {
		Block ExplorerBlock `json:"block"`
	}

	// ExplorerHashGET is the object returned as a response to a GET request to
	// /explorer/hash. The HashType will indicate whether the hash corresponds
	// to a block id, a transaction id, a siacoin output id, a file contract
	// id, or a siafund output id. In the case of a block id, 'Block' will be
	// filled out and all the rest of the fields will be blank. In the case of
	// a transaction id, 'Transaction' will be filled out and all the rest of
	// the fields will be blank. For everything else, 'Transactions' and
	// 'Blocks' will/may be filled out and everything else will be blank.
	ExplorerHashGET struct {
		HashType     string                `json:"hashtype"`
		Block        ExplorerBlock         `json:"block"`
		Blocks       []ExplorerBlock       `json:"blocks"`
		Transaction  ExplorerTransaction   `json:"transaction"`
		Transactions []ExplorerTransaction `json:"transactions"`
		Unconfirmed  bool                  `json:"unconfirmed"`
	}
)

// buildExplorerTransaction takes a transaction and the height + id of the
// block it appears in an uses that to build an explorer transaction.
func (api *API) buildExplorerTransaction(height types.BlockHeight, parent types.BlockID, txn types.Transaction) (et ExplorerTransaction) {
	// Get the header information for the transaction.
	et.ID = txn.ID()
	et.Height = height
	et.Parent = parent
	et.RawTransaction = txn
	et.HexTransaction = hex.EncodeToString(encoding.Marshal(txn))

	// Add the siacoin outputs that correspond with each siacoin input.
	for _, sci := range txn.CoinInputs {
		sco, exists := api.explorer.CoinOutput(sci.ParentID)
		if build.DEBUG && !exists {
			panic("could not find corresponding coin output")
		}
		et.CoinInputOutputs = append(et.CoinInputOutputs, ExplorerCoinOutput{
			CoinOutput: sco,
			UnlockHash: sco.Condition.UnlockHash(),
		})
	}

	for i, co := range txn.CoinOutputs {
		et.CoinOutputIDs = append(et.CoinOutputIDs, txn.CoinOutputID(uint64(i)))
		et.CoinOutputUnlockHashes = append(et.CoinOutputUnlockHashes, co.Condition.UnlockHash())
	}

	// Add the siafund outputs that correspond to each siacoin input.
	for _, sci := range txn.BlockStakeInputs {
		sco, exists := api.explorer.BlockStakeOutput(sci.ParentID)
		if build.DEBUG && !exists {
			panic("could not find corresponding blockstake output")
		}
		et.BlockStakeInputOutputs = append(et.BlockStakeInputOutputs, ExplorerBlockStakeOutput{
			BlockStakeOutput: sco,
			UnlockHash:       sco.Condition.UnlockHash(),
		})
	}

	for i, bso := range txn.BlockStakeOutputs {
		et.BlockStakeOutputIDs = append(et.BlockStakeOutputIDs, txn.BlockStakeOutputID(uint64(i)))
		et.BlockStakeOutputUnlockHashes = append(et.BlockStakeOutputUnlockHashes, bso.Condition.UnlockHash())
	}

	return et
}

// buildExplorerBlock takes a block and its height and uses it to construct an
// explorer block.
func (api *API) buildExplorerBlock(height types.BlockHeight, block types.Block) ExplorerBlock {
	var mpoids []types.CoinOutputID
	for i := range block.MinerPayouts {
		mpoids = append(mpoids, block.MinerPayoutID(uint64(i)))
	}

	var etxns []ExplorerTransaction
	for _, txn := range block.Transactions {
		etxns = append(etxns, api.buildExplorerTransaction(height, block.ID(), txn))
	}

	facts, exists := api.explorer.BlockFacts(height)
	if build.DEBUG && !exists {
		panic("incorrect request to buildExplorerBlock - block does not exist")
	}

	return ExplorerBlock{
		MinerPayoutIDs: mpoids,
		Transactions:   etxns,
		RawBlock:       block,
		HexBlock:       hex.EncodeToString(encoding.Marshal(block)),

		BlockFacts: facts,
	}
}

// explorerHandler handles API calls to /explorer/blocks/:height.
func (api *API) explorerBlocksHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
	// Parse the height that's being requested.
	var height types.BlockHeight
	_, err := fmt.Sscan(ps.ByName("height"), &height)
	if err != nil {
		WriteError(w, Error{err.Error()}, http.StatusBadRequest)
		return
	}

	// Fetch and return the explorer block.
	block, exists := api.cs.BlockAtHeight(height)
	if !exists {
		WriteError(w, Error{"no block found at input height in call to /explorer/block"}, http.StatusBadRequest)
		return
	}
	WriteJSON(w, ExplorerBlockGET{
		Block: api.buildExplorerBlock(height, block),
	})
}

// buildTransactionSet returns the blocks and transactions that are associated
// with a set of transaction ids.
func (api *API) buildTransactionSet(txids []types.TransactionID) (txns []ExplorerTransaction, blocks []ExplorerBlock) {
	for _, txid := range txids {
		// Get the block containing the transaction - in the case of miner
		// payouts, the block might be the transaction.
		block, height, exists := api.explorer.Transaction(txid)
		if !exists && build.DEBUG {
			panic("explorer pointing to nonexistent txn")
		}

		// Check if the block is the transaction.
		if types.TransactionID(block.ID()) == txid {
			blocks = append(blocks, api.buildExplorerBlock(height, block))
		} else {
			// Find the transaction within the block with the correct id.
			for _, t := range block.Transactions {
				if t.ID() == txid {
					txns = append(txns, api.buildExplorerTransaction(height, block.ID(), t))
					break
				}
			}
		}
	}
	return txns, blocks
}

// explorerHashHandler handles GET requests to /explorer/hash/:hash.
func (api *API) explorerHashHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
	// Scan the hash as a hash. If that fails, try scanning the hash as an
	// address.
	hash, err := scanHash(ps.ByName("hash"))
	if err != nil {
		addr, err := scanAddress(ps.ByName("hash"))
		if err != nil {
			WriteError(w, Error{err.Error()}, http.StatusBadRequest)
			return
		}

		// Try the hash as an unlock hash. Unlock hash is checked last because
		// unlock hashes do not have collision-free guarantees. Someone can create
		// an unlock hash that collides with another object id. They will not be
		// able to use the unlock hash, but they can disrupt the explorer. This is
		// handled by checking the unlock hash last. Anyone intentionally creating
		// a colliding unlock hash (such a collision can only happen if done
		// intentionally) will be unable to find their unlock hash in the
		// blockchain through the explorer hash lookup.
		txids := api.explorer.UnlockHash(addr)
		if len(txids) != 0 {
			txns, blocks := api.buildTransactionSet(txids)
			WriteJSON(w, ExplorerHashGET{
				HashType:     "unlockhash",
				Blocks:       blocks,
				Transactions: txns,
			})
			return
		}

		// Hash not found, return an error.
		WriteError(w, Error{"unrecognized hash used as input to /explorer/hash"}, http.StatusBadRequest)
		return
	}

	// TODO: lookups on the zero hash are too expensive to allow. Need a
	// better way to handle this case.
	if hash == (crypto.Hash{}) {
		WriteError(w, Error{"can't lookup the empty unlock hash"}, http.StatusBadRequest)
		return
	}

	// Try the hash as a block id.
	block, height, exists := api.explorer.Block(types.BlockID(hash))
	if exists {
		WriteJSON(w, ExplorerHashGET{
			HashType: "blockid",
			Block:    api.buildExplorerBlock(height, block),
		})
		return
	}

	// Try the hash as a transaction id.
	block, height, exists = api.explorer.Transaction(types.TransactionID(hash))
	if exists {
		var txn types.Transaction
		for _, t := range block.Transactions {
			if t.ID() == types.TransactionID(hash) {
				txn = t
			}
		}
		WriteJSON(w, ExplorerHashGET{
			HashType:    HashTypeTransactionIDStr,
			Transaction: api.buildExplorerTransaction(height, block.ID(), txn),
		})
		return
	}

	// Try the hash as a siacoin output id.
	txids := api.explorer.CoinOutputID(types.CoinOutputID(hash))
	if len(txids) != 0 {
		txns, blocks := api.buildTransactionSet(txids)
		WriteJSON(w, ExplorerHashGET{
			HashType:     HashTypeCoinOutputIDStr,
			Blocks:       blocks,
			Transactions: txns,
		})
		return
	}

	// Try the hash as a siafund output id.
	txids = api.explorer.BlockStakeOutputID(types.BlockStakeOutputID(hash))
	if len(txids) != 0 {
		txns, blocks := api.buildTransactionSet(txids)
		WriteJSON(w, ExplorerHashGET{
			HashType:     HashTypeBlockStakeOutputIDStr,
			Blocks:       blocks,
			Transactions: txns,
		})
		return
	}

	// if the transaction pool is available, try to use it
	if api.tpool != nil {
		// Try the hash as a transactionID in the transaction pool
		txn, err := api.tpool.Transaction(types.TransactionID(hash))
		if err == nil {
			WriteJSON(w, ExplorerHashGET{
				HashType:    HashTypeTransactionIDStr,
				Transaction: api.buildExplorerTransaction(0, types.BlockID{}, txn),
				Unconfirmed: true,
			})
			return
		}
		if err != modules.ErrTransactionNotFound {
			WriteError(w, Error{
				"error during call to /explorer/hash: failed to get txn from transaction pool: " + err.Error()},
				http.StatusInternalServerError)
		}
	}

	// Hash not found, return an error.
	WriteError(w, Error{"unrecognized hash used as input to /explorer/hash"}, http.StatusBadRequest)
}

// hash type string constants
const (
	HashTypeTransactionIDStr      = "transactionid"
	HashTypeCoinOutputIDStr       = "coinoutputid"
	HashTypeBlockStakeOutputIDStr = "blockstakeoutputid"
)

// explorerHandler handles API calls to /explorer
func (api *API) explorerHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
	facts := api.explorer.LatestBlockFacts()
	WriteJSON(w, ExplorerGET{
		BlockFacts: facts,
	})
}

func (api *API) constantsHandler(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) {
	WriteJSON(w, api.explorer.Constants())
}

func (api *API) historyStatsHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
	var history types.BlockHeight
	// GET request so the only place the vars can be is the queryparams
	q := req.URL.Query()
	_, err := fmt.Sscan(q.Get("history"), &history)
	if err != nil {
		WriteError(w, Error{err.Error()}, http.StatusBadRequest)
		return
	}
	stats, err := api.explorer.HistoryStats(history)
	if err != nil {
		WriteError(w, Error{err.Error()}, http.StatusBadRequest)
		return
	}
	WriteJSON(w, stats)
}

func (api *API) rangeStatsHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
	var start, end types.BlockHeight
	// GET request so the only place the vars can be is the queryparams
	q := req.URL.Query()
	_, err := fmt.Sscan(q.Get("start"), &start)
	if err != nil {
		WriteError(w, Error{err.Error()}, http.StatusBadRequest)
		return
	}
	_, err = fmt.Sscan(q.Get("end"), &end)
	if err != nil {
		WriteError(w, Error{err.Error()}, http.StatusBadRequest)
		return
	}
	stats, err := api.explorer.RangeStats(start, end)
	if err != nil {
		WriteError(w, Error{err.Error()}, http.StatusBadRequest)
		return
	}
	WriteJSON(w, stats)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"github.com/Sirupsen/logrus"
	"github.com/coreos/go-omaha/omaha"
	"strconv"
)

// parse an 'app' tag of request and generate a corresponding 'app' tag of response
func handleApiApp(logContext *logrus.Entry, appRequest, appResponse *omaha.App) {
	if appRequest.Id != coreOSAppID {
		appResponse.Status = "error-unknownApplication"
	} else {
		appResponse.Status = "ok"
	}

	// <UpdateCheck> tag
	if appRequest.UpdateCheck != nil {
		logContext.Debug("Handling UpdateCheck")
		ucResp := appResponse.AddUpdateCheck()

		appVersion, err := parseVersionString(appRequest.Version)
		if err != nil {
			logContext.Errorf("Could not parse client's version string: %v", err.Error())
			ucResp.Status = "error-invalidVersionString"
		} else {
			handleApiUpdateCheck(logContext, appVersion, appRequest.UpdateCheck, ucResp)
		}
	}

	// <ping> tag
	if appRequest.Ping != nil {
		// TODO register info from the ping

		// response is always "ok" according to the specs
		responsePing := appResponse.AddPing()
		responsePing.Status = "ok"
	}

	// <Event> tag
	for _, event := range appRequest.Events {
		logContext.Warnf("Event to handle: '%v', resultv '%v'", event.Type, event.Result)
		// TODO handle event
	}
}

// parse an 'UpdateCheck' tag of request and generate a corresponding 'UpdateCheck' tag of response
func handleApiUpdateCheck(logContext *logrus.Entry, appVersion payloadVersion, ucRequest, ucResp *omaha.UpdateCheck) {
	payload, err := db.GetNewerPayload(appVersion)
	if err != nil {
		if err == sql.ErrNoRows {
			// TODO already at the newest version response
			// TODO respond properly
			logContext.Infof("Client already up-to-date")
		} else {
			// TODO respond properly
			logContext.Errorf("Failed checking for newer payload: %v", err.Error())
		}
	} else {
		logContext.Infof("Found update to version '%v' (id %v)", "1.2.3.4.5.6", payload.Url)

		ucResp.Status = "ok"
		ucResp.AddUrl("http://10.0.2.2:8080/file?id=")

		manifest := ucResp.AddManifest("1.0.2")
		manifest.AddPackage(payload.SHA1, payload.Url, strconv.FormatInt(payload.Size, 10), true)
		action := manifest.AddAction("postinstall")
		action.Sha256 = payload.SHA256
		action.DisablePayloadBackoff = true
	}
}
<commit_msg>UpdateCheck response - noupdate and err<commit_after>package main

import (
	"database/sql"
	"github.com/Sirupsen/logrus"
	"github.com/coreos/go-omaha/omaha"
	"strconv"
)

// parse an 'app' tag of request and generate a corresponding 'app' tag of response
func handleApiApp(logContext *logrus.Entry, appRequest, appResponse *omaha.App) {
	if appRequest.Id != coreOSAppID {
		appResponse.Status = "error-unknownApplication"
	} else {
		appResponse.Status = "ok"
	}

	// <UpdateCheck> tag
	if appRequest.UpdateCheck != nil {
		logContext.Debug("Handling UpdateCheck")
		ucResp := appResponse.AddUpdateCheck()

		appVersion, err := parseVersionString(appRequest.Version)
		if err != nil {
			logContext.Errorf("Could not parse client's version string: %v", err.Error())
			ucResp.Status = "error-invalidVersionString"
		} else {
			handleApiUpdateCheck(logContext, appVersion, appRequest.UpdateCheck, ucResp)
		}
	}

	// <ping> tag
	if appRequest.Ping != nil {
		// TODO register info from the ping

		// response is always "ok" according to the specs
		responsePing := appResponse.AddPing()
		responsePing.Status = "ok"
	}

	// <Event> tag
	for _, event := range appRequest.Events {
		logContext.Warnf("Event to handle: '%v', resultv '%v'", event.Type, event.Result)
		// TODO handle event
	}
}

// parse an 'UpdateCheck' tag of request and generate a corresponding 'UpdateCheck' tag of response
func handleApiUpdateCheck(logContext *logrus.Entry, appVersion payloadVersion, ucRequest, ucResp *omaha.UpdateCheck) {
	payload, err := db.GetNewerPayload(appVersion)
	if err != nil {
		if err == sql.ErrNoRows {
			logContext.Infof("Client already up-to-date")
			ucRequest.Status = "noupdate"
		} else {
			logContext.Errorf("Failed checking for newer payload: %v", err.Error())
			ucRequest.Status = "error-internal"
		}
	} else {
		logContext.Infof("Found update to version '%v' (id %v)", "1.2.3.4.5.6", payload.Url)

		ucResp.Status = "ok"
		ucResp.AddUrl("http://10.0.2.2:8080/file?id=")

		manifest := ucResp.AddManifest("1.0.2")
		manifest.AddPackage(payload.SHA1, payload.Url, strconv.FormatInt(payload.Size, 10), true)
		action := manifest.AddAction("postinstall")
		action.Sha256 = payload.SHA256
		action.DisablePayloadBackoff = true
	}
}
<|endoftext|>


<commit_before>package app

import (
	"github.com/gilcrest/go-api-basic/datastore"
	"github.com/rs/zerolog"
	"os"
	"reflect"
	"testing"
)

func TestEnvName_String(t *testing.T) {
	tests := []struct {
		name string
		n    EnvName
		want string
	}{
		{"Production", Production, "Production"},
		{"Staging", Staging, "Staging"},
		{"QA", QA, "QA"},
		{"Local", Local, "Local"},
		{"Unknown Name", 99, "unknown_name"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := tt.n.String(); got != tt.want {
				t.Errorf("String() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestNewApplication(t *testing.T) {
	type args struct {
		en  EnvName
		ds  datastore.Datastorer
		log zerolog.Logger
	}
	tests := []struct {
		name string
		args args
		want *Application
	}{
		{"New Application", args{
			en:  Local,
			ds:  nil,
			log: zerolog.Logger{},
		}, &Application{
			EnvName:    Local,
			Mock:       false,
			Datastorer: nil,
			Logger:     zerolog.Logger{},
		}},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := NewApplication(tt.args.en, tt.args.ds, tt.args.log); !reflect.DeepEqual(got, tt.want) {
				t.Errorf("NewApplication() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestNewLogger(t *testing.T) {
	type args struct {
		lvl zerolog.Level
	}

	// empty string for TimeFieldFormat will write logs with UNIX time
	zerolog.TimeFieldFormat = ""
	// set logging level based on input
	zerolog.SetGlobalLevel(zerolog.DebugLevel)
	// start a new logger with Stdout as the target
	lgr := zerolog.New(os.Stdout).With().Timestamp().Logger()

	tests := []struct {
		name string
		args args
		want zerolog.Logger
	}{
		{"New Logger", args{zerolog.DebugLevel}, lgr},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := NewLogger(tt.args.lvl); !reflect.DeepEqual(got, tt.want) {
				t.Errorf("NewLogger() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestNewMockedApplication(t *testing.T) {
	type args struct {
		en  EnvName
		ds  datastore.Datastorer
		log zerolog.Logger
	}
	tests := []struct {
		name string
		args args
		want *Application
	}{
		{"New Mocked Application", args{
			en:  Local,
			ds:  nil,
			log: zerolog.Logger{},
		}, &Application{
			EnvName:    Local,
			Mock:       true,
			Datastorer: nil,
			Logger:     zerolog.Logger{},
		}},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := NewMockedApplication(tt.args.en, tt.args.ds, tt.args.log); !reflect.DeepEqual(got, tt.want) {
				t.Errorf("NewMockedApplication() = %v, want %v", got, tt.want)
			}
		})
	}
}
<commit_msg>test for GCPSeverityHook<commit_after>package app

import (
	"bytes"
	"encoding/json"
	"os"
	"reflect"
	"testing"

	"github.com/gilcrest/go-api-basic/datastore"
	"github.com/rs/zerolog"
)

func TestEnvName_String(t *testing.T) {
	tests := []struct {
		name string
		n    EnvName
		want string
	}{
		{"Production", Production, "Production"},
		{"Staging", Staging, "Staging"},
		{"QA", QA, "QA"},
		{"Local", Local, "Local"},
		{"Unknown Name", 99, "unknown_name"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := tt.n.String(); got != tt.want {
				t.Errorf("String() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestNewApplication(t *testing.T) {
	type args struct {
		en  EnvName
		ds  datastore.Datastorer
		log zerolog.Logger
	}
	tests := []struct {
		name string
		args args
		want *Application
	}{
		{"New Application", args{
			en:  Local,
			ds:  nil,
			log: zerolog.Logger{},
		}, &Application{
			EnvName:    Local,
			Datastorer: nil,
			Logger:     zerolog.Logger{},
		}},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := NewApplication(tt.args.en, tt.args.ds, tt.args.log); !reflect.DeepEqual(got, tt.want) {
				t.Errorf("NewApplication() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestNewLogger(t *testing.T) {
	type args struct {
		lvl zerolog.Level
	}

	// empty string for TimeFieldFormat will write logs with UNIX time
	zerolog.TimeFieldFormat = ""
	// set logging level based on input
	zerolog.SetGlobalLevel(zerolog.DebugLevel)
	// start a new logger with Stdout as the target
	lgr := zerolog.New(os.Stdout).With().Timestamp().Logger()
	lgr = lgr.Hook(GCPSeverityHook{})

	tests := []struct {
		name string
		args args
		want zerolog.Logger
	}{
		{"New Logger", args{zerolog.DebugLevel}, lgr},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := NewLogger(tt.args.lvl); !reflect.DeepEqual(got, tt.want) {
				t.Errorf("NewLogger() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestGCPSeverityHook_Run(t *testing.T) {
	// empty string for TimeFieldFormat will write logs with UNIX time
	zerolog.TimeFieldFormat = ""
	var b bytes.Buffer
	lgr := zerolog.New(&b).With().Timestamp().Logger().Hook(GCPSeverityHook{})

	type args struct {
		f     func()
		level zerolog.Level
		msg   string
		sev   string
	}
	tests := []struct {
		name string
		args args
	}{
		{zerolog.PanicLevel.String(), args{func() { lgr.Panic().Msg("") }, zerolog.PanicLevel, "", "EMERGENCY"}},
		//{zerolog.FatalLevel.String(), args{func() { lgr.Fatal().Msg("") }, zerolog.FatalLevel, "", "EMERGENCY"}},
		{zerolog.ErrorLevel.String(), args{func() { lgr.Error().Msg("") }, zerolog.ErrorLevel, "", "ERROR"}},
		{zerolog.WarnLevel.String(), args{func() { lgr.Warn().Msg("") }, zerolog.WarnLevel, "", "WARNING"}},
		{zerolog.InfoLevel.String(), args{func() { lgr.Info().Msg("") }, zerolog.InfoLevel, "", "INFO"}},
		{zerolog.DebugLevel.String(), args{func() { lgr.Debug().Msg("") }, zerolog.DebugLevel, "", "DEBUG"}},
		{zerolog.TraceLevel.String(), args{func() { lgr.Trace().Msg("") }, zerolog.TraceLevel, "", "DEBUG"}},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if tt.name == zerolog.PanicLevel.String() {
				defer func() {
					if r := recover(); r == nil {
						t.Errorf("Code should have panicked")
					}
				}()
			}
			b.Reset()
			tt.args.f()
			var dat map[string]interface{}
			if err := json.Unmarshal(b.Bytes(), &dat); err != nil {
				t.Fatalf("json Unmarshal error: %v", err)
			}
			got := dat["severity"].(string)
			want := tt.args.sev

			if got != want {
				t.Errorf("event.Msg() = %q, want %q", got, want)
			}
		})
	}
}
<|endoftext|>


<commit_before>// All this package really does is add a little structure and a little color
// to the standard log package

package logsip

import (
	"fmt"
	"io"
	"log"
	"os"

	"github.com/fatih/color"
)

// Logger is a modified log.Logger which provides logging levels.
type Logger struct {
	WarnPrefix  string
	FatalPrefix string
	InfoPrefix  string
	PanicPrefix string
	DebugPrefix string
	DebugMode   bool
	*log.Logger
}

// New returns a blank Logger so you can define your own prefixes.
func New(out io.Writer) *Logger {
	return &Logger{Logger: log.New(out, "", 0)}
}

// Default returns a new Logger with the default color and prefix options.
func Default() *Logger {
	return &Logger{
		WarnPrefix:  color.New(color.FgYellow).SprintFunc()("==> Warn: "),
		InfoPrefix:  color.New(color.FgCyan).SprintFunc()("==> Info: "),
		FatalPrefix: color.New(color.FgRed).SprintFunc()("==> Fatal: "),
		PanicPrefix: color.New(color.FgRed).SprintFunc()("==> Panic: "),
		DebugPrefix: color.New(color.FgMagenta).SprintFunc()("==> Debug: "),
		DebugMode:   false,
		Logger:      log.New(os.Stdout, "", 0),
	}
}

// Info works just like log.Print however adds the Info prefix.
func (l *Logger) Info(v ...interface{}) {
	l.Print(l.InfoPrefix + fmt.Sprint(v...))
}

// Debug works just like log.Print however adds the Debug prefix.
func (l *Logger) Debug(v ...interface{}) {
	if l.DebugMode {
		l.Print(l.DebugPrefix + fmt.Sprint(v...))
	}
}

// Warn works just like log.Print however adds the Warn prefix.
func (l *Logger) Warn(v ...interface{}) {
	l.Print(l.WarnPrefix + fmt.Sprint(v...))
}

// Fatal works just like log.Fatal however adds the Fatal prefix.
func (l *Logger) Fatal(v ...interface{}) {
	l.Fatal(l.FatalPrefix + fmt.Sprint(v...))
}

// Panic works just like log.Panic however adds the Panic prefix.
func (l *Logger) Panic(v ...interface{}) {
	l.Panic(l.PanicPrefix + fmt.Sprint(v...))
}

// Infof works just like log.Printf however adds the Info prefix.
func (l *Logger) Infof(format string, v ...interface{}) {
	l.Print(l.InfoPrefix + fmt.Sprintf(format, v...))
}

// Debugf works just like log.Printf however adds the Debug prefix.
func (l *Logger) Debugf(format string, v ...interface{}) {
	if l.DebugMode {
		l.Print(l.DebugPrefix + fmt.Sprintf(format, v...))
	}
}

// Warnf works just like log.Printf however adds the Warn prefix.
func (l *Logger) Warnf(format string, v ...interface{}) {
	l.Print(l.WarnPrefix + fmt.Sprintf(format, v...))
}

// Fatalf works just like log.Fatalf however adds the Fatal prefix.
func (l *Logger) Fatalf(format string, v ...interface{}) {
	l.Fatalf(l.FatalPrefix + fmt.Sprintf(format, v...))
}

// Panicf works just like log.Panicf however adds the Panic prefix.
func (l *Logger) Panicf(format string, v ...interface{}) {
	l.Panicf(l.PanicPrefix + fmt.Sprintf(format, v...))
}

// Infoln works just like log.Println however adds the Info prefix
func (l *Logger) Infoln(v ...interface{}) {
	l.Println(l.InfoPrefix + fmt.Sprint(v...))
}

// Debugln works just like log.Println however adds the Debug prefix.
func (l *Logger) Debugln(v ...interface{}) {
	if l.DebugMode {
		l.Println(l.DebugPrefix + fmt.Sprint(v...))
	}
}

// Warnln works just like log.Println however adds the Warnln prefix
func (l *Logger) Warnln(v ...interface{}) {
	l.Println(l.WarnPrefix + fmt.Sprint(v...))
}

// Fatalln works just like log.Fatalln however adds the Fatal prefix
func (l *Logger) Fatalln(v ...interface{}) {
	l.Fatalln(l.FatalPrefix + fmt.Sprint(v...))
}

// Panicln works just like log.Panicln however adds the Panic prefix
func (l *Logger) Panicln(v ...interface{}) {
	l.Panicln(l.PanicPrefix + fmt.Sprint(v...))
}
<commit_msg>Changed the godoc stuff to be a little more readable<commit_after>// All this package really does is add a little structure and a little color
// to the standard log package

package logsip

import (
	"fmt"
	"io"
	"log"
	"os"

	"github.com/fatih/color"
)

// Logger is a modified log.Logger which provides logging levels.
type Logger struct {
	WarnPrefix  string
	FatalPrefix string
	InfoPrefix  string
	PanicPrefix string
	DebugPrefix string
	DebugMode   bool
	*log.Logger
}

// New returns a blank Logger so you can define your own prefixes.
func New(out io.Writer) *Logger {
	return &Logger{Logger: log.New(out, "", 0)}
}

// Default returns a new Logger with the default color and prefix options.
func Default() *Logger {
	return &Logger{
		WarnPrefix:  color.New(color.FgYellow).SprintFunc()("==> Warn: "),
		InfoPrefix:  color.New(color.FgCyan).SprintFunc()("==> Info: "),
		FatalPrefix: color.New(color.FgRed).SprintFunc()("==> Fatal: "),
		PanicPrefix: color.New(color.FgRed).SprintFunc()("==> Panic: "),
		DebugPrefix: color.New(color.FgMagenta).SprintFunc()("==> Debug: "),
		DebugMode:   false,
		Logger:      log.New(os.Stdout, "", 0),
	}
}

// Info works just like log.Print, but with a Cyan "==> Info:" prefix.
func (l *Logger) Info(v ...interface{}) {
	l.Print(l.InfoPrefix + fmt.Sprint(v...))
}

// Debug works just like log.Print, but with a Magenta "==> Debug:" prefix.
func (l *Logger) Debug(v ...interface{}) {
	if l.DebugMode {
		l.Print(l.DebugPrefix + fmt.Sprint(v...))
	}
}

// Warn works just like log.Print, but with a Yellow "==> Warn:" prefix.
func (l *Logger) Warn(v ...interface{}) {
	l.Print(l.WarnPrefix + fmt.Sprint(v...))
}

// Fatal works just like log.Fatal, but with a Red "==> Fatal:" prefix.
func (l *Logger) Fatal(v ...interface{}) {
	l.Fatal(l.FatalPrefix + fmt.Sprint(v...))
}

// Panic works just like log.Panic, but with a Red "==> Panic:" prefix.
func (l *Logger) Panic(v ...interface{}) {
	l.Panic(l.PanicPrefix + fmt.Sprint(v...))
}

// Infof works just like log.Printf, but with a Cyan "==> Info:" prefix.
func (l *Logger) Infof(format string, v ...interface{}) {
	l.Print(l.InfoPrefix + fmt.Sprintf(format, v...))
}

// Debugf works just like log.Printf, but with a Magenta "==> Debug:" prefix.
func (l *Logger) Debugf(format string, v ...interface{}) {
	if l.DebugMode {
		l.Print(l.DebugPrefix + fmt.Sprintf(format, v...))
	}
}

// Warnf works just like log.Printf, but with a Yellow "==> Warn:" prefix.
func (l *Logger) Warnf(format string, v ...interface{}) {
	l.Print(l.WarnPrefix + fmt.Sprintf(format, v...))
}

// Fatalf works just like log.Fatalf, but with a Red "==> Fatal:" prefix.
func (l *Logger) Fatalf(format string, v ...interface{}) {
	l.Fatalf(l.FatalPrefix + fmt.Sprintf(format, v...))
}

// Panicf works just like log.Panicf, but with a Red "==> Panic:" prefix.
func (l *Logger) Panicf(format string, v ...interface{}) {
	l.Panicf(l.PanicPrefix + fmt.Sprintf(format, v...))
}

// Infoln works just like log.Println, but with a Cyan "==> Info:" prefix.
func (l *Logger) Infoln(v ...interface{}) {
	l.Println(l.InfoPrefix + fmt.Sprint(v...))
}

// Debugln works just like log.Println, but with a Magenta "==> Debug:" prefix.
func (l *Logger) Debugln(v ...interface{}) {
	if l.DebugMode {
		l.Println(l.DebugPrefix + fmt.Sprint(v...))
	}
}

// Warnln works just like log.Println, but with a Yellow "==> Warn:" prefix.
func (l *Logger) Warnln(v ...interface{}) {
	l.Println(l.WarnPrefix + fmt.Sprint(v...))
}

// Fatalln works just like log.Fatalln, but with a Red "==> Fatal:" prefix.
func (l *Logger) Fatalln(v ...interface{}) {
	l.Fatalln(l.FatalPrefix + fmt.Sprint(v...))
}

// Panicln works just like log.Panicln, but with a Red "==> Panic:" prefix.
func (l *Logger) Panicln(v ...interface{}) {
	l.Panicln(l.PanicPrefix + fmt.Sprint(v...))
}
<|endoftext|>


<commit_before>package benchmarks

import (
	"io/ioutil"
	"runtime"
	"testing"

	"github.com/tdewolff/minify/v2/minify"
	"github.com/tdewolff/parse/v2"
	"github.com/tdewolff/parse/v2/buffer"
)

func benchmark(b *testing.B, mediatype string, sample string) {
	m := minify.Default
	buf, err := ioutil.ReadFile(sample)
	if err != nil {
		panic(err)
	}
	b.Run(sample, func(b *testing.B) {
		b.SetBytes(int64(len(buf)))
		for i := 0; i < b.N; i++ {
			b.StopTimer()
			runtime.GC()
			r := buffer.NewReader(parse.Copy(buf))
			w := buffer.NewWriter(make([]byte, 0, len(buf)))
			b.StartTimer()

			if err := m.Minify(mediatype, w, r); err != nil {
				b.Fatal(err)
			}
		}
	})
}

func BenchmarkCSS(b *testing.B) {
	var samples = []string{
		"sample_bootstrap.css",
		"sample_gumby.css",
		"sample_fontawesome.css",
		"sample_normalize.css",
	}
	for _, sample := range samples {
		benchmark(b, "text/css", sample)
	}
}

func BenchmarkHTML(b *testing.B) {
	var samples = []string{
		"sample_amazon.html",
		"sample_bbc.html",
		"sample_blogpost.html",
		"sample_es6.html",
		"sample_stackoverflow.html",
		"sample_wikipedia.html",
	}
	for _, sample := range samples {
		benchmark(b, "text/html", sample)
	}
}

func BenchmarkJS(b *testing.B) {
	var samples = []string{
		"sample_ace.js",
		"sample_dot.js",
		"sample_jquery.js",
		"sample_jqueryui.js",
		"sample_moment.js",
	}
	for _, sample := range samples {
		benchmark(b, "application/javascript", sample)
	}
}

func BenchmarkJSON(b *testing.B) {
	var samples = []string{
		"sample_large.json",
		"sample_testsuite.json",
		"sample_twitter.json",
	}
	for _, sample := range samples {
		benchmark(b, "application/json", sample)
	}
}

func BenchmarkSVG(b *testing.B) {
	var samples = []string{
		"sample_arctic.svg",
		"sample_gopher.svg",
		"sample_usa.svg",
		"sample_car.svg",
		"sample_tiger.svg",
	}
	for _, sample := range samples {
		benchmark(b, "image/svg+xml", sample)
	}
}

func BenchmarkXML(b *testing.B) {
	var samples = []string{
		"sample_books.xml",
		"sample_catalog.xml",
		"sample_omg.xml",
	}
	for _, sample := range samples {
		benchmark(b, "application/xml", sample)
	}
}
<commit_msg>Update benchmark function<commit_after>package benchmarks

import (
	"io/ioutil"
	"runtime"
	"testing"

	"github.com/tdewolff/minify/v2/minify"
	"github.com/tdewolff/parse/v2"
	"github.com/tdewolff/parse/v2/buffer"
)

func benchmark(b *testing.B, mediatype string, sample string) {
	m := minify.Default
	in, err := ioutil.ReadFile(sample)
	if err != nil {
		panic(err)
	}
	b.Run(sample, func(b *testing.B) {
		out := make([]byte, 0, len(in))
		b.SetBytes(int64(len(in)))
		for i := 0; i < b.N; i++ {
			b.StopTimer()
			runtime.GC()
			r := buffer.NewReader(parse.Copy(in))
			w := buffer.NewWriter(out[:0])
			b.StartTimer()

			if err := m.Minify(mediatype, w, r); err != nil {
				b.Fatal(err)
			}
		}
	})
}

func BenchmarkCSS(b *testing.B) {
	var samples = []string{
		"sample_bootstrap.css",
		"sample_gumby.css",
		"sample_fontawesome.css",
		"sample_normalize.css",
	}
	for _, sample := range samples {
		benchmark(b, "text/css", sample)
	}
}

func BenchmarkHTML(b *testing.B) {
	var samples = []string{
		"sample_amazon.html",
		"sample_bbc.html",
		"sample_blogpost.html",
		"sample_es6.html",
		"sample_stackoverflow.html",
		"sample_wikipedia.html",
	}
	for _, sample := range samples {
		benchmark(b, "text/html", sample)
	}
}

func BenchmarkJS(b *testing.B) {
	var samples = []string{
		"sample_ace.js",
		"sample_dot.js",
		"sample_jquery.js",
		"sample_jqueryui.js",
		"sample_moment.js",
	}
	for _, sample := range samples {
		benchmark(b, "application/javascript", sample)
	}
}

func BenchmarkJSON(b *testing.B) {
	var samples = []string{
		"sample_large.json",
		"sample_testsuite.json",
		"sample_twitter.json",
	}
	for _, sample := range samples {
		benchmark(b, "application/json", sample)
	}
}

func BenchmarkSVG(b *testing.B) {
	var samples = []string{
		"sample_arctic.svg",
		"sample_gopher.svg",
		"sample_usa.svg",
		"sample_car.svg",
		"sample_tiger.svg",
	}
	for _, sample := range samples {
		benchmark(b, "image/svg+xml", sample)
	}
}

func BenchmarkXML(b *testing.B) {
	var samples = []string{
		"sample_books.xml",
		"sample_catalog.xml",
		"sample_omg.xml",
	}
	for _, sample := range samples {
		benchmark(b, "application/xml", sample)
	}
}
<|endoftext|>


<commit_before>package offheap

import (
	"fmt"
	"os"
	"syscall"

	"github.com/glycerine/gommap"
)

// The MmapMalloc struct represents either an anonymous, private
// region of memory (if path was "", or a memory mapped file if
// path was supplied to Malloc() at creation.
//
// Malloc() creates and returns an MmapMalloc struct, which can then
// be later Free()-ed. Malloc() calls request memory directly
// from the kernel via mmap(). Memory can optionally be backed
// by a file for simplicity/efficiency of saving to disk.
//
// For use when the Go GC overhead is too large, and you need to move
// the hash table off-heap.
//
type MmapMalloc struct {
	Path         string
	File         *os.File
	Fd           int
	FileBytesLen int64
	BytesAlloc   int64
	MMap         gommap.MMap // equiv to Mem, just avoids casts everywhere.
	Mem          []byte      // equiv to Mmap
}

// TruncateTo enlarges or shortens the file backing the
// memory map to be size newSize bytes. It only impacts
// the file underlying the mapping, not
// the mapping itself at this point.
func (mm *MmapMalloc) TruncateTo(newSize int64) {
	if mm.File == nil {
		panic("cannot call TruncateTo() on a non-file backed MmapMalloc.")
	}
	err := syscall.Ftruncate(int(mm.File.Fd()), newSize)
	if err != nil {
		panic(err)
	}
}

// Free eleases the memory allocation back to the OS by removing
// the (possibly anonymous and private) memroy mapped file that
// was backing it. Warning: any pointers still remaining will crash
// the program if dereferenced.
func (mm *MmapMalloc) Free() {
	if mm.File != nil {
		mm.File.Close()
	}
	err := mm.MMap.UnsafeUnmap()
	if err != nil {
		panic(err)
	}
}

// Malloc() creates a new memory region that is provided directly
// by OS via the mmap() call, and is thus not scanned by the Go
// garbage collector.
//
// If path is not empty then we memory map to the given path.
// Otherwise it is just like a call to malloc(): an anonymous memory allocation,
// outside the realm of the Go Garbage Collector.
// If numBytes is -1, then we take the size from the path file's size. Otherwise
// the file is expanded or truncated to be numBytes in size. If numBytes is -1
// then a path must be provided; otherwise we have no way of knowing the size
// to allocate, and the function will panic.
//
// The returned value's .Mem member holds a []byte pointing to the returned memory (as does .MMap, for use in other gommap calls).
//
func Malloc(numBytes int64, path string) *MmapMalloc {

	mm := MmapMalloc{
		Path: path,
	}

	flags := syscall.MAP_SHARED
	if path == "" {
		flags = syscall.MAP_ANON | syscall.MAP_PRIVATE
		mm.Fd = -1

		if numBytes < 0 {
			panic("numBytes was negative but path was also empty: don't know how much to allocate!")
		}

	} else {

		if dirExists(mm.Path) {
			panic(fmt.Sprintf("path '%s' already exists as a directory, so cannot be used as a memory mapped file.", mm.Path))
		}

		if !fileExists(mm.Path) {
			file, err := os.Create(mm.Path)
			if err != nil {
				panic(err)
			}
			mm.File = file
		} else {
			file, err := os.OpenFile(mm.Path, os.O_RDWR, 0777)
			if err != nil {
				panic(err)
			}
			mm.File = file
		}
		mm.Fd = int(mm.File.Fd())
	}

	sz := numBytes
	if path != "" {
		// file-backed memory
		if numBytes < 0 {

			var stat syscall.Stat_t
			if err := syscall.Fstat(mm.Fd, &stat); err != nil {
				panic(err)
			}
			sz = stat.Size

		} else {
			// set to the size requested
			err := syscall.Ftruncate(mm.Fd, numBytes)
			if err != nil {
				panic(err)
			}
		}
		mm.FileBytesLen = sz
	}
	// INVAR: sz holds non-negative length of memory/file.

	mm.BytesAlloc = sz

	prot := syscall.PROT_READ | syscall.PROT_WRITE

	vprintf("\n ------->> path = '%v',  mm.Fd = %v, with flags = %x, sz = %v,  prot = '%v'\n", path, mm.Fd, flags, sz, prot)

	var mmap []byte
	var err error
	if mm.Fd == -1 {

		flags = syscall.MAP_ANON | syscall.MAP_PRIVATE
		mmap, err = syscall.Mmap(-1, 0, int(sz), prot, flags)

	} else {

		flags = syscall.MAP_SHARED
		mmap, err = syscall.Mmap(mm.Fd, 0, int(sz), prot, flags)
	}
	if err != nil {
		panic(err)
	}

	// duplicate member to avoid casts all over the place.
	mm.MMap = mmap
	mm.Mem = mmap

	return &mm
}

// BlockUntilSync() returns only once the file is synced to disk.
func (mm *MmapMalloc) BlockUntilSync() {
	mm.MMap.Sync(gommap.MS_SYNC)
}

// BackgroundSync() schedules a sync to disk, but may return before it is done.
// Without a call to either BackgroundSync() or BlockUntilSync(), there
// is no guarantee that file has ever been written to disk at any point before
// the munmap() call that happens during Free(). See the man pages msync(2)
// and mmap(2) for details.
func (mm *MmapMalloc) BackgroundSync() {
	mm.MMap.Sync(gommap.MS_ASYNC)
}

// Growmap grows a memory mapping
func Growmap(oldmap *MmapMalloc, newSize int64) (newmap *MmapMalloc, err error) {

	if oldmap.Path == "" || oldmap.Fd <= 0 {
		return nil, fmt.Errorf("oldmap must be mapping to " +
			"actual file, so we can grow it")
	}
	if newSize <= oldmap.BytesAlloc || newSize <= oldmap.FileBytesLen {
		return nil, fmt.Errorf("mapping in Growmap must be larger than before")
	}

	newmap = &MmapMalloc{}
	newmap.Path = oldmap.Path
	newmap.Fd = oldmap.Fd

	// set to the size requested
	err = syscall.Ftruncate(newmap.Fd, newSize)
	if err != nil {
		panic(err)
		return nil, fmt.Errorf("syscall.Ftruncate to grow %v -> %v"+
			" returned error: '%v'", oldmap.BytesAlloc, newSize, err)
	}
	newmap.FileBytesLen = newSize
	newmap.BytesAlloc = newSize

	prot := syscall.PROT_READ | syscall.PROT_WRITE
	flags := syscall.MAP_SHARED

	p("\n ------->> path = '%v',  newmap.Fd = %v, with flags = %x, sz = %v,  prot = '%v'\n", newmap.Path, newmap.Fd, flags, newSize, prot)

	mmap, err := syscall.Mmap(newmap.Fd, 0, int(newSize), prot, flags)
	if err != nil {
		panic(err)
		return nil, fmt.Errorf("syscall.Mmap returned error: '%v'", err)
	}

	// duplicate member to avoid casts all over the place.
	newmap.MMap = mmap
	newmap.Mem = mmap

	return newmap, nil
}
<commit_msg>typo test of git flow<commit_after>package offheap

import (
	"fmt"
	"os"
	"syscall"

	"github.com/glycerine/gommap"
)

// The MmapMalloc struct represents either an anonymous, private
// region of memory (if path was "", or a memory mapped file if
// path was supplied to Malloc() at creation.
//
// Malloc() creates and returns an MmapMalloc struct, which can then
// be later Free()-ed. Malloc() calls request memory directly
// from the kernel via mmap(). Memory can optionally be backed
// by a file for simplicity/efficiency of saving to disk.
//
// For use when the Go GC overhead is too large, and you need to move
// the hash table off-heap.
//
type MmapMalloc struct {
	Path         string
	File         *os.File
	Fd           int
	FileBytesLen int64
	BytesAlloc   int64
	MMap         gommap.MMap // equiv to Mem, just avoids casts everywhere.
	Mem          []byte      // equiv to Mmap
}

// TruncateTo enlarges or shortens the file backing the
// memory map to be size newSize bytes. It only impacts
// the file underlying the mapping, not
// the mapping itself at this point.
func (mm *MmapMalloc) TruncateTo(newSize int64) {
	if mm.File == nil {
		panic("cannot call TruncateTo() on a non-file backed MmapMalloc.")
	}
	err := syscall.Ftruncate(int(mm.File.Fd()), newSize)
	if err != nil {
		panic(err)
	}
}

// Free releases the memory allocation back to the OS by removing
// the (possibly anonymous and private) memroy mapped file that
// was backing it. Warning: any pointers still remaining will crash
// the program if dereferenced.
func (mm *MmapMalloc) Free() {
	if mm.File != nil {
		mm.File.Close()
	}
	err := mm.MMap.UnsafeUnmap()
	if err != nil {
		panic(err)
	}
}

// Malloc() creates a new memory region that is provided directly
// by OS via the mmap() call, and is thus not scanned by the Go
// garbage collector.
//
// If path is not empty then we memory map to the given path.
// Otherwise it is just like a call to malloc(): an anonymous memory allocation,
// outside the realm of the Go Garbage Collector.
// If numBytes is -1, then we take the size from the path file's size. Otherwise
// the file is expanded or truncated to be numBytes in size. If numBytes is -1
// then a path must be provided; otherwise we have no way of knowing the size
// to allocate, and the function will panic.
//
// The returned value's .Mem member holds a []byte pointing to the returned memory (as does .MMap, for use in other gommap calls).
//
func Malloc(numBytes int64, path string) *MmapMalloc {

	mm := MmapMalloc{
		Path: path,
	}

	flags := syscall.MAP_SHARED
	if path == "" {
		flags = syscall.MAP_ANON | syscall.MAP_PRIVATE
		mm.Fd = -1

		if numBytes < 0 {
			panic("numBytes was negative but path was also empty: don't know how much to allocate!")
		}

	} else {

		if dirExists(mm.Path) {
			panic(fmt.Sprintf("path '%s' already exists as a directory, so cannot be used as a memory mapped file.", mm.Path))
		}

		if !fileExists(mm.Path) {
			file, err := os.Create(mm.Path)
			if err != nil {
				panic(err)
			}
			mm.File = file
		} else {
			file, err := os.OpenFile(mm.Path, os.O_RDWR, 0777)
			if err != nil {
				panic(err)
			}
			mm.File = file
		}
		mm.Fd = int(mm.File.Fd())
	}

	sz := numBytes
	if path != "" {
		// file-backed memory
		if numBytes < 0 {

			var stat syscall.Stat_t
			if err := syscall.Fstat(mm.Fd, &stat); err != nil {
				panic(err)
			}
			sz = stat.Size

		} else {
			// set to the size requested
			err := syscall.Ftruncate(mm.Fd, numBytes)
			if err != nil {
				panic(err)
			}
		}
		mm.FileBytesLen = sz
	}
	// INVAR: sz holds non-negative length of memory/file.

	mm.BytesAlloc = sz

	prot := syscall.PROT_READ | syscall.PROT_WRITE

	vprintf("\n ------->> path = '%v',  mm.Fd = %v, with flags = %x, sz = %v,  prot = '%v'\n", path, mm.Fd, flags, sz, prot)

	var mmap []byte
	var err error
	if mm.Fd == -1 {

		flags = syscall.MAP_ANON | syscall.MAP_PRIVATE
		mmap, err = syscall.Mmap(-1, 0, int(sz), prot, flags)

	} else {

		flags = syscall.MAP_SHARED
		mmap, err = syscall.Mmap(mm.Fd, 0, int(sz), prot, flags)
	}
	if err != nil {
		panic(err)
	}

	// duplicate member to avoid casts all over the place.
	mm.MMap = mmap
	mm.Mem = mmap

	return &mm
}

// BlockUntilSync() returns only once the file is synced to disk.
func (mm *MmapMalloc) BlockUntilSync() {
	mm.MMap.Sync(gommap.MS_SYNC)
}

// BackgroundSync() schedules a sync to disk, but may return before it is done.
// Without a call to either BackgroundSync() or BlockUntilSync(), there
// is no guarantee that file has ever been written to disk at any point before
// the munmap() call that happens during Free(). See the man pages msync(2)
// and mmap(2) for details.
func (mm *MmapMalloc) BackgroundSync() {
	mm.MMap.Sync(gommap.MS_ASYNC)
}

// Growmap grows a memory mapping
func Growmap(oldmap *MmapMalloc, newSize int64) (newmap *MmapMalloc, err error) {

	if oldmap.Path == "" || oldmap.Fd <= 0 {
		return nil, fmt.Errorf("oldmap must be mapping to " +
			"actual file, so we can grow it")
	}
	if newSize <= oldmap.BytesAlloc || newSize <= oldmap.FileBytesLen {
		return nil, fmt.Errorf("mapping in Growmap must be larger than before")
	}

	newmap = &MmapMalloc{}
	newmap.Path = oldmap.Path
	newmap.Fd = oldmap.Fd

	// set to the size requested
	err = syscall.Ftruncate(newmap.Fd, newSize)
	if err != nil {
		panic(err)
		return nil, fmt.Errorf("syscall.Ftruncate to grow %v -> %v"+
			" returned error: '%v'", oldmap.BytesAlloc, newSize, err)
	}
	newmap.FileBytesLen = newSize
	newmap.BytesAlloc = newSize

	prot := syscall.PROT_READ | syscall.PROT_WRITE
	flags := syscall.MAP_SHARED

	p("\n ------->> path = '%v',  newmap.Fd = %v, with flags = %x, sz = %v,  prot = '%v'\n", newmap.Path, newmap.Fd, flags, newSize, prot)

	mmap, err := syscall.Mmap(newmap.Fd, 0, int(newSize), prot, flags)
	if err != nil {
		panic(err)
		return nil, fmt.Errorf("syscall.Mmap returned error: '%v'", err)
	}

	// duplicate member to avoid casts all over the place.
	newmap.MMap = mmap
	newmap.Mem = mmap

	return newmap, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"strconv"
	"strings"
)

func processMbeans(e *Exporter, coreName string, data io.Reader) []error {
	mBeansData := &MBeansData{}
	errors := []error{}
	if err := json.NewDecoder(data).Decode(mBeansData); err != nil {
		errors = append(errors, fmt.Errorf("Failed to unmarshal mbeansdata JSON into struct: %v", err))
		return errors
	}

	var coreMetrics map[string]Core
	if err := json.Unmarshal(findMBeansData(mBeansData.SolrMbeans, "CORE"), &coreMetrics); err != nil {
		errors = append(errors, fmt.Errorf("Failed to unmarshal mbeans core metrics JSON into struct: %v", err))
		return errors
	}

	for name, metrics := range coreMetrics {
		if strings.Contains(name, "@") {
			continue
		}

		e.gaugeCore["deleted_docs"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.DeletedDocs))
		e.gaugeCore["max_docs"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.MaxDoc))
		e.gaugeCore["num_docs"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.NumDocs))
	}

	b := bytes.Replace(findMBeansData(mBeansData.SolrMbeans, "QUERY"), []byte(":\"NaN\""), []byte(":0.0"), -1)
	var queryMetrics map[string]QueryHandler
	if err := json.Unmarshal(b, &queryMetrics); err != nil {
		errors = append(errors, fmt.Errorf("Failed to unmarshal mbeans query metrics JSON into struct: %v, json : %s", err, b))
		return errors
	}

	for name, metrics := range queryMetrics {
		if strings.Contains(name, "@") || strings.Contains(name, "/admin") || strings.Contains(name, "/debug/dump") || strings.Contains(name, "/schema") || strings.Contains(name, "org.apache.solr.handler.admin") {
			continue
		}

		var FiveminRateRequestsPerSecond, One5minRateRequestsPerSecond float64
		if metrics.Stats.One5minRateReqsPerSecond == nil && metrics.Stats.FiveMinRateReqsPerSecond == nil {
			FiveminRateRequestsPerSecond = float64(metrics.Stats.FiveminRateRequestsPerSecond)
			One5minRateRequestsPerSecond = float64(metrics.Stats.One5minRateRequestsPerSecond)
		} else {
			FiveminRateRequestsPerSecond = float64(*metrics.Stats.FiveMinRateReqsPerSecond)
			One5minRateRequestsPerSecond = float64(*metrics.Stats.One5minRateReqsPerSecond)
		}

		e.gaugeQuery["15min_rate_reqs_per_second"].WithLabelValues(coreName, name, metrics.Class).Set(One5minRateRequestsPerSecond)
		e.gaugeQuery["5min_rate_reqs_per_second"].WithLabelValues(coreName, name, metrics.Class).Set(FiveminRateRequestsPerSecond)
		e.gaugeQuery["75th_pc_request_time"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Seven5thPcRequestTime))
		e.gaugeQuery["95th_pc_request_time"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Nine5thPcRequestTime))
		e.gaugeQuery["99th_pc_request_time"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Nine9thPcRequestTime))
		e.gaugeQuery["999th_pc_request_time"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Nine99thPcRequestTime))
		e.gaugeQuery["avg_requests_per_second"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.AvgRequestsPerSecond))
		e.gaugeQuery["avg_time_per_request"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.AvgTimePerRequest))
		e.gaugeQuery["errors"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Errors))
		e.gaugeQuery["handler_start"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.HandlerStart))
		e.gaugeQuery["median_request_time"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.MedianRequestTime))
		e.gaugeQuery["requests"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Requests))
		e.gaugeQuery["timeouts"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Timeouts))
		e.gaugeQuery["total_time"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.TotalTime))
	}

	var updateMetrics map[string]UpdateHandler
	if err := json.Unmarshal(findMBeansData(mBeansData.SolrMbeans, "UPDATE"), &updateMetrics); err != nil {
		errors = append(errors, fmt.Errorf("Failed to unmarshal mbeans update metrics JSON into struct: %v", err))
		return errors
	}

	for name, metrics := range updateMetrics {
		if strings.Contains(name, "@") || strings.HasPrefix(name, "/") {
			continue
		}
		var autoCommitMaxTime int
		if len(metrics.Stats.AutocommitMaxTime) > 2 {
			autoCommitMaxTime, _ = strconv.Atoi(metrics.Stats.AutocommitMaxTime[:len(metrics.Stats.AutocommitMaxTime)-2])
		}
		e.gaugeUpdate["adds"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Adds))
		e.gaugeUpdate["autocommit_max_docs"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.AutocommitMaxDocs))
		e.gaugeUpdate["autocommit_max_time"].WithLabelValues(coreName, name, metrics.Class).Set(float64(autoCommitMaxTime))
		e.gaugeUpdate["autocommits"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Autocommits))
		e.gaugeUpdate["commits"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Commits))
		e.gaugeUpdate["cumulative_adds"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.CumulativeAdds))
		e.gaugeUpdate["cumulative_deletes_by_id"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.CumulativeDeletesByID))
		e.gaugeUpdate["cumulative_deletes_by_query"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.CumulativeDeletesByQuery))
		e.gaugeUpdate["cumulative_errors"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.CumulativeErrors))
		e.gaugeUpdate["deletes_by_id"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.DeletesByID))
		e.gaugeUpdate["deletes_by_query"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.DeletesByQuery))
		e.gaugeUpdate["docs_pending"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.DocsPending))
		e.gaugeUpdate["errors"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Errors))
		e.gaugeUpdate["expunge_deletes"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.ExpungeDeletes))
		e.gaugeUpdate["optimizes"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Optimizes))
		e.gaugeUpdate["rollbacks"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Rollbacks))
		e.gaugeUpdate["soft_autocommits"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.SoftAutocommits))
	}

	// Try to decode solr > v5 cache metrics
	cacheData := findMBeansData(mBeansData.SolrMbeans, "CACHE")
	b = bytes.Replace(cacheData, []byte(":\"NaN\""), []byte(":0.0"), -1)

	// mbeans metric names change in Solr 7+
	mbeanerrs := handleCacheMbeanslt7(b, e, coreName)
	for _, e := range mbeanerrs {
		errors = append(errors, e)
	}
	return errors
}

func handleCacheMbeanslt7(data []byte, e *Exporter, coreName string) []error {
	var cacheMetrics map[string]Cache
	var errors = []error{}
	if err := json.Unmarshal(data, &cacheMetrics); err != nil {
		errors = append(errors, fmt.Errorf("Failed to unmarshal mbeans cache metrics JSON into struct (core : %s): %v, json : %s", coreName, err, data))
	} else {
		for name, metrics := range cacheMetrics {
			if metrics.Class == "org.apache.solr.search.SolrFieldCacheMBean" {
				continue
			}
			hitratio, err := strconv.ParseFloat(string(metrics.Stats.Hitratio), 64)
			if err != nil {
				errors = append(errors, fmt.Errorf("Fail to convert Hitratio in float: %v", err))
			}
			cumulativeHitratio, err := strconv.ParseFloat(string(metrics.Stats.CumulativeHitratio), 64)
			if err != nil {
				errors = append(errors, fmt.Errorf("Fail to convert Cumulative Hitratio in float: %v", err))
			}
			e.gaugeCache["cumulative_evictions"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.CumulativeEvictions))
			e.gaugeCache["cumulative_hitratio"].WithLabelValues(coreName, name, metrics.Class).Set(cumulativeHitratio)
			e.gaugeCache["cumulative_hits"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.CumulativeHits))
			e.gaugeCache["cumulative_inserts"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.CumulativeInserts))
			e.gaugeCache["cumulative_lookups"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.CumulativeLookups))
			e.gaugeCache["evictions"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Evictions))
			e.gaugeCache["hitratio"].WithLabelValues(coreName, name, metrics.Class).Set(hitratio)
			e.gaugeCache["hits"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Hits))
			e.gaugeCache["inserts"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Inserts))
			e.gaugeCache["lookups"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Lookups))
			e.gaugeCache["size"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Size))
			e.gaugeCache["warmup_time"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.WarmupTime))
		}
	}
	return errors
}
<commit_msg>Handle correctly mbeans solr7 stats<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"strconv"
	"strings"
)

func processMbeans(e *Exporter, coreName string, data io.Reader) []error {
	mBeansData := &MBeansData{}
	errors := []error{}
	if err := json.NewDecoder(data).Decode(mBeansData); err != nil {
		errors = append(errors, fmt.Errorf("Failed to unmarshal mbeansdata JSON into struct: %v", err))
		return errors
	}

	var coreMetrics map[string]Core
	if err := json.Unmarshal(findMBeansData(mBeansData.SolrMbeans, "CORE"), &coreMetrics); err != nil {
		errors = append(errors, fmt.Errorf("Failed to unmarshal mbeans core metrics JSON into struct: %v", err))
		return errors
	}

	for name, metrics := range coreMetrics {
		if strings.Contains(name, "@") {
			continue
		}

		e.gaugeCore["deleted_docs"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.DeletedDocs))
		e.gaugeCore["max_docs"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.MaxDoc))
		e.gaugeCore["num_docs"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.NumDocs))
	}

	b := bytes.Replace(findMBeansData(mBeansData.SolrMbeans, "QUERY"), []byte(":\"NaN\""), []byte(":0.0"), -1)
	var queryMetrics map[string]QueryHandler
	if err := json.Unmarshal(b, &queryMetrics); err != nil {
		errors = append(errors, fmt.Errorf("Failed to unmarshal mbeans query metrics JSON into struct: %v, json : %s", err, b))
		return errors
	}

	for name, metrics := range queryMetrics {
		if strings.Contains(name, "@") || strings.Contains(name, "/admin") || strings.Contains(name, "/debug/dump") || strings.Contains(name, "/schema") || strings.Contains(name, "org.apache.solr.handler.admin") {
			continue
		}

		var FiveminRateRequestsPerSecond, One5minRateRequestsPerSecond float64
		if metrics.Stats.One5minRateReqsPerSecond == nil && metrics.Stats.FiveMinRateReqsPerSecond == nil {
			FiveminRateRequestsPerSecond = float64(metrics.Stats.FiveminRateRequestsPerSecond)
			One5minRateRequestsPerSecond = float64(metrics.Stats.One5minRateRequestsPerSecond)
		} else {
			FiveminRateRequestsPerSecond = float64(*metrics.Stats.FiveMinRateReqsPerSecond)
			One5minRateRequestsPerSecond = float64(*metrics.Stats.One5minRateReqsPerSecond)
		}

		e.gaugeQuery["15min_rate_reqs_per_second"].WithLabelValues(coreName, name, metrics.Class).Set(One5minRateRequestsPerSecond)
		e.gaugeQuery["5min_rate_reqs_per_second"].WithLabelValues(coreName, name, metrics.Class).Set(FiveminRateRequestsPerSecond)
		e.gaugeQuery["75th_pc_request_time"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Seven5thPcRequestTime))
		e.gaugeQuery["95th_pc_request_time"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Nine5thPcRequestTime))
		e.gaugeQuery["99th_pc_request_time"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Nine9thPcRequestTime))
		e.gaugeQuery["999th_pc_request_time"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Nine99thPcRequestTime))
		e.gaugeQuery["avg_requests_per_second"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.AvgRequestsPerSecond))
		e.gaugeQuery["avg_time_per_request"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.AvgTimePerRequest))
		e.gaugeQuery["errors"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Errors))
		e.gaugeQuery["handler_start"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.HandlerStart))
		e.gaugeQuery["median_request_time"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.MedianRequestTime))
		e.gaugeQuery["requests"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Requests))
		e.gaugeQuery["timeouts"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Timeouts))
		e.gaugeQuery["total_time"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.TotalTime))
	}

	var updateMetrics map[string]UpdateHandler
	if err := json.Unmarshal(findMBeansData(mBeansData.SolrMbeans, "UPDATE"), &updateMetrics); err != nil {
		errors = append(errors, fmt.Errorf("Failed to unmarshal mbeans update metrics JSON into struct: %v", err))
		return errors
	}

	for name, metrics := range updateMetrics {
		if strings.Contains(name, "@") || strings.HasPrefix(name, "/") {
			continue
		}
		var autoCommitMaxTime int
		if len(metrics.Stats.AutocommitMaxTime) > 2 {
			autoCommitMaxTime, _ = strconv.Atoi(metrics.Stats.AutocommitMaxTime[:len(metrics.Stats.AutocommitMaxTime)-2])
		}
		e.gaugeUpdate["adds"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Adds))
		e.gaugeUpdate["autocommit_max_docs"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.AutocommitMaxDocs))
		e.gaugeUpdate["autocommit_max_time"].WithLabelValues(coreName, name, metrics.Class).Set(float64(autoCommitMaxTime))
		e.gaugeUpdate["autocommits"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Autocommits))
		e.gaugeUpdate["commits"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Commits))
		e.gaugeUpdate["cumulative_adds"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.CumulativeAdds))
		e.gaugeUpdate["cumulative_deletes_by_id"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.CumulativeDeletesByID))
		e.gaugeUpdate["cumulative_deletes_by_query"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.CumulativeDeletesByQuery))
		e.gaugeUpdate["cumulative_errors"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.CumulativeErrors))
		e.gaugeUpdate["deletes_by_id"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.DeletesByID))
		e.gaugeUpdate["deletes_by_query"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.DeletesByQuery))
		e.gaugeUpdate["docs_pending"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.DocsPending))
		e.gaugeUpdate["errors"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Errors))
		e.gaugeUpdate["expunge_deletes"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.ExpungeDeletes))
		e.gaugeUpdate["optimizes"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Optimizes))
		e.gaugeUpdate["rollbacks"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Rollbacks))
		e.gaugeUpdate["soft_autocommits"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.SoftAutocommits))
	}

	// Try to decode solr > v5 cache metrics
	cacheData := findMBeansData(mBeansData.SolrMbeans, "CACHE")
	b = bytes.Replace(cacheData, []byte(":\"NaN\""), []byte(":0.0"), -1)
	b = bytes.Replace(b, []byte("CACHE.searcher.perSegFilter."), []byte(""), -1)
	b = bytes.Replace(b, []byte("CACHE.searcher.queryResultCache."), []byte(""), -1)
	b = bytes.Replace(b, []byte("CACHE.searcher.fieldValueCache."), []byte(""), -1)
	b = bytes.Replace(b, []byte("CACHE.searcher.filterCache."), []byte(""), -1)
	b = bytes.Replace(b, []byte("CACHE.searcher.documentCache."), []byte(""), -1)
	fmt.Println(string(b))

	// mbeans metric names change in Solr 7+
	mbeanerrs := handleCacheMbeanslt7(b, e, coreName)
	for _, e := range mbeanerrs {
		errors = append(errors, e)
	}
	return errors
}

func handleCacheMbeanslt7(data []byte, e *Exporter, coreName string) []error {
	var cacheMetrics map[string]Cache
	var errors = []error{}
	if err := json.Unmarshal(data, &cacheMetrics); err != nil {
		errors = append(errors, fmt.Errorf("Failed to unmarshal mbeans cache metrics JSON into struct (core : %s): %v, json : %s", coreName, err, data))
	} else {
		for name, metrics := range cacheMetrics {
			if metrics.Class == "org.apache.solr.search.SolrFieldCacheMBean" || metrics.Class == "org.apache.solr.search.SolrFieldCacheBean" {
				continue
			}
			hitratio, err := strconv.ParseFloat(string(metrics.Stats.Hitratio), 64)
			if err != nil {
				errors = append(errors, fmt.Errorf("Fail to convert Hitratio in float: %v", err))
			}
			cumulativeHitratio, err := strconv.ParseFloat(string(metrics.Stats.CumulativeHitratio), 64)
			if err != nil {
				errors = append(errors, fmt.Errorf("Fail to convert Cumulative Hitratio in float: %v", err))
			}
			e.gaugeCache["cumulative_evictions"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.CumulativeEvictions))
			e.gaugeCache["cumulative_hitratio"].WithLabelValues(coreName, name, metrics.Class).Set(cumulativeHitratio)
			e.gaugeCache["cumulative_hits"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.CumulativeHits))
			e.gaugeCache["cumulative_inserts"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.CumulativeInserts))
			e.gaugeCache["cumulative_lookups"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.CumulativeLookups))
			e.gaugeCache["evictions"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Evictions))
			e.gaugeCache["hitratio"].WithLabelValues(coreName, name, metrics.Class).Set(hitratio)
			e.gaugeCache["hits"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Hits))
			e.gaugeCache["inserts"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Inserts))
			e.gaugeCache["lookups"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Lookups))
			e.gaugeCache["size"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.Size))
			e.gaugeCache["warmup_time"].WithLabelValues(coreName, name, metrics.Class).Set(float64(metrics.Stats.WarmupTime))
		}
	}
	return errors
}
<|endoftext|>


<commit_before>package sql_test

import (
	"bytes"
	"testing"

	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/testutils"
	"github.com/cockroachdb/cockroach/testutils/storageutils"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/leaktest"
)

func checkCounterEQ(t *testing.T, s *testServer, key string, e int64) {
	if a := s.MustGetSQLCounter(key); !(a == e) {
		t.Error(util.ErrorfSkipFrames(1, "stat %s: actual %d != expected %d", key, a, e))
	}
}

func checkCounterGE(t *testing.T, s *testServer, key string, e int64) {
	if a := s.MustGetSQLCounter(key); !(a >= e) {
		t.Error(util.ErrorfSkipFrames(1, "stat %s: expected: actual %d >= %d", key, a, e))
	}
}

func TestQueryCounts(t *testing.T) {
	defer leaktest.AfterTest(t)()
	s, sqlDB, _ := setup(t)
	defer cleanup(s, sqlDB)

	var testcases = []struct {
		query            string
		txnBeginCount    int64
		selectCount      int64
		updateCount      int64
		insertCount      int64
		deleteCount      int64
		ddlCount         int64
		miscCount        int64
		txnCommitCount   int64
		txnRollbackCount int64
	}{
		{"", 0, 0, 0, 0, 0, 0, 0, 0, 0},
		{"BEGIN; END", 1, 0, 0, 0, 0, 0, 0, 1, 0},
		{"SELECT 1", 1, 1, 0, 0, 0, 0, 0, 1, 0},
		{"CREATE DATABASE mt", 1, 1, 0, 0, 0, 1, 0, 1, 0},
		{"CREATE TABLE mt.n (num INTEGER)", 1, 1, 0, 0, 0, 2, 0, 1, 0},
		{"INSERT INTO mt.n VALUES (3)", 1, 1, 0, 1, 0, 2, 0, 1, 0},
		{"UPDATE mt.n SET num = num + 1", 1, 1, 1, 1, 0, 2, 0, 1, 0},
		{"DELETE FROM mt.n", 1, 1, 1, 1, 1, 2, 0, 1, 0},
		{"ALTER TABLE mt.n ADD COLUMN num2 INTEGER", 1, 1, 1, 1, 1, 3, 0, 1, 0},
		{"EXPLAIN SELECT * FROM mt.n", 1, 1, 1, 1, 1, 3, 1, 1, 0},
		{"BEGIN; UPDATE mt.n SET num = num + 1; END", 2, 1, 2, 1, 1, 3, 1, 2, 0},
		{"SELECT * FROM mt.n; SELECT * FROM mt.n; SELECT * FROM mt.n", 2, 4, 2, 1, 1, 3, 1, 2, 0},
		{"DROP TABLE mt.n", 2, 4, 2, 1, 1, 4, 1, 2, 0},
		{"SET database = system", 2, 4, 2, 1, 1, 4, 2, 2, 0},
	}

	for _, tc := range testcases {
		if tc.query != "" {
			if _, err := sqlDB.Exec(tc.query); err != nil {
				t.Fatalf("unexpected error executing '%s': %s'", tc.query, err)
			}
		}

		// Force metric snapshot refresh.
		if err := s.WriteSummaries(); err != nil {
			t.Fatal(err)
		}

		checkCounterEQ(t, s, "txn.begin.count", tc.txnBeginCount)
		checkCounterEQ(t, s, "select.count", tc.selectCount)
		checkCounterEQ(t, s, "update.count", tc.updateCount)
		checkCounterEQ(t, s, "insert.count", tc.insertCount)
		checkCounterEQ(t, s, "delete.count", tc.deleteCount)
		checkCounterEQ(t, s, "ddl.count", tc.ddlCount)
		checkCounterEQ(t, s, "misc.count", tc.miscCount)
		checkCounterEQ(t, s, "txn.commit.count", tc.txnCommitCount)
		checkCounterEQ(t, s, "txn.rollback.count", tc.txnRollbackCount)
		checkCounterEQ(t, s, "txn.abort.count", 0)

		// Everything after this query will also fail, so quit now to avoid deluge of errors.
		if t.Failed() {
			t.FailNow()
		}
	}
}

func TestAbortCountConflictingWrites(t *testing.T) {
	defer leaktest.AfterTest(t)()

	ctx, cmdFilters := createTestServerContext()
	s, sqlDB, _ := setupWithContext(t, ctx)
	defer cleanup(s, sqlDB)

	if _, err := sqlDB.Exec("CREATE DATABASE db"); err != nil {
		t.Fatal(err)
	}
	if _, err := sqlDB.Exec("CREATE TABLE db.t (k TEXT PRIMARY KEY, v TEXT)"); err != nil {
		t.Fatal(err)
	}

	// Inject errors on the INSERT below.
	restarted := false
	cmdFilters.AppendFilter(func(args storageutils.FilterArgs) *roachpb.Error {
		switch req := args.Req.(type) {
		// SQL INSERT generates ConditionalPuts for unique indexes (such as the PK).
		case *roachpb.ConditionalPutRequest:
			if bytes.Contains(req.Value.RawBytes, []byte("marker")) && !restarted {
				restarted = true
				return roachpb.NewError(roachpb.NewTransactionAbortedError())
			}
		}
		return nil
	}, false)

	txn, err := sqlDB.Begin()
	if err != nil {
		t.Fatal(err)
	}
	_, err = txn.Exec("INSERT INTO db.t VALUES ('key', 'marker')")
	if !testutils.IsError(err, "aborted") {
		t.Fatal(err)
	}

	if err = txn.Rollback(); err != nil {
		t.Fatal(err)
	}

	checkCounterEQ(t, s, "txn.abort.count", 1)
	checkCounterEQ(t, s, "txn.begin.count", 1)
	checkCounterEQ(t, s, "txn.rollback.count", 0)
	checkCounterEQ(t, s, "txn.commit.count", 0)
	checkCounterEQ(t, s, "insert.count", 1)
}

// TestErrorDuringTransaction tests that the transaction abort count goes up when a query
// results in an error during a txn.
func TestAbortCountErrorDuringTransaction(t *testing.T) {
	defer leaktest.AfterTest(t)()
	s, sqlDB, _ := setup(t)
	defer cleanup(s, sqlDB)

	txn, err := sqlDB.Begin()
	if err != nil {
		t.Fatal(err)
	}

	if _, err := txn.Query("SELECT * FROM i_do.not_exist"); err == nil {
		t.Fatal("Expected an error but didn't get one")
	}

	checkCounterEQ(t, s, "txn.abort.count", 1)
	checkCounterEQ(t, s, "txn.begin.count", 1)
	checkCounterEQ(t, s, "select.count", 1)
}
<commit_msg>sql: use correct error creation fun in test<commit_after>package sql_test

import (
	"bytes"
	"testing"

	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/testutils"
	"github.com/cockroachdb/cockroach/testutils/storageutils"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/leaktest"
)

func checkCounterEQ(t *testing.T, s *testServer, key string, e int64) {
	if a := s.MustGetSQLCounter(key); !(a == e) {
		t.Error(util.ErrorfSkipFrames(1, "stat %s: actual %d != expected %d", key, a, e))
	}
}

func checkCounterGE(t *testing.T, s *testServer, key string, e int64) {
	if a := s.MustGetSQLCounter(key); !(a >= e) {
		t.Error(util.ErrorfSkipFrames(1, "stat %s: expected: actual %d >= %d", key, a, e))
	}
}

func TestQueryCounts(t *testing.T) {
	defer leaktest.AfterTest(t)()
	s, sqlDB, _ := setup(t)
	defer cleanup(s, sqlDB)

	var testcases = []struct {
		query            string
		txnBeginCount    int64
		selectCount      int64
		updateCount      int64
		insertCount      int64
		deleteCount      int64
		ddlCount         int64
		miscCount        int64
		txnCommitCount   int64
		txnRollbackCount int64
	}{
		{"", 0, 0, 0, 0, 0, 0, 0, 0, 0},
		{"BEGIN; END", 1, 0, 0, 0, 0, 0, 0, 1, 0},
		{"SELECT 1", 1, 1, 0, 0, 0, 0, 0, 1, 0},
		{"CREATE DATABASE mt", 1, 1, 0, 0, 0, 1, 0, 1, 0},
		{"CREATE TABLE mt.n (num INTEGER)", 1, 1, 0, 0, 0, 2, 0, 1, 0},
		{"INSERT INTO mt.n VALUES (3)", 1, 1, 0, 1, 0, 2, 0, 1, 0},
		{"UPDATE mt.n SET num = num + 1", 1, 1, 1, 1, 0, 2, 0, 1, 0},
		{"DELETE FROM mt.n", 1, 1, 1, 1, 1, 2, 0, 1, 0},
		{"ALTER TABLE mt.n ADD COLUMN num2 INTEGER", 1, 1, 1, 1, 1, 3, 0, 1, 0},
		{"EXPLAIN SELECT * FROM mt.n", 1, 1, 1, 1, 1, 3, 1, 1, 0},
		{"BEGIN; UPDATE mt.n SET num = num + 1; END", 2, 1, 2, 1, 1, 3, 1, 2, 0},
		{"SELECT * FROM mt.n; SELECT * FROM mt.n; SELECT * FROM mt.n", 2, 4, 2, 1, 1, 3, 1, 2, 0},
		{"DROP TABLE mt.n", 2, 4, 2, 1, 1, 4, 1, 2, 0},
		{"SET database = system", 2, 4, 2, 1, 1, 4, 2, 2, 0},
	}

	for _, tc := range testcases {
		if tc.query != "" {
			if _, err := sqlDB.Exec(tc.query); err != nil {
				t.Fatalf("unexpected error executing '%s': %s'", tc.query, err)
			}
		}

		// Force metric snapshot refresh.
		if err := s.WriteSummaries(); err != nil {
			t.Fatal(err)
		}

		checkCounterEQ(t, s, "txn.begin.count", tc.txnBeginCount)
		checkCounterEQ(t, s, "select.count", tc.selectCount)
		checkCounterEQ(t, s, "update.count", tc.updateCount)
		checkCounterEQ(t, s, "insert.count", tc.insertCount)
		checkCounterEQ(t, s, "delete.count", tc.deleteCount)
		checkCounterEQ(t, s, "ddl.count", tc.ddlCount)
		checkCounterEQ(t, s, "misc.count", tc.miscCount)
		checkCounterEQ(t, s, "txn.commit.count", tc.txnCommitCount)
		checkCounterEQ(t, s, "txn.rollback.count", tc.txnRollbackCount)
		checkCounterEQ(t, s, "txn.abort.count", 0)

		// Everything after this query will also fail, so quit now to avoid deluge of errors.
		if t.Failed() {
			t.FailNow()
		}
	}
}

func TestAbortCountConflictingWrites(t *testing.T) {
	defer leaktest.AfterTest(t)()

	ctx, cmdFilters := createTestServerContext()
	s, sqlDB, _ := setupWithContext(t, ctx)
	defer cleanup(s, sqlDB)

	if _, err := sqlDB.Exec("CREATE DATABASE db"); err != nil {
		t.Fatal(err)
	}
	if _, err := sqlDB.Exec("CREATE TABLE db.t (k TEXT PRIMARY KEY, v TEXT)"); err != nil {
		t.Fatal(err)
	}

	// Inject errors on the INSERT below.
	restarted := false
	cmdFilters.AppendFilter(func(args storageutils.FilterArgs) *roachpb.Error {
		switch req := args.Req.(type) {
		// SQL INSERT generates ConditionalPuts for unique indexes (such as the PK).
		case *roachpb.ConditionalPutRequest:
			if bytes.Contains(req.Value.RawBytes, []byte("marker")) && !restarted {
				restarted = true
				return roachpb.NewErrorWithTxn(
					roachpb.NewTransactionAbortedError(), args.Hdr.Txn)
			}
		}
		return nil
	}, false)

	txn, err := sqlDB.Begin()
	if err != nil {
		t.Fatal(err)
	}
	_, err = txn.Exec("INSERT INTO db.t VALUES ('key', 'marker')")
	if !testutils.IsError(err, "aborted") {
		t.Fatal(err)
	}

	if err = txn.Rollback(); err != nil {
		t.Fatal(err)
	}

	checkCounterEQ(t, s, "txn.abort.count", 1)
	checkCounterEQ(t, s, "txn.begin.count", 1)
	checkCounterEQ(t, s, "txn.rollback.count", 0)
	checkCounterEQ(t, s, "txn.commit.count", 0)
	checkCounterEQ(t, s, "insert.count", 1)
}

// TestErrorDuringTransaction tests that the transaction abort count goes up when a query
// results in an error during a txn.
func TestAbortCountErrorDuringTransaction(t *testing.T) {
	defer leaktest.AfterTest(t)()
	s, sqlDB, _ := setup(t)
	defer cleanup(s, sqlDB)

	txn, err := sqlDB.Begin()
	if err != nil {
		t.Fatal(err)
	}

	if _, err := txn.Query("SELECT * FROM i_do.not_exist"); err == nil {
		t.Fatal("Expected an error but didn't get one")
	}

	checkCounterEQ(t, s, "txn.abort.count", 1)
	checkCounterEQ(t, s, "txn.begin.count", 1)
	checkCounterEQ(t, s, "select.count", 1)
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package http

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"syscall"
)

// A connection to a particular server over a particular protocol (HTTP or
// HTTPS).
type Conn interface {
	// Call the server with the supplied request, returning a response if and
	// only if a response was received from the server. (That is, a 500 error
	// from the server will be returned here as a response with a nil error).
	SendRequest(r *Request) (*Response, error)
}

// Return a connection to the supplied endpoint, based on its scheme and host
// fields.
func NewConn(endpoint *url.URL) (Conn, error) {
	switch endpoint.Scheme {
	case "http", "https":
	default:
		return nil, fmt.Errorf("Unsupported scheme: %s", endpoint.Scheme)
	}

	return &conn{endpoint}, nil
}

type conn struct {
	endpoint *url.URL
}

func makeRawQuery(r *Request) string {
	values := url.Values{}
	for key, val := range r.Parameters {
		values.Set(key, val)
	}

	return values.Encode()
}

func (c *conn) SendRequest(r *Request) (*Response, error) {
	// Create an appropriate URL.
	url := url.URL{
		Scheme:   c.endpoint.Scheme,
		Host:     c.endpoint.Host,
		Path:     r.Path,
		RawQuery: makeRawQuery(r),
	}

	urlStr := url.String()

	// Create a request to the system HTTP library.
	sysReq, err := http.NewRequest(r.Verb, urlStr, bytes.NewBuffer(r.Body))
	if err != nil {
		return nil, fmt.Errorf("http.NewRequest: %v", err)
	}

	// Copy headers.
	for key, val := range r.Headers {
		sysReq.Header.Set(key, val)
	}

	// Call the system HTTP library.
	sysResp, err := http.DefaultClient.Do(sysReq)
	if err != nil {
		// TODO(jacobsa): Remove this logging once it has yielded useful results
		// for investigating this issue:
		//
		//     https://github.com/jacobsa/comeback/issues/11
		//
		log.Println(
			"http.DefaultClient.Do:",
			reflect.TypeOf(err),
			reflect.ValueOf(err),
		)

		if opErr, ok := err.(*net.OpError); ok {
			log.Println("Op:        ", opErr.Op)
			log.Println("Net:       ", opErr.Net)
			log.Println("Addr:      ", opErr.Addr)
			log.Println("Err:       ", opErr.Err)
			log.Println("Temporary: ", opErr.Temporary())
			log.Println("Timeout:   ", opErr.Timeout())

			if errno, ok := opErr.Err.(syscall.Errno); ok {
				log.Printf("Errno: %u\n", errno)
				log.Printf("EPIPE: %u\n", syscall.EPIPE)
			}
		}

		return nil, fmt.Errorf("http.DefaultClient.Do: %v", err)
	}

	// Convert the response.
	resp := &Response{
		StatusCode: sysResp.StatusCode,
	}

	if resp.Body, err = ioutil.ReadAll(sysResp.Body); err != nil {
		return nil, fmt.Errorf("Reading body: %v", err)
	}

	sysResp.Body.Close()

	return resp, nil
}
<commit_msg>Switched to named returns.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package http

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"syscall"
)

// A connection to a particular server over a particular protocol (HTTP or
// HTTPS).
type Conn interface {
	// Call the server with the supplied request, returning a response if and
	// only if a response was received from the server. (That is, a 500 error
	// from the server will be returned here as a response with a nil error).
	SendRequest(r *Request) (*Response, error)
}

// Return a connection to the supplied endpoint, based on its scheme and host
// fields.
func NewConn(endpoint *url.URL) (c Conn, err error) {
	switch endpoint.Scheme {
	case "http", "https":
	default:
		err = fmt.Errorf("Unsupported scheme: %s", endpoint.Scheme)
		return
	}

	c = &conn{endpoint}
	return
}

type conn struct {
	endpoint *url.URL
}

func makeRawQuery(r *Request) string {
	values := url.Values{}
	for key, val := range r.Parameters {
		values.Set(key, val)
	}

	return values.Encode()
}

func (c *conn) SendRequest(r *Request) (resp *Response, err error) {
	// Create an appropriate URL.
	url := url.URL{
		Scheme:   c.endpoint.Scheme,
		Host:     c.endpoint.Host,
		Path:     r.Path,
		RawQuery: makeRawQuery(r),
	}

	urlStr := url.String()

	// Create a request to the system HTTP library.
	sysReq, err := http.NewRequest(r.Verb, urlStr, bytes.NewBuffer(r.Body))
	if err != nil {
		err = fmt.Errorf("http.NewRequest: %v", err)
		return
	}

	// Copy headers.
	for key, val := range r.Headers {
		sysReq.Header.Set(key, val)
	}

	// Call the system HTTP library.
	sysResp, err := http.DefaultClient.Do(sysReq)
	if err != nil {
		// TODO(jacobsa): Remove this logging once it has yielded useful results
		// for investigating this issue:
		//
		//     https://github.com/jacobsa/comeback/issues/11
		//
		log.Println(
			"http.DefaultClient.Do:",
			reflect.TypeOf(err),
			reflect.ValueOf(err),
		)

		if opErr, ok := err.(*net.OpError); ok {
			log.Println("Op:        ", opErr.Op)
			log.Println("Net:       ", opErr.Net)
			log.Println("Addr:      ", opErr.Addr)
			log.Println("Err:       ", opErr.Err)
			log.Println("Temporary: ", opErr.Temporary())
			log.Println("Timeout:   ", opErr.Timeout())

			if errno, ok := opErr.Err.(syscall.Errno); ok {
				log.Printf("Errno: %u\n", errno)
				log.Printf("EPIPE: %u\n", syscall.EPIPE)
			}
		}

		err = fmt.Errorf("http.DefaultClient.Do: %v", err)
		return
	}

	// Convert the response.
	resp = &Response{
		StatusCode: sysResp.StatusCode,
	}

	if resp.Body, err = ioutil.ReadAll(sysResp.Body); err != nil {
		err = fmt.Errorf("Reading body: %v", err)
		return
	}

	sysResp.Body.Close()

	return
}
<|endoftext|>