<commit_before>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package format

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/OWASP/Amass/v3/net"
	"github.com/OWASP/Amass/v3/requests"
	"github.com/fatih/color"
)

// Banner is the ASCII art logo used within help output.
const Banner = `
        .+++:.            :                             .+++.
      +W@@@@@@8        &+W@#               o8W8:      +W@@@@@@#.   oW@@@W#+
     &@#+   .o@##.    .@@@o@W.o@@o       :@@#&W8o    .@#:  .:oW+  .@#+++&#&
    +@&        &@&     #@8 +@W@&8@+     :@W.   +@8   +@:          .@8
    8@          @@     8@o  8@8  WW    .@W      W@+  .@W.          o@#:
    WW          &@o    &@:  o@+  o@+   #@.      8@o   +W@#+.        +W@8:
    #@          :@W    &@+  &@+   @8  :@o       o@o     oW@@W+        oW@8
    o@+          @@&   &@+  &@+   #@  &@.      .W@W       .+#@&         o@W.
     WW         +@W@8. &@+  :&    o@+ #@      :@W&@&         &@:  ..     :@o
     :@W:      o@# +Wo &@+        :W: +@W&o++o@W. &@&  8@#o+&@W.  #@:    o@+
      :W@@WWWW@@8       +              :&W@@@@&    &W  .o#@@W&.   :W@WWW@@&
        +o&&&&+.                                                    +oooo.
`

const (
	// Version is used to display the current version of Amass.
	Version = "v3.7.3"

	// Author is used to display the Amass Project Team.
	Author = "OWASP Amass Project - @owaspamass"

	// Description is the slogan for the Amass Project.
	Description = "In-depth Attack Surface Mapping and Asset Discovery"
)

var (
	// Colors used to ease the reading of program output
	y      = color.New(color.FgHiYellow)
	g      = color.New(color.FgHiGreen)
	r      = color.New(color.FgHiRed)
	b      = color.New(color.FgHiBlue)
	fgR    = color.New(color.FgRed)
	fgY    = color.New(color.FgYellow)
	yellow = color.New(color.FgHiYellow).SprintFunc()
	green  = color.New(color.FgHiGreen).SprintFunc()
	blue   = color.New(color.FgHiBlue).SprintFunc()
)

// ASNSummaryData stores information related to discovered ASs and netblocks.
type ASNSummaryData struct {
	Name      string
	Netblocks map[string]int
}

// UpdateSummaryData updates the summary maps using the provided requests.Output data.
func UpdateSummaryData(output *requests.Output, tags map[string]int, asns map[int]*ASNSummaryData) {
	tags[output.Tag]++

	for _, addr := range output.Addresses {
		data, found := asns[addr.ASN]
		if !found {
			asns[addr.ASN] = &ASNSummaryData{
				Name:      addr.Description,
				Netblocks: make(map[string]int),
			}
			data = asns[addr.ASN]
		}
		// Increment how many IPs were in this netblock
		data.Netblocks[addr.Netblock.String()]++
	}
}

// PrintEnumerationSummary outputs the summary information utilized by the command-line tools.
func PrintEnumerationSummary(total int, tags map[string]int, asns map[int]*ASNSummaryData, demo bool) {
	pad := func(num int, chr string) {
		for i := 0; i < num; i++ {
			b.Fprint(color.Error, chr)
		}
	}

	fmt.Fprintln(color.Error)
	// Print the header information
	title := "OWASP Amass "
	site := "https://github.com/OWASP/Amass"
	b.Fprint(color.Error, title+Version)
	num := 80 - (len(title) + len(Version) + len(site))
	pad(num, " ")
	b.Fprintf(color.Error, "%s\n", site)
	pad(8, "----------")
	fmt.Fprintf(color.Error, "\n%s%s", yellow(strconv.Itoa(total)), green(" names discovered - "))
	// Print the stats using tag information
	num, length := 1, len(tags)
	for k, v := range tags {
		fmt.Fprintf(color.Error, "%s: %s", green(k), yellow(strconv.Itoa(v)))
		if num < length {
			g.Fprint(color.Error, ", ")
		}
		num++
	}
	fmt.Fprintln(color.Error)

	if len(asns) == 0 {
		return
	}
	// Another line gets printed
	pad(8, "----------")
	fmt.Fprintln(color.Error)
	// Print the ASN and netblock information
	for asn, data := range asns {
		asnstr := strconv.Itoa(asn)
		datastr := data.Name

		if demo && asn > 0 {
			asnstr = censorString(asnstr, 0, len(asnstr))
			datastr = censorString(datastr, 0, len(datastr))
		}

		fmt.Fprintf(color.Error, "%s%s %s %s\n",
			blue("ASN: "), yellow(asnstr), green("-"), green(datastr))

		for cidr, ips := range data.Netblocks {
			countstr := strconv.Itoa(ips)
			cidrstr := cidr

			if demo {
				cidrstr = censorNetBlock(cidrstr)
			}

			countstr = fmt.Sprintf("\t%-4s", countstr)
			cidrstr = fmt.Sprintf("\t%-18s", cidrstr)

			fmt.Fprintf(color.Error, "%s%s %s\n",
				yellow(cidrstr), yellow(countstr), blue("Subdomain Name(s)"))
		}
	}
}

// PrintBanner outputs the Amass banner the same for all tools.
func PrintBanner() {
	y := color.New(color.FgHiYellow)
	r := color.New(color.FgHiRed)
	rightmost := 76

	pad := func(num int) {
		for i := 0; i < num; i++ {
			fmt.Fprint(color.Error, " ")
		}
	}
	r.Fprintln(color.Error, Banner)
	pad(rightmost - len(Version))
	y.Fprintln(color.Error, Version)
	pad(rightmost - len(Author))
	y.Fprintln(color.Error, Author)
	pad(rightmost - len(Description))
	y.Fprintf(color.Error, "%s\n\n\n", Description)
}

func censorDomain(input string) string {
	return censorString(input, strings.Index(input, "."), len(input))
}

func censorIP(input string) string {
	return censorString(input, 0, strings.LastIndex(input, "."))
}

func censorNetBlock(input string) string {
	return censorString(input, 0, strings.Index(input, "/"))
}

func censorString(input string, start, end int) string {
	runes := []rune(input)
	for i := start; i < end; i++ {
		if runes[i] == '.' ||
			runes[i] == '/' ||
			runes[i] == '-' ||
			runes[i] == ' ' {
			continue
		}
		runes[i] = 'x'
	}
	return string(runes)
}

// OutputLineParts returns the parts of a line to be printed for a requests.Output.
func OutputLineParts(out *requests.Output, src, addrs, demo bool) (source, name, ips string) {
	if src {
		source = fmt.Sprintf("%-18s", "["+out.Source+"] ")
	}
	if addrs {
		for i, a := range out.Addresses {
			if i != 0 {
				ips += ","
			}
			if demo {
				ips += censorIP(a.Address.String())
			} else {
				ips += a.Address.String()
			}
		}
		if ips == "" {
			ips = "N/A"
		}
	}
	name = out.Name
	if demo {
		name = censorDomain(name)
	}
	return
}

// DesiredAddrTypes removes undesired address types from the AddressInfo slice.
func DesiredAddrTypes(addrs []requests.AddressInfo, ipv4, ipv6 bool) []requests.AddressInfo {
	if !ipv4 && !ipv6 {
		return addrs
	}

	var keep []requests.AddressInfo
	for _, addr := range addrs {
		if net.IsIPv4(addr.Address) && !ipv4 {
			continue
		} else if net.IsIPv6(addr.Address) && !ipv6 {
			continue
		}
		keep = append(keep, addr)
	}
	return keep
}
<commit_msg>v3.7.4 release<commit_after>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package format

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/OWASP/Amass/v3/net"
	"github.com/OWASP/Amass/v3/requests"
	"github.com/fatih/color"
)

// Banner is the ASCII art logo used within help output.
const Banner = `
        .+++:.            :                             .+++.
      +W@@@@@@8        &+W@#               o8W8:      +W@@@@@@#.   oW@@@W#+
     &@#+   .o@##.    .@@@o@W.o@@o       :@@#&W8o    .@#:  .:oW+  .@#+++&#&
    +@&        &@&     #@8 +@W@&8@+     :@W.   +@8   +@:          .@8
    8@          @@     8@o  8@8  WW    .@W      W@+  .@W.          o@#:
    WW          &@o    &@:  o@+  o@+   #@.      8@o   +W@#+.        +W@8:
    #@          :@W    &@+  &@+   @8  :@o       o@o     oW@@W+        oW@8
    o@+          @@&   &@+  &@+   #@  &@.      .W@W       .+#@&         o@W.
     WW         +@W@8. &@+  :&    o@+ #@      :@W&@&         &@:  ..     :@o
     :@W:      o@# +Wo &@+        :W: +@W&o++o@W. &@&  8@#o+&@W.  #@:    o@+
      :W@@WWWW@@8       +              :&W@@@@&    &W  .o#@@W&.   :W@WWW@@&
        +o&&&&+.                                                    +oooo.
`

const (
	// Version is used to display the current version of Amass.
	Version = "v3.7.4"

	// Author is used to display the Amass Project Team.
	Author = "OWASP Amass Project - @owaspamass"

	// Description is the slogan for the Amass Project.
	Description = "In-depth Attack Surface Mapping and Asset Discovery"
)

var (
	// Colors used to ease the reading of program output
	y      = color.New(color.FgHiYellow)
	g      = color.New(color.FgHiGreen)
	r      = color.New(color.FgHiRed)
	b      = color.New(color.FgHiBlue)
	fgR    = color.New(color.FgRed)
	fgY    = color.New(color.FgYellow)
	yellow = color.New(color.FgHiYellow).SprintFunc()
	green  = color.New(color.FgHiGreen).SprintFunc()
	blue   = color.New(color.FgHiBlue).SprintFunc()
)

// ASNSummaryData stores information related to discovered ASs and netblocks.
type ASNSummaryData struct {
	Name      string
	Netblocks map[string]int
}

// UpdateSummaryData updates the summary maps using the provided requests.Output data.
func UpdateSummaryData(output *requests.Output, tags map[string]int, asns map[int]*ASNSummaryData) {
	tags[output.Tag]++

	for _, addr := range output.Addresses {
		data, found := asns[addr.ASN]
		if !found {
			asns[addr.ASN] = &ASNSummaryData{
				Name:      addr.Description,
				Netblocks: make(map[string]int),
			}
			data = asns[addr.ASN]
		}
		// Increment how many IPs were in this netblock
		data.Netblocks[addr.Netblock.String()]++
	}
}

// PrintEnumerationSummary outputs the summary information utilized by the command-line tools.
func PrintEnumerationSummary(total int, tags map[string]int, asns map[int]*ASNSummaryData, demo bool) {
	pad := func(num int, chr string) {
		for i := 0; i < num; i++ {
			b.Fprint(color.Error, chr)
		}
	}

	fmt.Fprintln(color.Error)
	// Print the header information
	title := "OWASP Amass "
	site := "https://github.com/OWASP/Amass"
	b.Fprint(color.Error, title+Version)
	num := 80 - (len(title) + len(Version) + len(site))
	pad(num, " ")
	b.Fprintf(color.Error, "%s\n", site)
	pad(8, "----------")
	fmt.Fprintf(color.Error, "\n%s%s", yellow(strconv.Itoa(total)), green(" names discovered - "))
	// Print the stats using tag information
	num, length := 1, len(tags)
	for k, v := range tags {
		fmt.Fprintf(color.Error, "%s: %s", green(k), yellow(strconv.Itoa(v)))
		if num < length {
			g.Fprint(color.Error, ", ")
		}
		num++
	}
	fmt.Fprintln(color.Error)

	if len(asns) == 0 {
		return
	}
	// Another line gets printed
	pad(8, "----------")
	fmt.Fprintln(color.Error)
	// Print the ASN and netblock information
	for asn, data := range asns {
		asnstr := strconv.Itoa(asn)
		datastr := data.Name

		if demo && asn > 0 {
			asnstr = censorString(asnstr, 0, len(asnstr))
			datastr = censorString(datastr, 0, len(datastr))
		}

		fmt.Fprintf(color.Error, "%s%s %s %s\n",
			blue("ASN: "), yellow(asnstr), green("-"), green(datastr))

		for cidr, ips := range data.Netblocks {
			countstr := strconv.Itoa(ips)
			cidrstr := cidr

			if demo {
				cidrstr = censorNetBlock(cidrstr)
			}

			countstr = fmt.Sprintf("\t%-4s", countstr)
			cidrstr = fmt.Sprintf("\t%-18s", cidrstr)

			fmt.Fprintf(color.Error, "%s%s %s\n",
				yellow(cidrstr), yellow(countstr), blue("Subdomain Name(s)"))
		}
	}
}

// PrintBanner outputs the Amass banner the same for all tools.
func PrintBanner() {
	y := color.New(color.FgHiYellow)
	r := color.New(color.FgHiRed)
	rightmost := 76

	pad := func(num int) {
		for i := 0; i < num; i++ {
			fmt.Fprint(color.Error, " ")
		}
	}
	r.Fprintln(color.Error, Banner)
	pad(rightmost - len(Version))
	y.Fprintln(color.Error, Version)
	pad(rightmost - len(Author))
	y.Fprintln(color.Error, Author)
	pad(rightmost - len(Description))
	y.Fprintf(color.Error, "%s\n\n\n", Description)
}

func censorDomain(input string) string {
	return censorString(input, strings.Index(input, "."), len(input))
}

func censorIP(input string) string {
	return censorString(input, 0, strings.LastIndex(input, "."))
}

func censorNetBlock(input string) string {
	return censorString(input, 0, strings.Index(input, "/"))
}

func censorString(input string, start, end int) string {
	runes := []rune(input)
	for i := start; i < end; i++ {
		if runes[i] == '.' ||
			runes[i] == '/' ||
			runes[i] == '-' ||
			runes[i] == ' ' {
			continue
		}
		runes[i] = 'x'
	}
	return string(runes)
}

// OutputLineParts returns the parts of a line to be printed for a requests.Output.
func OutputLineParts(out *requests.Output, src, addrs, demo bool) (source, name, ips string) {
	if src {
		source = fmt.Sprintf("%-18s", "["+out.Source+"] ")
	}
	if addrs {
		for i, a := range out.Addresses {
			if i != 0 {
				ips += ","
			}
			if demo {
				ips += censorIP(a.Address.String())
			} else {
				ips += a.Address.String()
			}
		}
		if ips == "" {
			ips = "N/A"
		}
	}
	name = out.Name
	if demo {
		name = censorDomain(name)
	}
	return
}

// DesiredAddrTypes removes undesired address types from the AddressInfo slice.
func DesiredAddrTypes(addrs []requests.AddressInfo, ipv4, ipv6 bool) []requests.AddressInfo {
	if !ipv4 && !ipv6 {
		return addrs
	}

	var keep []requests.AddressInfo
	for _, addr := range addrs {
		if net.IsIPv4(addr.Address) && !ipv4 {
			continue
		} else if net.IsIPv6(addr.Address) && !ipv6 {
			continue
		}
		keep = append(keep, addr)
	}
	return keep
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2014 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vm

import (
	"flag"
	"fmt"

	"github.com/vmware/govmomi"
	"github.com/vmware/govmomi/govc/cli"
	"github.com/vmware/govmomi/govc/flags"
)

type create struct {
	*flags.ResourcePoolFlag
	*flags.HostSystemFlag
	*flags.DatastoreFlag

	memory  int
	cpus    int
	guestID string
}

func init() {
	cli.Register(&create{})
}

func (c *create) Register(f *flag.FlagSet) {
	f.IntVar(&c.memory, "m", 128, "Size in MB of memory")
	f.IntVar(&c.cpus, "c", 1, "Number of CPUs")
	f.StringVar(&c.guestID, "g", "otherGuest", "Guest OS")
}

func (c *create) Process() error { return nil }

func (c *create) Run(f *flag.FlagSet) error {
	if len(f.Args()) != 1 {
		return flag.ErrHelp
	}

	var pool *govmomi.ResourcePool

	host, err := c.HostSystem()
	if err != nil {
		return err
	}

	if host == nil { // -host is optional
		if pool, err = c.ResourcePool(); err != nil {
			return err
		}
	} else {
		if pool, err = c.HostResourcePool(); err != nil {
			return err
		}
	}

	ds, err := c.DatastoreFlag.Name()
	if err != nil {
		return err
	}

	fmt.Printf("create %s VM '%s' on datastore %s, with %d MB memory (pool=%s)\n",
		c.guestID, f.Arg(0), ds, c.memory, pool.Value)

	return nil
}
<commit_msg>Implement vm.create cli command<commit_after>/*
Copyright (c) 2014 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vm

import (
	"errors"
	"flag"
	"fmt"
	"path/filepath"

	"github.com/vmware/govmomi"
	"github.com/vmware/govmomi/govc/cli"
	"github.com/vmware/govmomi/govc/flags"
	"github.com/vmware/govmomi/vim25/types"
)

type create struct {
	*flags.ResourcePoolFlag
	*flags.HostSystemFlag
	*flags.DatastoreFlag
	*flags.VmFolderFlag
	*flags.NetworkFlag
	*flags.DiskFlag

	memory  int
	cpus    int
	guestID string
}

func init() {
	cli.Register(&create{})
}

func (c *create) Register(f *flag.FlagSet) {
	f.IntVar(&c.memory, "m", 128, "Size in MB of memory")
	f.IntVar(&c.cpus, "c", 1, "Number of CPUs")
	f.StringVar(&c.guestID, "g", "otherGuest", "Guest OS")
}

func (c *create) Process() error { return nil }

func (c *create) Run(f *flag.FlagSet) error {
	if len(f.Args()) != 1 {
		return flag.ErrHelp
	}

	var pool *govmomi.ResourcePool

	host, err := c.HostSystem()
	if err != nil {
		return err
	}

	if host == nil { // -host is optional
		if pool, err = c.ResourcePool(); err != nil {
			return err
		}
	} else {
		if pool, err = c.HostResourcePool(); err != nil {
			return err
		}
	}

	ds, err := c.DatastoreFlag.Name()
	if err != nil {
		return err
	}

	name := f.Arg(0)

	spec := types.VirtualMachineConfigSpec{
		Name:     name,
		GuestId:  c.guestID,
		Files:    &types.VirtualMachineFileInfo{VmPathName: fmt.Sprintf("[%s]", ds)},
		NumCPUs:  c.cpus,
		MemoryMB: int64(c.memory),
	}

	if err = c.addDisk(&spec); err != nil {
		return err
	}

	if err = c.addNetwork(&spec); err != nil {
		return err
	}

	client, err := c.DatastoreFlag.Client()
	if err != nil {
		return err
	}

	folder, err := c.VmFolder()
	if err != nil {
		return err
	}

	return folder.CreateVM(client, spec, pool, host)
}

func (c *create) addDevice(spec *types.VirtualMachineConfigSpec, device types.BaseVirtualDevice) {
	spec.DeviceChange = append(spec.DeviceChange, &types.VirtualDeviceConfigSpec{
		Operation: types.VirtualDeviceConfigSpecOperationAdd,
		Device:    device,
	})
}

func (c *create) addNetwork(spec *types.VirtualMachineConfigSpec) error {
	network, err := c.NetworkFlag.Device()
	if err != nil {
		return err
	}

	c.addDevice(spec, network)

	return nil
}

func (c *create) addDisk(spec *types.VirtualMachineConfigSpec) error {
	if !c.DiskFlag.IsSet() {
		return nil
	}

	diskPath, err := c.DiskFlag.Name()
	if err != nil {
		return err
	}

	switch filepath.Ext(diskPath) {
	case ".vmdk":
		device, err := c.DiskFlag.Controller()
		if err != nil {
			return err
		}
		c.addDevice(spec, device)

		device, err = c.DiskFlag.Copy(fmt.Sprintf("%s/%s.vmdk", spec.Name, spec.Name))
		if err != nil {
			return err
		}
		c.addDevice(spec, device)
	case ".iso":
		return errors.New("TODO: .iso not supported yet")
	default:
		return errors.New("unsupported disk type")
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Only build when go-fuzz is in use
// +build gofuzz

package promql

import "github.com/prometheus/prometheus/pkg/textparse"

// PromQL parser fuzzing instrumentation for use with
// https://github.com/dvyukov/go-fuzz.
//
// Fuzz each parser by building appropriately instrumented parser, ex.
// FuzzParseMetric and execute it with it's
//
//     go-fuzz-build -func FuzzParseMetric -o FuzzParseMetric.zip github.com/prometheus/prometheus/promql
//
// And then run the tests with the appropriate inputs
//
//     go-fuzz -bin FuzzParseMetric.zip -workdir fuzz-data/ParseMetric
//
// Further input samples should go in the folders fuzz-data/ParseMetric/corpus.
//
// Repeat for ParseMetricSelection, ParseExpr and ParseStmt.

// Tuning which value is returned from Fuzz*-functions has a strong influence
// on how quick the fuzzer converges on "interesting" cases. At least try
// switching between fuzzMeh (= included in corpus, but not a priority) and
// fuzzDiscard (=don't use this input for re-building later inputs) when
// experimenting.
const (
	fuzzInteresting = 1
	fuzzMeh         = 0
	fuzzDiscard     = -1
)

// Fuzz the metric parser.
//
// Note that his is not the parser for the text-based exposition-format; that
// lives in github.com/prometheus/client_golang/text.
func FuzzParseMetric(in []byte) int {
	p := textparse.New(in)
	for p.Next() {
	}

	if p.Err() == nil {
		return fuzzInteresting
	}

	return fuzzMeh
}

// Fuzz the metric selector parser.
func FuzzParseMetricSelector(in []byte) int {
	_, err := ParseMetricSelector(string(in))
	if err == nil {
		return fuzzInteresting
	}

	return fuzzMeh
}

// Fuzz the expression parser.
func FuzzParseExpr(in []byte) int {
	_, err := ParseExpr(string(in))
	if err == nil {
		return fuzzInteresting
	}

	return fuzzMeh
}

// Fuzz the parser.
func FuzzParseStmts(in []byte) int {
	_, err := ParseStmts(string(in))
	if err == nil {
		return fuzzInteresting
	}

	return fuzzMeh
}
<commit_msg>fix fuzz targets (#5851)<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Only build when go-fuzz is in use
// +build gofuzz

package promql

import (
	"io"

	"github.com/prometheus/prometheus/pkg/textparse"
)

// PromQL parser fuzzing instrumentation for use with
// https://github.com/dvyukov/go-fuzz.
//
// Fuzz each parser by building appropriately instrumented parser, ex.
// FuzzParseMetric and execute it with it's
//
//     go-fuzz-build -func FuzzParseMetric -o FuzzParseMetric.zip github.com/prometheus/prometheus/promql
//
// And then run the tests with the appropriate inputs
//
//     go-fuzz -bin FuzzParseMetric.zip -workdir fuzz-data/ParseMetric
//
// Further input samples should go in the folders fuzz-data/ParseMetric/corpus.
//
// Repeat for FuzzParseOpenMetric, FuzzParseMetricSelector and FuzzParseExpr.

// Tuning which value is returned from Fuzz*-functions has a strong influence
// on how quick the fuzzer converges on "interesting" cases. At least try
// switching between fuzzMeh (= included in corpus, but not a priority) and
// fuzzDiscard (=don't use this input for re-building later inputs) when
// experimenting.
const (
	fuzzInteresting = 1
	fuzzMeh         = 0
	fuzzDiscard     = -1
)

func fuzzParseMetricWithContentType(in []byte, contentType string) int {
	p := textparse.New(in, contentType)
	var err error
	for {
		_, err = p.Next()
		if err != nil {
			break
		}
	}
	if err == io.EOF {
		err = nil
	}

	if err == nil {
		return fuzzInteresting
	}

	return fuzzMeh
}

// Fuzz the metric parser.
//
// Note that this is not the parser for the text-based exposition-format; that
// lives in github.com/prometheus/client_golang/text.
func FuzzParseMetric(in []byte) int {
	return fuzzParseMetricWithContentType(in, "")
}

func FuzzParseOpenMetric(in []byte) int {
	return fuzzParseMetricWithContentType(in, "application/openmetrics-text")
}

// Fuzz the metric selector parser.
func FuzzParseMetricSelector(in []byte) int {
	_, err := ParseMetricSelector(string(in))
	if err == nil {
		return fuzzInteresting
	}

	return fuzzMeh
}

// Fuzz the expression parser.
func FuzzParseExpr(in []byte) int {
	_, err := ParseExpr(string(in))
	if err == nil {
		return fuzzInteresting
	}

	return fuzzMeh
}
<|endoftext|>


<commit_before>package nsq

const ProtocolV2Magic = "  V2"

const (
	ClientStateV2Init       = 0
	ClientStateV2Subscribed = 1
	ClientStateV2Closing    = 2 // close has started. responses are ok, but no new messages will be sent
)

const (
	FrameTypeResponse  int32 = 0
	FrameTypeError     int32 = 1
	FrameTypeMessage   int32 = 2
	FrameTypeCloseWait int32 = 3
)

var (
	ClientErrV2Invalid    = ClientError{"E_INVALID"}
	ClientErrV2BadTopic   = ClientError{"E_BAD_TOPIC"}
	ClientErrV2BadChannel = ClientError{"E_BAD_CHANNEL"}
	ClientErrV2BadMessage = ClientError{"E_BAD_MESSAGE"}
)
<commit_msg>better protocol v2 error handling<commit_after>package nsq

const ProtocolV2Magic = "  V2"

const (
	ClientStateV2Init       = 0
	ClientStateV2Subscribed = 1
	ClientStateV2Closing    = 2 // close has started. responses are ok, but no new messages will be sent
)

const (
	FrameTypeResponse  int32 = 0
	FrameTypeError     int32 = 1
	FrameTypeMessage   int32 = 2
	FrameTypeCloseWait int32 = 3
)

var (
	ClientErrV2Invalid       = ClientError{"E_INVALID"}
	ClientErrV2BadTopic      = ClientError{"E_BAD_TOPIC"}
	ClientErrV2BadChannel    = ClientError{"E_BAD_CHANNEL"}
	ClientErrV2BadMessage    = ClientError{"E_BAD_MESSAGE"}
	ClientErrV2RequeueFailed = ClientError{"E_REQ_FAILED"}
	ClientErrV2FinishFailed  = ClientError{"E_FIN_FAILED"}
)
<|endoftext|>


<commit_before>package taskreaper

import (
	"sort"
	"time"

	"github.com/docker/go-events"
	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/log"
	"github.com/docker/swarmkit/manager/state"
	"github.com/docker/swarmkit/manager/state/store"
	"golang.org/x/net/context"
)

const (
	// maxDirty is the size threshold for running a task pruning operation.
	maxDirty = 1000
	// reaperBatchingInterval is how often to prune old tasks.
	reaperBatchingInterval = 250 * time.Millisecond
)

type instanceTuple struct {
	instance  uint64 // unset for global tasks
	serviceID string
	nodeID    string // unset for replicated tasks
}

// A TaskReaper deletes old tasks when more than TaskHistoryRetentionLimit tasks
// exist for the same service/instance or service/nodeid combination.
type TaskReaper struct {
	store *store.MemoryStore
	// taskHistory is the number of tasks to keep
	taskHistory int64
	dirty       map[instanceTuple]struct{}
	orphaned    []string
	watcher     chan events.Event
	cancelWatch func()
	stopChan    chan struct{}
	doneChan    chan struct{}
}

// New creates a new TaskReaper.
func New(store *store.MemoryStore) *TaskReaper {
	watcher, cancel := state.Watch(store.WatchQueue(), state.EventCreateTask{}, state.EventUpdateTask{}, state.EventUpdateCluster{})

	return &TaskReaper{
		store:       store,
		watcher:     watcher,
		cancelWatch: cancel,
		dirty:       make(map[instanceTuple]struct{}),
		stopChan:    make(chan struct{}),
		doneChan:    make(chan struct{}),
	}
}

// Run is the TaskReaper's main loop.
func (tr *TaskReaper) Run() {
	defer close(tr.doneChan)

	var tasks []*api.Task
	tr.store.View(func(readTx store.ReadTx) {
		var err error

		clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName))
		if err == nil && len(clusters) == 1 {
			tr.taskHistory = clusters[0].Spec.Orchestration.TaskHistoryRetentionLimit
		}

		tasks, err = store.FindTasks(readTx, store.ByTaskState(api.TaskStateOrphaned))
		if err != nil {
			log.G(context.TODO()).WithError(err).Error("failed to find Orphaned tasks in task reaper init")
		}
	})

	if len(tasks) > 0 {
		for _, t := range tasks {
			// Do not reap service tasks immediately
			if t.ServiceID != "" {
				continue
			}

			tr.orphaned = append(tr.orphaned, t.ID)
		}

		if len(tr.orphaned) > 0 {
			tr.tick()
		}
	}

	timer := time.NewTimer(reaperBatchingInterval)

	for {
		select {
		case event := <-tr.watcher:
			switch v := event.(type) {
			case state.EventCreateTask:
				t := v.Task
				tr.dirty[instanceTuple{
					instance:  t.Slot,
					serviceID: t.ServiceID,
					nodeID:    t.NodeID,
				}] = struct{}{}
			case state.EventUpdateTask:
				t := v.Task
				if t.Status.State >= api.TaskStateOrphaned && t.ServiceID == "" {
					tr.orphaned = append(tr.orphaned, t.ID)
				}
			case state.EventUpdateCluster:
				tr.taskHistory = v.Cluster.Spec.Orchestration.TaskHistoryRetentionLimit
			}

			if len(tr.dirty)+len(tr.orphaned) > maxDirty {
				timer.Stop()
				tr.tick()
			} else {
				timer.Reset(reaperBatchingInterval)
			}
		case <-timer.C:
			timer.Stop()
			tr.tick()
		case <-tr.stopChan:
			timer.Stop()
			return
		}
	}
}

func (tr *TaskReaper) tick() {
	if len(tr.dirty) == 0 && len(tr.orphaned) == 0 {
		return
	}

	defer func() {
		tr.dirty = make(map[instanceTuple]struct{})
		tr.orphaned = nil
	}()

	deleteTasks := tr.orphaned
	tr.store.View(func(tx store.ReadTx) {
		for dirty := range tr.dirty {
			service := store.GetService(tx, dirty.serviceID)
			if service == nil {
				continue
			}

			taskHistory := tr.taskHistory

			if taskHistory < 0 {
				continue
			}

			var historicTasks []*api.Task

			switch service.Spec.GetMode().(type) {
			case *api.ServiceSpec_Replicated:
				var err error
				historicTasks, err = store.FindTasks(tx, store.BySlot(dirty.serviceID, dirty.instance))
				if err != nil {
					continue
				}

			case *api.ServiceSpec_Global:
				tasksByNode, err := store.FindTasks(tx, store.ByNodeID(dirty.nodeID))
				if err != nil {
					continue
				}

				for _, t := range tasksByNode {
					if t.ServiceID == dirty.serviceID {
						historicTasks = append(historicTasks, t)
					}
				}
			}

			if int64(len(historicTasks)) <= taskHistory {
				continue
			}

			// TODO(aaronl): This could filter for non-running tasks and use quickselect
			// instead of sorting the whole slice.
			sort.Sort(tasksByTimestamp(historicTasks))

			for _, t := range historicTasks {
				if t.DesiredState <= api.TaskStateRunning {
					// Don't delete running tasks
					continue
				}

				deleteTasks = append(deleteTasks, t.ID)

				taskHistory++
				if int64(len(historicTasks)) <= taskHistory {
					break
				}
			}

		}
	})

	if len(deleteTasks) > 0 {
		tr.store.Batch(func(batch *store.Batch) error {
			for _, taskID := range deleteTasks {
				batch.Update(func(tx store.Tx) error {
					return store.DeleteTask(tx, taskID)
				})
			}
			return nil
		})
	}
}

// Stop stops the TaskReaper and waits for the main loop to exit.
func (tr *TaskReaper) Stop() {
	tr.cancelWatch()
	close(tr.stopChan)
	<-tr.doneChan
}

type tasksByTimestamp []*api.Task

func (t tasksByTimestamp) Len() int {
	return len(t)
}
func (t tasksByTimestamp) Swap(i, j int) {
	t[i], t[j] = t[j], t[i]
}
func (t tasksByTimestamp) Less(i, j int) bool {
	if t[i].Status.Timestamp == nil {
		return true
	}
	if t[j].Status.Timestamp == nil {
		return false
	}
	if t[i].Status.Timestamp.Seconds < t[j].Status.Timestamp.Seconds {
		return true
	}
	if t[i].Status.Timestamp.Seconds > t[j].Status.Timestamp.Seconds {
		return false
	}
	return t[i].Status.Timestamp.Nanos < t[j].Status.Timestamp.Nanos
}
<commit_msg>taskreaper: Wait for tasks to stop running<commit_after>package taskreaper

import (
	"sort"
	"time"

	"github.com/docker/go-events"
	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/log"
	"github.com/docker/swarmkit/manager/state"
	"github.com/docker/swarmkit/manager/state/store"
	"golang.org/x/net/context"
)

const (
	// maxDirty is the size threshold for running a task pruning operation.
	maxDirty = 1000
	// reaperBatchingInterval is how often to prune old tasks.
	reaperBatchingInterval = 250 * time.Millisecond
)

type instanceTuple struct {
	instance  uint64 // unset for global tasks
	serviceID string
	nodeID    string // unset for replicated tasks
}

// A TaskReaper deletes old tasks when more than TaskHistoryRetentionLimit tasks
// exist for the same service/instance or service/nodeid combination.
type TaskReaper struct {
	store *store.MemoryStore
	// taskHistory is the number of tasks to keep
	taskHistory int64
	dirty       map[instanceTuple]struct{}
	orphaned    []string
	watcher     chan events.Event
	cancelWatch func()
	stopChan    chan struct{}
	doneChan    chan struct{}
}

// New creates a new TaskReaper.
func New(store *store.MemoryStore) *TaskReaper {
	watcher, cancel := state.Watch(store.WatchQueue(), state.EventCreateTask{}, state.EventUpdateTask{}, state.EventUpdateCluster{})

	return &TaskReaper{
		store:       store,
		watcher:     watcher,
		cancelWatch: cancel,
		dirty:       make(map[instanceTuple]struct{}),
		stopChan:    make(chan struct{}),
		doneChan:    make(chan struct{}),
	}
}

// Run is the TaskReaper's main loop.
func (tr *TaskReaper) Run() {
	defer close(tr.doneChan)

	var tasks []*api.Task
	tr.store.View(func(readTx store.ReadTx) {
		var err error

		clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName))
		if err == nil && len(clusters) == 1 {
			tr.taskHistory = clusters[0].Spec.Orchestration.TaskHistoryRetentionLimit
		}

		tasks, err = store.FindTasks(readTx, store.ByTaskState(api.TaskStateOrphaned))
		if err != nil {
			log.G(context.TODO()).WithError(err).Error("failed to find Orphaned tasks in task reaper init")
		}
	})

	if len(tasks) > 0 {
		for _, t := range tasks {
			// Do not reap service tasks immediately
			if t.ServiceID != "" {
				continue
			}

			tr.orphaned = append(tr.orphaned, t.ID)
		}

		if len(tr.orphaned) > 0 {
			tr.tick()
		}
	}

	timer := time.NewTimer(reaperBatchingInterval)

	for {
		select {
		case event := <-tr.watcher:
			switch v := event.(type) {
			case state.EventCreateTask:
				t := v.Task
				tr.dirty[instanceTuple{
					instance:  t.Slot,
					serviceID: t.ServiceID,
					nodeID:    t.NodeID,
				}] = struct{}{}
			case state.EventUpdateTask:
				t := v.Task
				if t.Status.State >= api.TaskStateOrphaned && t.ServiceID == "" {
					tr.orphaned = append(tr.orphaned, t.ID)
				}
			case state.EventUpdateCluster:
				tr.taskHistory = v.Cluster.Spec.Orchestration.TaskHistoryRetentionLimit
			}

			if len(tr.dirty)+len(tr.orphaned) > maxDirty {
				timer.Stop()
				tr.tick()
			} else {
				timer.Reset(reaperBatchingInterval)
			}
		case <-timer.C:
			timer.Stop()
			tr.tick()
		case <-tr.stopChan:
			timer.Stop()
			return
		}
	}
}

func (tr *TaskReaper) tick() {
	if len(tr.dirty) == 0 && len(tr.orphaned) == 0 {
		return
	}

	defer func() {
		tr.orphaned = nil
	}()

	deleteTasks := make(map[string]struct{})
	for _, tID := range tr.orphaned {
		deleteTasks[tID] = struct{}{}
	}
	tr.store.View(func(tx store.ReadTx) {
		for dirty := range tr.dirty {
			service := store.GetService(tx, dirty.serviceID)
			if service == nil {
				continue
			}

			taskHistory := tr.taskHistory

			if taskHistory < 0 {
				continue
			}

			var historicTasks []*api.Task

			switch service.Spec.GetMode().(type) {
			case *api.ServiceSpec_Replicated:
				var err error
				historicTasks, err = store.FindTasks(tx, store.BySlot(dirty.serviceID, dirty.instance))
				if err != nil {
					continue
				}

			case *api.ServiceSpec_Global:
				tasksByNode, err := store.FindTasks(tx, store.ByNodeID(dirty.nodeID))
				if err != nil {
					continue
				}

				for _, t := range tasksByNode {
					if t.ServiceID == dirty.serviceID {
						historicTasks = append(historicTasks, t)
					}
				}
			}

			if int64(len(historicTasks)) <= taskHistory {
				continue
			}

			// TODO(aaronl): This could filter for non-running tasks and use quickselect
			// instead of sorting the whole slice.
			sort.Sort(tasksByTimestamp(historicTasks))

			runningTasks := 0
			for _, t := range historicTasks {
				if t.DesiredState <= api.TaskStateRunning || t.Status.State <= api.TaskStateRunning {
					// Don't delete running tasks
					runningTasks++
					continue
				}

				deleteTasks[t.ID] = struct{}{}

				taskHistory++
				if int64(len(historicTasks)) <= taskHistory {
					break
				}
			}

			if runningTasks <= 1 {
				delete(tr.dirty, dirty)
			}
		}
	})

	if len(deleteTasks) > 0 {
		tr.store.Batch(func(batch *store.Batch) error {
			for taskID := range deleteTasks {
				batch.Update(func(tx store.Tx) error {
					return store.DeleteTask(tx, taskID)
				})
			}
			return nil
		})
	}
}

// Stop stops the TaskReaper and waits for the main loop to exit.
func (tr *TaskReaper) Stop() {
	tr.cancelWatch()
	close(tr.stopChan)
	<-tr.doneChan
}

type tasksByTimestamp []*api.Task

func (t tasksByTimestamp) Len() int {
	return len(t)
}
func (t tasksByTimestamp) Swap(i, j int) {
	t[i], t[j] = t[j], t[i]
}
func (t tasksByTimestamp) Less(i, j int) bool {
	if t[i].Status.Timestamp == nil {
		return true
	}
	if t[j].Status.Timestamp == nil {
		return false
	}
	if t[i].Status.Timestamp.Seconds < t[j].Status.Timestamp.Seconds {
		return true
	}
	if t[i].Status.Timestamp.Seconds > t[j].Status.Timestamp.Seconds {
		return false
	}
	return t[i].Status.Timestamp.Nanos < t[j].Status.Timestamp.Nanos
}
<|endoftext|>


<commit_before>package util

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	rspecs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/opencontainers/runtime-tools/generate"
	"github.com/opencontainers/runtime-tools/specerror"
)

// Runtime represents the basic requirement of a container runtime
type Runtime struct {
	RuntimeCommand string
	BundleDir      string
	PidFile        string
	ID             string
	stdout         bytes.Buffer
	stderr         bytes.Buffer
}

// DefaultSignal represents the default signal sends to a container
const DefaultSignal = "TERM"

// NewRuntime create a runtime by command and the bundle directory
func NewRuntime(runtimeCommand string, bundleDir string) (Runtime, error) {
	var r Runtime
	var err error
	r.RuntimeCommand, err = exec.LookPath(runtimeCommand)
	if err != nil {
		return Runtime{}, err
	}

	r.BundleDir = bundleDir
	return r, err
}

// bundleDir returns the bundle directory.  Generally this is
// BundleDir, but when BundleDir is the empty string, it falls back to
// ., as specified in the CLI spec.
func (r *Runtime) bundleDir() (bundleDir string) {
	if r.BundleDir == "" {
		return "."
	}
	return r.BundleDir
}

// SetConfig creates a 'config.json' by the generator
func (r *Runtime) SetConfig(g *generate.Generator) error {
	if g == nil {
		return errors.New("cannot set a nil config")
	}
	return g.SaveToFile(filepath.Join(r.bundleDir(), "config.json"), generate.ExportOptions{})
}

// SetID sets the container ID
func (r *Runtime) SetID(id string) {
	r.ID = id
}

// Create a container
func (r *Runtime) Create() (err error) {
	var args []string
	args = append(args, "create")
	if r.PidFile != "" {
		args = append(args, "--pid-file", r.PidFile)
	}
	if r.BundleDir != "" {
		args = append(args, "--bundle", r.BundleDir)
	}
	if r.ID != "" {
		args = append(args, r.ID)
	}
	cmd := exec.Command(r.RuntimeCommand, args...)
	r.stdout.Reset()
	cmd.Stdout = &r.stdout
	r.stderr.Reset()
	cmd.Stderr = &r.stderr

	err = cmd.Run()
	if err == nil {
		return err
	}

	if e, ok := err.(*exec.ExitError); ok {
		stdout, stderr := r.StandardStreams()
		if len(stderr) == 0 {
			stderr = stdout
		}
		e.Stderr = stderr
		return e
	}
	return err
}

// StandardStreams returns content from the stdout and stderr buffers.
func (r *Runtime) StandardStreams() (stdout, stderr []byte) {
	return r.stdout.Bytes(), r.stderr.Bytes()
}

// Start a container
func (r *Runtime) Start() (err error) {
	var args []string
	args = append(args, "start")
	if r.ID != "" {
		args = append(args, r.ID)
	}

	cmd := exec.Command(r.RuntimeCommand, args...)
	return execWithStderrFallbackToStdout(cmd)
}

// State a container information
func (r *Runtime) State() (rspecs.State, error) {
	var args []string
	args = append(args, "state")
	if r.ID != "" {
		args = append(args, r.ID)
	}

	out, err := exec.Command(r.RuntimeCommand, args...).Output()
	if err != nil {
		if e, ok := err.(*exec.ExitError); ok {
			if len(e.Stderr) == 0 {
				e.Stderr = out
				return rspecs.State{}, e
			}
		}
		return rspecs.State{}, err
	}

	var state rspecs.State
	err = json.Unmarshal(out, &state)
	if err != nil {
		return rspecs.State{}, specerror.NewError(specerror.DefaultStateJSONPattern, fmt.Errorf("when serialized in JSON, the format MUST adhere to the default pattern"), rspecs.Version)
	}
	return state, err
}

// Kill a container
func (r *Runtime) Kill(sig string) (err error) {
	var args []string
	args = append(args, "kill")
	if r.ID != "" {
		args = append(args, r.ID)
	}
	if sig != "" {
		// TODO: runc does not support this
		//	args = append(args, "--signal", sig)
		args = append(args, sig)
	} else {
		args = append(args, DefaultSignal)
	}

	cmd := exec.Command(r.RuntimeCommand, args...)
	return execWithStderrFallbackToStdout(cmd)
}

// Delete a container
func (r *Runtime) Delete() (err error) {
	var args []string
	args = append(args, "delete")
	if r.ID != "" {
		args = append(args, r.ID)
	}

	cmd := exec.Command(r.RuntimeCommand, args...)
	return execWithStderrFallbackToStdout(cmd)
}

// Clean deletes the container.  If removeBundle is set, the bundle
// directory is removed after the container is deleted successfully or, if
// forceRemoveBundle is true, after the deletion attempt regardless of
// whether it was successful or not.
func (r *Runtime) Clean(removeBundle bool, forceRemoveBundle bool) error {
	r.Kill("KILL")
	WaitingForStatus(*r, LifecycleStatusStopped, time.Second*10, time.Second/10)

	err := r.Delete()

	if removeBundle && (err == nil || forceRemoveBundle) {
		err2 := os.RemoveAll(r.bundleDir())
		if err2 != nil && err == nil {
			err = err2
		}
	}

	return err
}

func execWithStderrFallbackToStdout(cmd *exec.Cmd) (err error) {
	stdout, err := cmd.Output()
	if err == nil {
		return err
	}

	if e, ok := err.(*exec.ExitError); ok {
		if len(e.Stderr) == 0 {
			e.Stderr = stdout
			return e
		}
	}
	return err
}
<commit_msg>validation: fix Clean<commit_after>package util

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	rspecs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/opencontainers/runtime-tools/generate"
	"github.com/opencontainers/runtime-tools/specerror"
)

// Runtime represents the basic requirement of a container runtime
type Runtime struct {
	RuntimeCommand string
	BundleDir      string
	PidFile        string
	ID             string
	stdout         bytes.Buffer
	stderr         bytes.Buffer
}

// DefaultSignal represents the default signal sends to a container
const DefaultSignal = "TERM"

// NewRuntime create a runtime by command and the bundle directory
func NewRuntime(runtimeCommand string, bundleDir string) (Runtime, error) {
	var r Runtime
	var err error
	r.RuntimeCommand, err = exec.LookPath(runtimeCommand)
	if err != nil {
		return Runtime{}, err
	}

	r.BundleDir = bundleDir
	return r, err
}

// bundleDir returns the bundle directory.  Generally this is
// BundleDir, but when BundleDir is the empty string, it falls back to
// ., as specified in the CLI spec.
func (r *Runtime) bundleDir() (bundleDir string) {
	if r.BundleDir == "" {
		return "."
	}
	return r.BundleDir
}

// SetConfig creates a 'config.json' by the generator
func (r *Runtime) SetConfig(g *generate.Generator) error {
	if g == nil {
		return errors.New("cannot set a nil config")
	}
	return g.SaveToFile(filepath.Join(r.bundleDir(), "config.json"), generate.ExportOptions{})
}

// SetID sets the container ID
func (r *Runtime) SetID(id string) {
	r.ID = id
}

// Create a container
func (r *Runtime) Create() (err error) {
	var args []string
	args = append(args, "create")
	if r.PidFile != "" {
		args = append(args, "--pid-file", r.PidFile)
	}
	if r.BundleDir != "" {
		args = append(args, "--bundle", r.BundleDir)
	}
	if r.ID != "" {
		args = append(args, r.ID)
	}
	cmd := exec.Command(r.RuntimeCommand, args...)
	r.stdout.Reset()
	cmd.Stdout = &r.stdout
	r.stderr.Reset()
	cmd.Stderr = &r.stderr

	err = cmd.Run()
	if err == nil {
		return err
	}

	if e, ok := err.(*exec.ExitError); ok {
		stdout, stderr := r.StandardStreams()
		if len(stderr) == 0 {
			stderr = stdout
		}
		e.Stderr = stderr
		return e
	}
	return err
}

// StandardStreams returns content from the stdout and stderr buffers.
func (r *Runtime) StandardStreams() (stdout, stderr []byte) {
	return r.stdout.Bytes(), r.stderr.Bytes()
}

// Start a container
func (r *Runtime) Start() (err error) {
	var args []string
	args = append(args, "start")
	if r.ID != "" {
		args = append(args, r.ID)
	}

	cmd := exec.Command(r.RuntimeCommand, args...)
	return execWithStderrFallbackToStdout(cmd)
}

// State a container information
func (r *Runtime) State() (rspecs.State, error) {
	var args []string
	args = append(args, "state")
	if r.ID != "" {
		args = append(args, r.ID)
	}

	out, err := exec.Command(r.RuntimeCommand, args...).Output()
	if err != nil {
		if e, ok := err.(*exec.ExitError); ok {
			if len(e.Stderr) == 0 {
				e.Stderr = out
				return rspecs.State{}, e
			}
		}
		return rspecs.State{}, err
	}

	var state rspecs.State
	err = json.Unmarshal(out, &state)
	if err != nil {
		return rspecs.State{}, specerror.NewError(specerror.DefaultStateJSONPattern, fmt.Errorf("when serialized in JSON, the format MUST adhere to the default pattern"), rspecs.Version)
	}
	return state, err
}

// Kill a container
func (r *Runtime) Kill(sig string) (err error) {
	var args []string
	args = append(args, "kill")
	if r.ID != "" {
		args = append(args, r.ID)
	}
	if sig != "" {
		// TODO: runc does not support this
		//	args = append(args, "--signal", sig)
		args = append(args, sig)
	} else {
		args = append(args, DefaultSignal)
	}

	cmd := exec.Command(r.RuntimeCommand, args...)
	return execWithStderrFallbackToStdout(cmd)
}

// Delete a container
func (r *Runtime) Delete() (err error) {
	var args []string
	args = append(args, "delete")
	if r.ID != "" {
		args = append(args, r.ID)
	}

	cmd := exec.Command(r.RuntimeCommand, args...)
	return execWithStderrFallbackToStdout(cmd)
}

// Clean deletes the container.  If removeBundle is set, the bundle
// directory is removed after the container is deleted successfully or, if
// forceRemoveBundle is true, after the deletion attempt regardless of
// whether it was successful or not.
func (r *Runtime) Clean(removeBundle bool, forceRemoveBundle bool) {
	if err := r.Kill("KILL"); err != nil {
		fmt.Fprintf(os.Stderr, "Clean: Kill: %v", err)
	}
	if err := WaitingForStatus(*r, LifecycleStatusStopped, time.Second*10, time.Second/10); err != nil {
		fmt.Fprintf(os.Stderr, "Clean: %v", err)
	}

	err := r.Delete()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Clean: Delete: %v", err)
	}

	if removeBundle && (err == nil || forceRemoveBundle) {
		err := os.RemoveAll(r.bundleDir())
		if err != nil {
			fmt.Fprintf(os.Stderr, "Clean: %v", err)
		}
	}
}

func execWithStderrFallbackToStdout(cmd *exec.Cmd) (err error) {
	stdout, err := cmd.Output()
	if err == nil {
		return err
	}

	if e, ok := err.(*exec.ExitError); ok {
		if len(e.Stderr) == 0 {
			e.Stderr = stdout
			return e
		}
	}
	return err
}
<|endoftext|>


<commit_before>package vault

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/hashicorp/vault/logical"
	"github.com/hashicorp/vault/logical/framework"
)

// logical.Factory
func PassthroughBackendFactory(map[string]string) (logical.Backend, error) {
	var b PassthroughBackend
	b.Backend = &framework.Backend{
		Help: strings.TrimSpace(passthroughHelp),

		Paths: []*framework.Path{
			&framework.Path{
				Pattern: ".*",
				Fields: map[string]*framework.FieldSchema{
					"lease": &framework.FieldSchema{
						Type:        framework.TypeString,
						Description: "Lease time for this key when read. Ex: 1h",
					},
				},

				Callbacks: map[logical.Operation]framework.OperationFunc{
					logical.ReadOperation:   b.handleRead,
					logical.WriteOperation:  b.handleWrite,
					logical.DeleteOperation: b.handleDelete,
					logical.ListOperation:   b.handleList,
				},

				HelpSynopsis:    strings.TrimSpace(passthroughHelpSynopsis),
				HelpDescription: strings.TrimSpace(passthroughHelpDescription),
			},
		},

		Secrets: []*framework.Secret{
			&framework.Secret{
				Type: "generic",

				Renew:  b.handleRead,
				Revoke: b.handleRevoke,
			},
		},
	}

	return b, nil
}

// PassthroughBackend is used storing secrets directly into the physical
// backend. The secrest are encrypted in the durable storage and custom lease
// information can be specified, but otherwise this backend doesn't do anything
// fancy.
type PassthroughBackend struct {
	*framework.Backend
}

func (b *PassthroughBackend) handleRevoke(
	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	// This is a no-op
	return nil, nil
}

func (b *PassthroughBackend) handleRead(
	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	// Read the path
	out, err := req.Storage.Get(req.Path)
	if err != nil {
		return nil, fmt.Errorf("read failed: %v", err)
	}

	// Fast-path the no data case
	if out == nil {
		return nil, nil
	}

	// Decode the data
	var rawData map[string]interface{}
	if err := json.Unmarshal(out.Value, &rawData); err != nil {
		return nil, fmt.Errorf("json decoding failed: %v", err)
	}

	// Generate the response
	resp := b.Secret("generic").Response(rawData, nil)
	resp.Secret.Renewable = false

	// Check if there is a lease key
	leaseVal, ok := rawData["lease"].(string)
	if ok {
		leaseDuration, err := time.ParseDuration(leaseVal)
		if err == nil {
			resp.Secret.Renewable = true
			resp.Secret.Lease = leaseDuration
		}
	}

	return resp, nil
}

func (b *PassthroughBackend) handleWrite(
	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	// Check that some fields are given
	if len(req.Data) == 0 {
		return nil, fmt.Errorf("missing data fields")
	}

	// JSON encode the data
	buf, err := json.Marshal(req.Data)
	if err != nil {
		return nil, fmt.Errorf("json encoding failed: %v", err)
	}

	// Write out a new key
	entry := &logical.StorageEntry{
		Key:   req.Path,
		Value: buf,
	}
	if err := req.Storage.Put(entry); err != nil {
		return nil, fmt.Errorf("failed to write: %v", err)
	}

	return nil, nil
}

func (b *PassthroughBackend) handleDelete(
	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	// Delete the key at the request path
	if err := req.Storage.Delete(req.Path); err != nil {
		return nil, err
	}

	return nil, nil
}

func (b *PassthroughBackend) handleList(
	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	// List the keys at the prefix given by the request
	keys, err := req.Storage.List(req.Path)
	if err != nil {
		return nil, err
	}

	// Generate the response
	return logical.ListResponse(keys), nil
}

const passthroughHelp = `
The generic backend reads and writes arbitrary secrets to the backend.
The secrets are encrypted/decrypted by Vault: they are never stored
unencrypted in the backend and the backend never has an opportunity to
see the unencrypted value.

Leases can be set on a per-secret basis. These leases will be sent down
when that secret is read, and it is assumed that some outside process will
revoke and/or replace the secret at that path.
`

const passthroughHelpSynopsis = `
Pass-through secret storage to the physical backend, allowing you to
read/write arbitrary data into secret storage.
`

const passthroughHelpDescription = `
The pass-through backend reads and writes arbitrary data into secret storage,
encrypting it along the way.

A lease can be specified when writing with the "lease" field. If given, then
when the secret is read, Vault will report a lease with that duration. It
is expected that the consumer of this backend properly writes renewed keys
before the lease is up. In addition, revocation must be handled by the
user of this backend.
`
<commit_msg>vault: physical -> storage for clarity<commit_after>package vault

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/hashicorp/vault/logical"
	"github.com/hashicorp/vault/logical/framework"
)

// logical.Factory
func PassthroughBackendFactory(map[string]string) (logical.Backend, error) {
	var b PassthroughBackend
	b.Backend = &framework.Backend{
		Help: strings.TrimSpace(passthroughHelp),

		Paths: []*framework.Path{
			&framework.Path{
				Pattern: ".*",
				Fields: map[string]*framework.FieldSchema{
					"lease": &framework.FieldSchema{
						Type:        framework.TypeString,
						Description: "Lease time for this key when read. Ex: 1h",
					},
				},

				Callbacks: map[logical.Operation]framework.OperationFunc{
					logical.ReadOperation:   b.handleRead,
					logical.WriteOperation:  b.handleWrite,
					logical.DeleteOperation: b.handleDelete,
					logical.ListOperation:   b.handleList,
				},

				HelpSynopsis:    strings.TrimSpace(passthroughHelpSynopsis),
				HelpDescription: strings.TrimSpace(passthroughHelpDescription),
			},
		},

		Secrets: []*framework.Secret{
			&framework.Secret{
				Type: "generic",

				Renew:  b.handleRead,
				Revoke: b.handleRevoke,
			},
		},
	}

	return b, nil
}

// PassthroughBackend is used storing secrets directly into the physical
// backend. The secrest are encrypted in the durable storage and custom lease
// information can be specified, but otherwise this backend doesn't do anything
// fancy.
type PassthroughBackend struct {
	*framework.Backend
}

func (b *PassthroughBackend) handleRevoke(
	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	// This is a no-op
	return nil, nil
}

func (b *PassthroughBackend) handleRead(
	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	// Read the path
	out, err := req.Storage.Get(req.Path)
	if err != nil {
		return nil, fmt.Errorf("read failed: %v", err)
	}

	// Fast-path the no data case
	if out == nil {
		return nil, nil
	}

	// Decode the data
	var rawData map[string]interface{}
	if err := json.Unmarshal(out.Value, &rawData); err != nil {
		return nil, fmt.Errorf("json decoding failed: %v", err)
	}

	// Generate the response
	resp := b.Secret("generic").Response(rawData, nil)
	resp.Secret.Renewable = false

	// Check if there is a lease key
	leaseVal, ok := rawData["lease"].(string)
	if ok {
		leaseDuration, err := time.ParseDuration(leaseVal)
		if err == nil {
			resp.Secret.Renewable = true
			resp.Secret.Lease = leaseDuration
		}
	}

	return resp, nil
}

func (b *PassthroughBackend) handleWrite(
	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	// Check that some fields are given
	if len(req.Data) == 0 {
		return nil, fmt.Errorf("missing data fields")
	}

	// JSON encode the data
	buf, err := json.Marshal(req.Data)
	if err != nil {
		return nil, fmt.Errorf("json encoding failed: %v", err)
	}

	// Write out a new key
	entry := &logical.StorageEntry{
		Key:   req.Path,
		Value: buf,
	}
	if err := req.Storage.Put(entry); err != nil {
		return nil, fmt.Errorf("failed to write: %v", err)
	}

	return nil, nil
}

func (b *PassthroughBackend) handleDelete(
	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	// Delete the key at the request path
	if err := req.Storage.Delete(req.Path); err != nil {
		return nil, err
	}

	return nil, nil
}

func (b *PassthroughBackend) handleList(
	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	// List the keys at the prefix given by the request
	keys, err := req.Storage.List(req.Path)
	if err != nil {
		return nil, err
	}

	// Generate the response
	return logical.ListResponse(keys), nil
}

const passthroughHelp = `
The generic backend reads and writes arbitrary secrets to the backend.
The secrets are encrypted/decrypted by Vault: they are never stored
unencrypted in the backend and the backend never has an opportunity to
see the unencrypted value.

Leases can be set on a per-secret basis. These leases will be sent down
when that secret is read, and it is assumed that some outside process will
revoke and/or replace the secret at that path.
`

const passthroughHelpSynopsis = `
Pass-through secret storage to the storage backend, allowing you to
read/write arbitrary data into secret storage.
`

const passthroughHelpDescription = `
The pass-through backend reads and writes arbitrary data into secret storage,
encrypting it along the way.

A lease can be specified when writing with the "lease" field. If given, then
when the secret is read, Vault will report a lease with that duration. It
is expected that the consumer of this backend properly writes renewed keys
before the lease is up. In addition, revocation must be handled by the
user of this backend.
`
<|endoftext|>


<commit_before>/* Copyright (c) 2016 Jason Ish
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

package resources

import (
	"bytes"
	"fmt"
	"github.com/jasonish/evebox/log"
	"io"
	"net/http"
	"strings"
)

//go:generate go-bindata -pkg resources -ignore bindata\.go ./...

// AssetString returns an asset as a string.
func AssetString(name string) (string, error) {
	bytes, err := Asset(name)
	if err != nil {
		return "", err
	}
	return string(bytes), nil
}

// GetReader returns an asset as a reader.
func GetReader(name string) (io.Reader, error) {
	data, err := Asset(name)
	if err != nil {
		return nil, err
	}
	return bytes.NewReader(data), nil
}

type FileServer struct {
}

func (s FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var path string

	if r.URL.String() == "/" {
		path = "index.html"
	} else {
		path = strings.TrimPrefix(r.URL.String(), "/")
	}

	log.Info("Static file request for %s.", path)

	asset := fmt.Sprintf("public/%s", path)
	bytes, err := Asset(asset)
	if err != nil {
		log.Error("Public file not found: %s", path)
		w.WriteHeader(http.StatusNotFound)
	} else {
		w.Write(bytes)
	}
}
<commit_msg>handle static file request with query params...<commit_after>/* Copyright (c) 2016 Jason Ish
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

package resources

import (
	"bytes"
	"fmt"
	"github.com/jasonish/evebox/log"
	"io"
	"net/http"
	"strings"
)

//go:generate go-bindata -pkg resources -ignore bindata\.go ./...

// AssetString returns an asset as a string.
func AssetString(name string) (string, error) {
	bytes, err := Asset(name)
	if err != nil {
		return "", err
	}
	return string(bytes), nil
}

// GetReader returns an asset as a reader.
func GetReader(name string) (io.Reader, error) {
	data, err := Asset(name)
	if err != nil {
		return nil, err
	}
	return bytes.NewReader(data), nil
}

type FileServer struct {
}

func (s FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var path string

	if r.URL.String() == "/" {
		path = "index.html"
	} else {
		path = strings.TrimPrefix(r.URL.String(), "/")
	}

	// Remove any query string parameters...
	parts := strings.SplitN(path, "?", 2)
	path = parts[0]

	log.Info("Static file request for %s.", path)

	asset := fmt.Sprintf("public/%s", path)
	bytes, err := Asset(asset)
	if err != nil {
		log.Error("Public file not found: %s", path)
		w.WriteHeader(http.StatusNotFound)
	} else {
		w.Write(bytes)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"time"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	prowv1 "k8s.io/test-infra/prow/client/clientset/versioned/typed/prowjobs/v1"
	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/flagutil"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/logrusutil"
	"k8s.io/test-infra/prow/pjutil"
)

type options struct {
	runOnce       bool
	configPath    string
	jobConfigPath string
	dryRun        flagutil.Bool
	kubernetes    flagutil.ExperimentalKubernetesOptions
}

const (
	// TODO(fejta): require setting this explicitly
	defaultConfigPath = "/etc/config/config.yaml"

	reasonAged     = "aged"
	reasonOrphaned = "orphaned"
)

func gatherOptions(fs *flag.FlagSet, args ...string) options {
	o := options{}
	fs.BoolVar(&o.runOnce, "run-once", false, "If true, run only once then quit.")
	fs.StringVar(&o.configPath, "config-path", defaultConfigPath, "Path to config.yaml.")
	fs.StringVar(&o.jobConfigPath, "job-config-path", "", "Path to prow job configs.")

	// TODO(fejta): switch dryRun to be a bool, defaulting to true after March 15, 2019.
	fs.Var(&o.dryRun, "dry-run", "Whether or not to make mutating API calls to Kubernetes.")

	o.kubernetes.AddFlags(fs)
	fs.Parse(args)
	return o
}

func (o *options) Validate() error {
	if err := o.kubernetes.Validate(o.dryRun.Value); err != nil {
		return err
	}

	if o.configPath == "" {
		return errors.New("--config-path is required")
	}

	return nil
}

func main() {
	o := gatherOptions(flag.NewFlagSet(os.Args[0], flag.ExitOnError), os.Args[1:]...)
	if err := o.Validate(); err != nil {
		logrus.WithError(err).Fatal("Invalid options")
	}

	// for pprof endpoints
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()

	logrus.SetFormatter(
		logrusutil.NewDefaultFieldsFormatter(nil, logrus.Fields{"component": "sinker"}),
	)
	if !o.dryRun.Explicit {
		logrus.Warning("Sinker requies --dry-run=false to function correctly in production.")
		logrus.Warning("--dry-run will soon default to true. Set --dry-run=false by March 15.")
	}

	configAgent := &config.Agent{}
	if err := configAgent.Start(o.configPath, o.jobConfigPath); err != nil {
		logrus.WithError(err).Fatal("Error starting config agent.")
	}
	cfg := configAgent.Config

	prowJobClient, err := o.kubernetes.ProwJobClient(cfg().ProwJobNamespace, o.dryRun.Value)
	if err != nil {
		logrus.WithError(err).Fatal("Error creating ProwJob client.")
	}

	buildClusterClients, err := o.kubernetes.BuildClusterClients(cfg().PodNamespace, o.dryRun.Value)
	if err != nil {
		logrus.WithError(err).Fatal("Error creating build cluster clients.")
	}

	var podClients []corev1.PodInterface
	for _, client := range buildClusterClients {
		// sinker doesn't care about build cluster aliases
		podClients = append(podClients, client)
	}

	c := controller{
		logger:        logrus.NewEntry(logrus.StandardLogger()),
		prowJobClient: prowJobClient,
		podClients:    podClients,
		config:        cfg,
	}

	// Clean now and regularly from now on.
	for {
		start := time.Now()
		c.clean()
		logrus.Infof("Sync time: %v", time.Since(start))
		if o.runOnce {
			break
		}
		time.Sleep(cfg().Sinker.ResyncPeriod)
	}
}

type controller struct {
	logger        *logrus.Entry
	prowJobClient prowv1.ProwJobInterface
	podClients    []corev1.PodInterface
	config        config.Getter
}

type sinkerReconciliationMetrics struct {
	podsCreated int
	startAt     time.Time
	finishedAt  time.Time
	podsRemoved map[string]int
}

func (m *sinkerReconciliationMetrics) getPodsTotalRemoved() int {
	result := 0
	for _, v := range m.podsRemoved {
		result += v
	}
	return result
}

func (m *sinkerReconciliationMetrics) logFields() logrus.Fields {
	return logrus.Fields{
		"podsCreated":      m.podsCreated,
		"timeUsed":         m.finishedAt.Sub(m.startAt),
		"podsTotalRemoved": m.getPodsTotalRemoved(),
		"podsRemoved":      m.podsRemoved,
	}
}

func (c *controller) clean() {

	metrics := sinkerReconciliationMetrics{podsRemoved: make(map[string]int)}
	metrics.startAt = time.Now()

	// Clean up old prow jobs first.
	prowJobs, err := c.prowJobClient.List(metav1.ListOptions{})
	if err != nil {
		c.logger.WithError(err).Error("Error listing prow jobs.")
		return
	}

	// Only delete pod if its prowjob is marked as finished
	isExist := sets.NewString()
	isFinished := sets.NewString()

	maxProwJobAge := c.config().Sinker.MaxProwJobAge
	for _, prowJob := range prowJobs.Items {
		isExist.Insert(prowJob.ObjectMeta.Name)
		// Handle periodics separately.
		if prowJob.Spec.Type == prowapi.PeriodicJob {
			continue
		}
		if !prowJob.Complete() {
			continue
		}
		isFinished.Insert(prowJob.ObjectMeta.Name)
		if time.Since(prowJob.Status.StartTime.Time) <= maxProwJobAge {
			continue
		}
		if err := c.prowJobClient.Delete(prowJob.ObjectMeta.Name, &metav1.DeleteOptions{}); err == nil {
			c.logger.WithFields(pjutil.ProwJobFields(&prowJob)).Info("Deleted prowjob.")
		} else {
			c.logger.WithFields(pjutil.ProwJobFields(&prowJob)).WithError(err).Error("Error deleting prowjob.")
		}
	}

	// Keep track of what periodic jobs are in the config so we will
	// not clean up their last prowjob.
	isActivePeriodic := make(map[string]bool)
	for _, p := range c.config().Periodics {
		isActivePeriodic[p.Name] = true
	}

	// Get the jobs that we need to retain so horologium can continue working
	// as intended.
	latestPeriodics := pjutil.GetLatestProwJobs(prowJobs.Items, prowapi.PeriodicJob)
	for _, prowJob := range prowJobs.Items {
		if prowJob.Spec.Type != prowapi.PeriodicJob {
			continue
		}

		latestPJ := latestPeriodics[prowJob.Spec.Job]
		if isActivePeriodic[prowJob.Spec.Job] && prowJob.ObjectMeta.Name == latestPJ.ObjectMeta.Name {
			// Ignore deleting this one.
			continue
		}
		if !prowJob.Complete() {
			continue
		}
		isFinished.Insert(prowJob.ObjectMeta.Name)
		if time.Since(prowJob.Status.StartTime.Time) <= maxProwJobAge {
			continue
		}
		if err := c.prowJobClient.Delete(prowJob.ObjectMeta.Name, &metav1.DeleteOptions{}); err == nil {
			c.logger.WithFields(pjutil.ProwJobFields(&prowJob)).Info("Deleted prowjob.")
		} else {
			c.logger.WithFields(pjutil.ProwJobFields(&prowJob)).WithError(err).Error("Error deleting prowjob.")
		}
	}

	// Now clean up old pods.
	selector := fmt.Sprintf("%s = %s", kube.CreatedByProw, "true")
	for _, client := range c.podClients {
		pods, err := client.List(metav1.ListOptions{LabelSelector: selector})
		if err != nil {
			c.logger.WithError(err).Error("Error listing pods.")
			return
		}
		metrics.podsCreated = len(pods.Items)
		maxPodAge := c.config().Sinker.MaxPodAge
		for _, pod := range pods.Items {
			clean := !pod.Status.StartTime.IsZero() && time.Since(pod.Status.StartTime.Time) > maxPodAge
			reason := reasonAged
			if !isFinished.Has(pod.ObjectMeta.Name) {
				// prowjob exists and is not marked as completed yet
				// deleting the pod now will result in plank creating a brand new pod
				clean = false
			}
			if !isExist.Has(pod.ObjectMeta.Name) {
				// prowjob has gone, we want to clean orphan pods regardless of the state
				reason = reasonOrphaned
				clean = true
			}

			if !clean {
				continue
			}

			// Delete old finished or orphan pods. Don't quit if we fail to delete one.
			if err := client.Delete(pod.ObjectMeta.Name, &metav1.DeleteOptions{}); err == nil {
				c.logger.WithField("pod", pod.ObjectMeta.Name).Info("Deleted old completed pod.")
				metrics.podsRemoved[reason]++
			} else {
				c.logger.WithField("pod", pod.ObjectMeta.Name).WithError(err).Error("Error deleting pod.")
			}
		}
	}

	metrics.finishedAt = time.Now()
	c.logger.WithFields(metrics.logFields()).Info("Show sinkerReconciliationMetrics.")
}
<commit_msg>Better logging msg<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"time"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	prowv1 "k8s.io/test-infra/prow/client/clientset/versioned/typed/prowjobs/v1"
	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/flagutil"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/logrusutil"
	"k8s.io/test-infra/prow/pjutil"
)

type options struct {
	runOnce       bool
	configPath    string
	jobConfigPath string
	dryRun        flagutil.Bool
	kubernetes    flagutil.ExperimentalKubernetesOptions
}

const (
	// TODO(fejta): require setting this explicitly
	defaultConfigPath = "/etc/config/config.yaml"

	reasonAged     = "aged"
	reasonOrphaned = "orphaned"
)

func gatherOptions(fs *flag.FlagSet, args ...string) options {
	o := options{}
	fs.BoolVar(&o.runOnce, "run-once", false, "If true, run only once then quit.")
	fs.StringVar(&o.configPath, "config-path", defaultConfigPath, "Path to config.yaml.")
	fs.StringVar(&o.jobConfigPath, "job-config-path", "", "Path to prow job configs.")

	// TODO(fejta): switch dryRun to be a bool, defaulting to true after March 15, 2019.
	fs.Var(&o.dryRun, "dry-run", "Whether or not to make mutating API calls to Kubernetes.")

	o.kubernetes.AddFlags(fs)
	fs.Parse(args)
	return o
}

func (o *options) Validate() error {
	if err := o.kubernetes.Validate(o.dryRun.Value); err != nil {
		return err
	}

	if o.configPath == "" {
		return errors.New("--config-path is required")
	}

	return nil
}

func main() {
	o := gatherOptions(flag.NewFlagSet(os.Args[0], flag.ExitOnError), os.Args[1:]...)
	if err := o.Validate(); err != nil {
		logrus.WithError(err).Fatal("Invalid options")
	}

	// for pprof endpoints
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()

	logrus.SetFormatter(
		logrusutil.NewDefaultFieldsFormatter(nil, logrus.Fields{"component": "sinker"}),
	)
	if !o.dryRun.Explicit {
		logrus.Warning("Sinker requies --dry-run=false to function correctly in production.")
		logrus.Warning("--dry-run will soon default to true. Set --dry-run=false by March 15.")
	}

	configAgent := &config.Agent{}
	if err := configAgent.Start(o.configPath, o.jobConfigPath); err != nil {
		logrus.WithError(err).Fatal("Error starting config agent.")
	}
	cfg := configAgent.Config

	prowJobClient, err := o.kubernetes.ProwJobClient(cfg().ProwJobNamespace, o.dryRun.Value)
	if err != nil {
		logrus.WithError(err).Fatal("Error creating ProwJob client.")
	}

	buildClusterClients, err := o.kubernetes.BuildClusterClients(cfg().PodNamespace, o.dryRun.Value)
	if err != nil {
		logrus.WithError(err).Fatal("Error creating build cluster clients.")
	}

	var podClients []corev1.PodInterface
	for _, client := range buildClusterClients {
		// sinker doesn't care about build cluster aliases
		podClients = append(podClients, client)
	}

	c := controller{
		logger:        logrus.NewEntry(logrus.StandardLogger()),
		prowJobClient: prowJobClient,
		podClients:    podClients,
		config:        cfg,
	}

	// Clean now and regularly from now on.
	for {
		start := time.Now()
		c.clean()
		logrus.Infof("Sync time: %v", time.Since(start))
		if o.runOnce {
			break
		}
		time.Sleep(cfg().Sinker.ResyncPeriod)
	}
}

type controller struct {
	logger        *logrus.Entry
	prowJobClient prowv1.ProwJobInterface
	podClients    []corev1.PodInterface
	config        config.Getter
}

type sinkerReconciliationMetrics struct {
	podsCreated int
	startAt     time.Time
	finishedAt  time.Time
	podsRemoved map[string]int
}

func (m *sinkerReconciliationMetrics) getPodsTotalRemoved() int {
	result := 0
	for _, v := range m.podsRemoved {
		result += v
	}
	return result
}

func (m *sinkerReconciliationMetrics) logFields() logrus.Fields {
	return logrus.Fields{
		"podsCreated":      m.podsCreated,
		"timeUsed":         m.finishedAt.Sub(m.startAt),
		"podsTotalRemoved": m.getPodsTotalRemoved(),
		"podsRemoved":      m.podsRemoved,
	}
}

func (c *controller) clean() {

	metrics := sinkerReconciliationMetrics{podsRemoved: make(map[string]int)}
	metrics.startAt = time.Now()

	// Clean up old prow jobs first.
	prowJobs, err := c.prowJobClient.List(metav1.ListOptions{})
	if err != nil {
		c.logger.WithError(err).Error("Error listing prow jobs.")
		return
	}

	// Only delete pod if its prowjob is marked as finished
	isExist := sets.NewString()
	isFinished := sets.NewString()

	maxProwJobAge := c.config().Sinker.MaxProwJobAge
	for _, prowJob := range prowJobs.Items {
		isExist.Insert(prowJob.ObjectMeta.Name)
		// Handle periodics separately.
		if prowJob.Spec.Type == prowapi.PeriodicJob {
			continue
		}
		if !prowJob.Complete() {
			continue
		}
		isFinished.Insert(prowJob.ObjectMeta.Name)
		if time.Since(prowJob.Status.StartTime.Time) <= maxProwJobAge {
			continue
		}
		if err := c.prowJobClient.Delete(prowJob.ObjectMeta.Name, &metav1.DeleteOptions{}); err == nil {
			c.logger.WithFields(pjutil.ProwJobFields(&prowJob)).Info("Deleted prowjob.")
		} else {
			c.logger.WithFields(pjutil.ProwJobFields(&prowJob)).WithError(err).Error("Error deleting prowjob.")
		}
	}

	// Keep track of what periodic jobs are in the config so we will
	// not clean up their last prowjob.
	isActivePeriodic := make(map[string]bool)
	for _, p := range c.config().Periodics {
		isActivePeriodic[p.Name] = true
	}

	// Get the jobs that we need to retain so horologium can continue working
	// as intended.
	latestPeriodics := pjutil.GetLatestProwJobs(prowJobs.Items, prowapi.PeriodicJob)
	for _, prowJob := range prowJobs.Items {
		if prowJob.Spec.Type != prowapi.PeriodicJob {
			continue
		}

		latestPJ := latestPeriodics[prowJob.Spec.Job]
		if isActivePeriodic[prowJob.Spec.Job] && prowJob.ObjectMeta.Name == latestPJ.ObjectMeta.Name {
			// Ignore deleting this one.
			continue
		}
		if !prowJob.Complete() {
			continue
		}
		isFinished.Insert(prowJob.ObjectMeta.Name)
		if time.Since(prowJob.Status.StartTime.Time) <= maxProwJobAge {
			continue
		}
		if err := c.prowJobClient.Delete(prowJob.ObjectMeta.Name, &metav1.DeleteOptions{}); err == nil {
			c.logger.WithFields(pjutil.ProwJobFields(&prowJob)).Info("Deleted prowjob.")
		} else {
			c.logger.WithFields(pjutil.ProwJobFields(&prowJob)).WithError(err).Error("Error deleting prowjob.")
		}
	}

	// Now clean up old pods.
	selector := fmt.Sprintf("%s = %s", kube.CreatedByProw, "true")
	for _, client := range c.podClients {
		pods, err := client.List(metav1.ListOptions{LabelSelector: selector})
		if err != nil {
			c.logger.WithError(err).Error("Error listing pods.")
			return
		}
		metrics.podsCreated = len(pods.Items)
		maxPodAge := c.config().Sinker.MaxPodAge
		for _, pod := range pods.Items {
			clean := !pod.Status.StartTime.IsZero() && time.Since(pod.Status.StartTime.Time) > maxPodAge
			reason := reasonAged
			if !isFinished.Has(pod.ObjectMeta.Name) {
				// prowjob exists and is not marked as completed yet
				// deleting the pod now will result in plank creating a brand new pod
				clean = false
			}
			if !isExist.Has(pod.ObjectMeta.Name) {
				// prowjob has gone, we want to clean orphan pods regardless of the state
				reason = reasonOrphaned
				clean = true
			}

			if !clean {
				continue
			}

			// Delete old finished or orphan pods. Don't quit if we fail to delete one.
			if err := client.Delete(pod.ObjectMeta.Name, &metav1.DeleteOptions{}); err == nil {
				c.logger.WithField("pod", pod.ObjectMeta.Name).Info("Deleted old completed pod.")
				metrics.podsRemoved[reason]++
			} else {
				c.logger.WithField("pod", pod.ObjectMeta.Name).WithError(err).Error("Error deleting pod.")
			}
		}
	}

	metrics.finishedAt = time.Now()
	c.logger.WithFields(metrics.logFields()).Info("Sinker reconciliation complete.")
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package dog adds dog images to issues in response to a /woof comment
package dog

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strings"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/pluginhelp"
	"k8s.io/test-infra/prow/plugins"
)

var (
	match = regexp.MustCompile(`(?mi)^/(woof|bark)\s*$`)
)

const (
	dogURL     = realPack("https://random.dog/woof.json")
	pluginName = "dog"
)

func init() {
	plugins.RegisterGenericCommentHandler(pluginName, handleGenericComment, helpProvider)
}

func helpProvider(config *plugins.Configuration, enabledRepos []string) (*pluginhelp.PluginHelp, error) {
	// The Config field is omitted because this plugin is not configurable.
	// TODO(qhuynh96): Removes all the fields of pluginHelp except Description.
	pluginHelp := &pluginhelp.PluginHelp{
		Description: "The dog plugin adds a dog image to an issue in response to the `/woof` command.",
		WhoCanUse:   "Anyone",
		Usage:       "/(woof|bark)",
		Examples:    []string{"/woof", "/bark"},
	}
	pluginHelp.AddCommand(pluginhelp.Command{
		Usage:       "/woof | /bark",
		Description: "Add a dog image to the issue",
		Featured:    false,
		WhoCanUse:   "Anyone",
		Examples:    []string{"/woof", "/bark"},
	})
	return pluginHelp, nil
}

type githubClient interface {
	CreateComment(owner, repo string, number int, comment string) error
}

type pack interface {
	readDog() (string, error)
}

type realPack string

var client = http.Client{}

type dogResult struct {
	URL string `json:"url"`
}

func (dr dogResult) Format() (string, error) {
	if dr.URL == "" {
		return "", errors.New("empty url")
	}
	src, err := url.ParseRequestURI(dr.URL)
	if err != nil {
		return "", fmt.Errorf("invalid url %s: %v", dr.URL, err)
	}
	return fmt.Sprintf("[![dog image](%s)](%s)", src, src), nil
}

func (u realPack) readDog() (string, error) {
	uri := string(u)
	req, err := http.NewRequest("GET", uri, nil)
	if err != nil {
		return "", fmt.Errorf("could not create request %s: %v", uri, err)
	}
	req.Header.Add("Accept", "application/json")
	resp, err := client.Do(req)
	if err != nil {
		return "", fmt.Errorf("could not read dog from %s: %v", uri, err)
	}
	defer resp.Body.Close()
	var a dogResult
	if err = json.NewDecoder(resp.Body).Decode(&a); err != nil {
		return "", err
	}
	// GitHub doesn't support videos :(
	if strings.HasSuffix(a.URL, ".mp4") {
		return "", errors.New("unsupported doggo :( github doesn't support .mp4")
	}
	return a.Format()
}

func handleGenericComment(pc plugins.PluginClient, e github.GenericCommentEvent) error {
	return handle(pc.GitHubClient, pc.Logger, &e, dogURL)
}

func handle(gc githubClient, log *logrus.Entry, e *github.GenericCommentEvent, p pack) error {
	// Only consider new comments.
	if e.Action != github.GenericCommentActionCreated {
		return nil
	}
	// Make sure they are requesting a dog
	mat := match.FindStringSubmatch(e.Body)
	if mat == nil {
		return nil
	}

	org := e.Repo.Owner.Login
	repo := e.Repo.Name
	number := e.Number

	for i := 0; i < 5; i++ {
		resp, err := p.readDog()
		if err != nil {
			log.WithError(err).Println("Failed to get dog img")
			continue
		}
		return gc.CreateComment(org, repo, number, plugins.FormatResponseRaw(e.Body, e.HTMLURL, e.User.Login, resp))
	}

	return errors.New("could not find a valid dog image")
}
<commit_msg>fix dog plugin usage in commands<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package dog adds dog images to issues in response to a /woof comment
package dog

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strings"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/pluginhelp"
	"k8s.io/test-infra/prow/plugins"
)

var (
	match = regexp.MustCompile(`(?mi)^/(woof|bark)\s*$`)
)

const (
	dogURL     = realPack("https://random.dog/woof.json")
	pluginName = "dog"
)

func init() {
	plugins.RegisterGenericCommentHandler(pluginName, handleGenericComment, helpProvider)
}

func helpProvider(config *plugins.Configuration, enabledRepos []string) (*pluginhelp.PluginHelp, error) {
	// The Config field is omitted because this plugin is not configurable.
	// TODO(qhuynh96): Removes all the fields of pluginHelp except Description.
	pluginHelp := &pluginhelp.PluginHelp{
		Description: "The dog plugin adds a dog image to an issue in response to the `/woof` command.",
		WhoCanUse:   "Anyone",
		Usage:       "/(woof|bark)",
		Examples:    []string{"/woof", "/bark"},
	}
	pluginHelp.AddCommand(pluginhelp.Command{
		Usage:       "/(woof|bark)",
		Description: "Add a dog image to the issue",
		Featured:    false,
		WhoCanUse:   "Anyone",
		Examples:    []string{"/woof", "/bark"},
	})
	return pluginHelp, nil
}

type githubClient interface {
	CreateComment(owner, repo string, number int, comment string) error
}

type pack interface {
	readDog() (string, error)
}

type realPack string

var client = http.Client{}

type dogResult struct {
	URL string `json:"url"`
}

func (dr dogResult) Format() (string, error) {
	if dr.URL == "" {
		return "", errors.New("empty url")
	}
	src, err := url.ParseRequestURI(dr.URL)
	if err != nil {
		return "", fmt.Errorf("invalid url %s: %v", dr.URL, err)
	}
	return fmt.Sprintf("[![dog image](%s)](%s)", src, src), nil
}

func (u realPack) readDog() (string, error) {
	uri := string(u)
	req, err := http.NewRequest("GET", uri, nil)
	if err != nil {
		return "", fmt.Errorf("could not create request %s: %v", uri, err)
	}
	req.Header.Add("Accept", "application/json")
	resp, err := client.Do(req)
	if err != nil {
		return "", fmt.Errorf("could not read dog from %s: %v", uri, err)
	}
	defer resp.Body.Close()
	var a dogResult
	if err = json.NewDecoder(resp.Body).Decode(&a); err != nil {
		return "", err
	}
	// GitHub doesn't support videos :(
	if strings.HasSuffix(a.URL, ".mp4") {
		return "", errors.New("unsupported doggo :( github doesn't support .mp4")
	}
	return a.Format()
}

func handleGenericComment(pc plugins.PluginClient, e github.GenericCommentEvent) error {
	return handle(pc.GitHubClient, pc.Logger, &e, dogURL)
}

func handle(gc githubClient, log *logrus.Entry, e *github.GenericCommentEvent, p pack) error {
	// Only consider new comments.
	if e.Action != github.GenericCommentActionCreated {
		return nil
	}
	// Make sure they are requesting a dog
	mat := match.FindStringSubmatch(e.Body)
	if mat == nil {
		return nil
	}

	org := e.Repo.Owner.Login
	repo := e.Repo.Name
	number := e.Number

	for i := 0; i < 5; i++ {
		resp, err := p.readDog()
		if err != nil {
			log.WithError(err).Println("Failed to get dog img")
			continue
		}
		return gc.CreateComment(org, repo, number, plugins.FormatResponseRaw(e.Body, e.HTMLURL, e.User.Login, resp))
	}

	return errors.New("could not find a valid dog image")
}
<|endoftext|>


<commit_before>// Package logdb provides an efficient log-structured database supporting efficient insertion of new entries
// and removal from either end of the log.
package logdb

// A LogDB is a log-structured database.
type LogDB interface {
	// Append writes a new entry to the log.
	//
	// , and 'WriteError' value if the
	// database files could not be written to.
	Append(entry []byte) error

	// AppendEntries atomically writes a collection of new entries to the log.
	//
	// Returns the same errors as 'Append', and an 'AtomicityError' value if any entry fails to
	// append and rolling back the log failed.
	AppendEntries(entries [][]byte) error

	// Get looks up an entry by ID.
	//
	// Returns 'ErrIDOutOfRange' if the requested ID is lesser than the oldest or greater than the
	// newest.
	Get(id uint64) ([]byte, error)

	// Forget removes entries from the end of the log.
	//
	// If the new "oldest" ID is older than the current, this is a no-op.
	//
	// Returns 'ErrIDOutOfRange' if the ID is newer than the "newest" ID.
	Forget(newOldestID uint64) error

	// Rollback removes entries from the head of the log.
	//
	// If the new "newest" ID is newer than the current, this is a no-op.
	//
	// Returns the same errors as 'Forget', with 'ErrIDOutOfRange' being returned if the ID is older
	// than the "oldest" ID.
	Rollback(newNewestID uint64) error

	// Truncate performs a 'Forget' followed by a 'Rollback' atomically. The semantics are that if
	// the 'Forget' fails, the 'Rollback' is not performed; but the 'Forget' is not undone either.
	//
	// Returns the same errors as 'Forget' and 'Rollback', and also an 'ErrIDOutOfRange' if the new
	// newest < the new oldest.
	Truncate(newOldestID, newNewestID uint64) error

	// OldestID gets the ID of the oldest log entry.
	//
	// For an empty database, this will return 0.
	OldestID() uint64

	// NewestID gets the ID of the newest log entry.
	//
	// For an empty database, this will return 0.
	NewestID() uint64
}

// A PersistDB is a database which can be persisted in some fashion. In addition to defining methods, a
// 'PersistDB' which is also a 'LogDB' changes some existing behaviours:
//
//  - 'Append', 'AppendEntries', 'Forget', 'Rollback', and 'Truncate' can now cause a 'Sync', if
//    'SetSync' has been called.
//
//  - The above may return a 'SyncError' value if a periodic synchronisation failed.
type PersistDB interface {
	// SetSync configures the database to synchronise the data after touching (appending, forgetting,
	// or rolling back) at most this many entries.
	//
	// <0 disables periodic syncing, and 'Sync' must be called instead. The default value is 256.
	// Both 0 and 1 cause a 'Sync' after every write.
	//
	// Returns a 'SyncError' value if this triggered an immediate synchronisation which failed, and
	// 'ErrClosed' if the handle is closed.
	SetSync(every int) error

	// Sync persists the data now.
	//
	// May return a SyncError value, and 'ErrClosed' if the handle is closed.
	Sync() error
}

// A BoundedDB has a maximum entry size. A 'BoundedDB' which is also a 'LogDB' changes the behaviour of
// 'Append' and 'AppendEntries': they now return 'ErrTooBig' if an entry appended is larger than the maximum
// size.
type BoundedDB interface {
	// The maximum size of an entry. It is an error to try to insert an entry larger than this.
	MaxEntrySize() uint64
}

// A CloseDB can be closed, which may perform some clean-up.
//
// If a 'CloseDB' is also a 'PersistDB', then 'Sync' should be called during 'Close'. In addition, all
// 'LogDB' and 'PersistDB' with an error return will fail after calling 'Close', with 'ErrClosed'.
type CloseDB interface {
	// Close performs some database-specific clean-up. It is an error to try to use a database after
	// closing it.
	//
	// Returns 'ErrClosed' if the database is already closed.
	Close() error
}
<commit_msg>Malformed comment.<commit_after>// Package logdb provides an efficient log-structured database supporting efficient insertion of new entries
// and removal from either end of the log.
package logdb

// A LogDB is a log-structured database.
type LogDB interface {
	// Append writes a new entry to the log.
	//
	// Returns 'WriteError' value if the database files could not be written to.
	Append(entry []byte) error

	// AppendEntries atomically writes a collection of new entries to the log.
	//
	// Returns the same errors as 'Append', and an 'AtomicityError' value if any entry fails to
	// append and rolling back the log failed.
	AppendEntries(entries [][]byte) error

	// Get looks up an entry by ID.
	//
	// Returns 'ErrIDOutOfRange' if the requested ID is lesser than the oldest or greater than the
	// newest.
	Get(id uint64) ([]byte, error)

	// Forget removes entries from the end of the log.
	//
	// If the new "oldest" ID is older than the current, this is a no-op.
	//
	// Returns 'ErrIDOutOfRange' if the ID is newer than the "newest" ID.
	Forget(newOldestID uint64) error

	// Rollback removes entries from the head of the log.
	//
	// If the new "newest" ID is newer than the current, this is a no-op.
	//
	// Returns the same errors as 'Forget', with 'ErrIDOutOfRange' being returned if the ID is older
	// than the "oldest" ID.
	Rollback(newNewestID uint64) error

	// Truncate performs a 'Forget' followed by a 'Rollback' atomically. The semantics are that if
	// the 'Forget' fails, the 'Rollback' is not performed; but the 'Forget' is not undone either.
	//
	// Returns the same errors as 'Forget' and 'Rollback', and also an 'ErrIDOutOfRange' if the new
	// newest < the new oldest.
	Truncate(newOldestID, newNewestID uint64) error

	// OldestID gets the ID of the oldest log entry.
	//
	// For an empty database, this will return 0.
	OldestID() uint64

	// NewestID gets the ID of the newest log entry.
	//
	// For an empty database, this will return 0.
	NewestID() uint64
}

// A PersistDB is a database which can be persisted in some fashion. In addition to defining methods, a
// 'PersistDB' which is also a 'LogDB' changes some existing behaviours:
//
//  - 'Append', 'AppendEntries', 'Forget', 'Rollback', and 'Truncate' can now cause a 'Sync', if
//    'SetSync' has been called.
//
//  - The above may return a 'SyncError' value if a periodic synchronisation failed.
type PersistDB interface {
	// SetSync configures the database to synchronise the data after touching (appending, forgetting,
	// or rolling back) at most this many entries.
	//
	// <0 disables periodic syncing, and 'Sync' must be called instead. The default value is 256.
	// Both 0 and 1 cause a 'Sync' after every write.
	//
	// Returns a 'SyncError' value if this triggered an immediate synchronisation which failed, and
	// 'ErrClosed' if the handle is closed.
	SetSync(every int) error

	// Sync persists the data now.
	//
	// May return a SyncError value, and 'ErrClosed' if the handle is closed.
	Sync() error
}

// A BoundedDB has a maximum entry size. A 'BoundedDB' which is also a 'LogDB' changes the behaviour of
// 'Append' and 'AppendEntries': they now return 'ErrTooBig' if an entry appended is larger than the maximum
// size.
type BoundedDB interface {
	// The maximum size of an entry. It is an error to try to insert an entry larger than this.
	MaxEntrySize() uint64
}

// A CloseDB can be closed, which may perform some clean-up.
//
// If a 'CloseDB' is also a 'PersistDB', then 'Sync' should be called during 'Close'. In addition, all
// 'LogDB' and 'PersistDB' with an error return will fail after calling 'Close', with 'ErrClosed'.
type CloseDB interface {
	// Close performs some database-specific clean-up. It is an error to try to use a database after
	// closing it.
	//
	// Returns 'ErrClosed' if the database is already closed.
	Close() error
}
<|endoftext|>