<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"io/ioutil"

	"golang.org/x/net/context"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol"
	rpc "github.com/keybase/go-framed-msgpack-rpc"
)

func NewCmdVerify(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:  "verify",
		Usage: "Verify message or file signatures for keybase users",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdVerify{Contextified: libkb.NewContextified(g)}, "verify", c)
		},
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "d, detached",
				Usage: "Specify a detached signature file.",
			},
			cli.StringFlag{
				Name:  "i, infile",
				Usage: "Specify an input file.",
			},
			cli.StringFlag{
				Name:  "m, message",
				Usage: "Provide the message to verify on the command line.",
			},
			cli.StringFlag{
				Name:  "S, signed-by",
				Usage: "Assert signed by the given user (can use user assertion format).",
			},
		},
	}
}

type CmdVerify struct {
	libkb.Contextified
	UnixFilter
	detachedData []byte
	signedBy     string
}

func (c *CmdVerify) ParseArgv(ctx *cli.Context) error {
	if len(ctx.Args()) > 0 {
		return UnexpectedArgsError("verify")
	}

	msg := ctx.String("message")
	infile := ctx.String("infile")
	if err := c.FilterInit(msg, infile, "/dev/null"); err != nil {
		return err
	}
	c.signedBy = ctx.String("signed-by")
	detachedFilename := ctx.String("detached")

	if len(detachedFilename) > 0 {
		data, err := ioutil.ReadFile(detachedFilename)
		if err != nil {
			return err
		}
		c.detachedData = data
	}

	return nil
}

func (c *CmdVerify) Run() (err error) {
	protocols := []rpc.Protocol{
		NewStreamUIProtocol(),
		NewSecretUIProtocol(c.G()),
	}

	cli, err := GetSaltPackClient(c.G())
	if err != nil {
		return err
	}
	if err = RegisterProtocols(protocols); err != nil {
		return err
	}
	snk, src, err := c.ClientFilterOpen()
	if err == nil {
		arg := keybase1.SaltPackVerifyArg{
			Source: src,
			Sink:   snk,
			Opts: keybase1.SaltPackVerifyOptions{
				Signature: c.detachedData,
				SignedBy:  c.signedBy,
			},
		}
		err = cli.SaltPackVerify(context.TODO(), arg)
	}
	cerr := c.Close(err)
	return libkb.PickFirstError(err, cerr)
}

func (c *CmdVerify) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config:    true,
		API:       true,
		KbKeyring: true,
	}
}
<commit_msg>Add identifyui protocol<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"io/ioutil"

	"golang.org/x/net/context"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol"
	rpc "github.com/keybase/go-framed-msgpack-rpc"
)

func NewCmdVerify(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:  "verify",
		Usage: "Verify message or file signatures for keybase users",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdVerify{Contextified: libkb.NewContextified(g)}, "verify", c)
		},
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "d, detached",
				Usage: "Specify a detached signature file.",
			},
			cli.StringFlag{
				Name:  "i, infile",
				Usage: "Specify an input file.",
			},
			cli.StringFlag{
				Name:  "m, message",
				Usage: "Provide the message to verify on the command line.",
			},
			cli.StringFlag{
				Name:  "S, signed-by",
				Usage: "Assert signed by the given user (can use user assertion format).",
			},
		},
	}
}

type CmdVerify struct {
	libkb.Contextified
	UnixFilter
	detachedData []byte
	signedBy     string
}

func (c *CmdVerify) ParseArgv(ctx *cli.Context) error {
	if len(ctx.Args()) > 0 {
		return UnexpectedArgsError("verify")
	}

	msg := ctx.String("message")
	infile := ctx.String("infile")
	if err := c.FilterInit(msg, infile, "/dev/null"); err != nil {
		return err
	}
	c.signedBy = ctx.String("signed-by")
	detachedFilename := ctx.String("detached")

	if len(detachedFilename) > 0 {
		data, err := ioutil.ReadFile(detachedFilename)
		if err != nil {
			return err
		}
		c.detachedData = data
	}

	return nil
}

func (c *CmdVerify) Run() (err error) {
	protocols := []rpc.Protocol{
		NewStreamUIProtocol(),
		NewSecretUIProtocol(c.G()),
		NewIdentifyUIProtocol(c.G()),
	}

	cli, err := GetSaltPackClient(c.G())
	if err != nil {
		return err
	}
	if err = RegisterProtocols(protocols); err != nil {
		return err
	}
	snk, src, err := c.ClientFilterOpen()
	if err == nil {
		arg := keybase1.SaltPackVerifyArg{
			Source: src,
			Sink:   snk,
			Opts: keybase1.SaltPackVerifyOptions{
				Signature: c.detachedData,
				SignedBy:  c.signedBy,
			},
		}
		err = cli.SaltPackVerify(context.TODO(), arg)
	}
	cerr := c.Close(err)
	return libkb.PickFirstError(err, cerr)
}

func (c *CmdVerify) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config:    true,
		API:       true,
		KbKeyring: true,
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

// +build windows

package client

import (
	"bytes"
	"io"
	"os"
	"syscall"
)

var (
	kernel32DLL                 = syscall.NewLazyDLL("kernel32.dll")
	setConsoleTextAttributeProc = kernel32DLL.NewProc("SetConsoleTextAttribute")
)

type WORD uint16

const (
	// Character attributes
	// Note:
	// -- The attributes are combined to produce various colors (e.g., Blue + Green will create Cyan).
	//    Clearing all foreground or background colors results in black; setting all creates white.
	// See https://msdn.microsoft.com/en-us/library/windows/desktop/ms682088(v=vs.85).aspx#_win32_character_attributes.
	fgBlack     WORD = 0x0000
	fgBlue      WORD = 0x0001
	fgGreen     WORD = 0x0002
	fgCyan      WORD = 0x0003
	fgRed       WORD = 0x0004
	fgMagenta   WORD = 0x0005
	fgYellow    WORD = 0x0006
	fgWhite     WORD = 0x0007
	fgIntensity WORD = 0x0008
	fgMask      WORD = 0x000F

	bgBlack     WORD = 0x0000
	bgBlue      WORD = 0x0010
	bgGreen     WORD = 0x0020
	bgCyan      WORD = 0x0030
	bgRed       WORD = 0x0040
	bgMagenta   WORD = 0x0050
	bgYellow    WORD = 0x0060
	bgWhite     WORD = 0x0070
	bgIntensity WORD = 0x0080
	bgMask      WORD = 0x00F0
)

var codesWin = map[byte]WORD{
	0:  fgWhite | bgBlack,     //	"reset":
	1:  fgIntensity | fgWhite, //CpBold          = CodePair{1, 22}
	22: fgWhite,               //	UnBold:        // Just assume this means reset to white fg
	39: fgWhite,               //	"resetfg":
	49: fgWhite,               //	"resetbg":        // Just assume this means reset to white fg
	30: fgBlack,               //	"black":
	31: fgRed,                 //	"red":
	32: fgGreen,               //	"green":
	33: fgYellow,              //	"yellow":
	34: fgBlue,                //	"blue":
	35: fgMagenta,             //	"magenta":
	36: fgCyan,                //	"cyan":
	37: fgWhite,               //	"white":
	90: fgWhite,               //	"grey":
	40: bgBlack,               //	"bgBlack":
	41: bgRed,                 //	"bgRed":
	42: bgGreen,               //	"bgGreen":
	43: bgYellow,              //	"bgYellow":
	44: bgBlue,                //	"bgBlue":
	45: bgMagenta,             //	"bgMagenta":
	46: bgCyan,                //	"bgCyan":
	47: bgWhite,               //	"bgWhite":
}

// Return our writer so we can override Write()
func (ui *UI) OutputWriter() io.Writer {
	return &ColorWriter{os.Stdout, io.Stdout.Fd()}
}

// Return our writer so we can override Write()
func (ui *UI) ErrorWriter() io.Writer {
	return &ColorWriter{os.Stderr, io.Stderr.Fd()}
}

type ColorWriter struct {
	w  io.Writer
	fd int
}

// Rough emulation of Ansi terminal codes.
// Parse the string, pick out the codes, and convert to
// calls to SetConsoleTextAttribute
func (cw *ColorWriter) Write(p []byte) (n int, err error) {
	var totalWritten = len(p)
	ctlStart := []byte{0x1b, '['}

	for nextIndex := 0; len(p) > 0; {

		var controlIndex int = -1
		// search for the next control code
		nextIndex = bytes.Index(p, ctlStart)
		if nextIndex == -1 {
			nextIndex = len(p)
		} else {
			controlIndex = nextIndex + 2
		}

		cw.w.Write(p[0:nextIndex])

		if controlIndex != -1 {
			// The control code is written as separate ascii digits. usually 2,
			// ending with 'm'.
			// Stop at 4 as a sanity check.
			controlCode := p[controlIndex] - '0'
			controlIndex++
			for i := 0; p[controlIndex] != 'm' && i < 4; i++ {
				controlCode *= 10
				controlCode += p[controlIndex] - '0'
				controlIndex++
			}
			if code, ok := codesWin[controlCode]; ok {
				setConsoleTextAttribute(cw.fd, code)
			}
			nextIndex = controlIndex + 1
		}
		p = p[nextIndex:]
	}
	return totalWritten, nil
}

// setConsoleTextAttribute sets the attributes of characters written to the
// console screen buffer by the WriteFile or WriteConsole function.
// See http://msdn.microsoft.com/en-us/library/windows/desktop/ms686047(v=vs.85).aspx.
func setConsoleTextAttribute(handle uintptr, attribute WORD) error {
	r1, r2, err := setConsoleTextAttributeProc.Call(handle, uintptr(attribute), 0)
	use(attribute)
	return checkError(r1, r2, err)
}

// checkError evaluates the results of a Windows API call and returns the error if it failed.
func checkError(r1, r2 uintptr, err error) error {
	// Windows APIs return non-zero to indicate success
	if r1 != 0 {
		return nil
	}

	// Return the error if provided, otherwise default to EINVAL
	if err != nil {
		return err
	}
	return syscall.EINVAL
}

// use is a no-op, but the compiler cannot see that it is.
// Calling use(p) ensures that p is kept live until that point.
func use(p interface{}) {}
<commit_msg>recompile for windows<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

// +build windows

package client

import (
	"bytes"
	"io"
	"os"
	"syscall"
)

var (
	kernel32DLL                 = syscall.NewLazyDLL("kernel32.dll")
	setConsoleTextAttributeProc = kernel32DLL.NewProc("SetConsoleTextAttribute")
)

type WORD uint16

const (
	// Character attributes
	// Note:
	// -- The attributes are combined to produce various colors (e.g., Blue + Green will create Cyan).
	//    Clearing all foreground or background colors results in black; setting all creates white.
	// See https://msdn.microsoft.com/en-us/library/windows/desktop/ms682088(v=vs.85).aspx#_win32_character_attributes.
	fgBlack     WORD = 0x0000
	fgBlue      WORD = 0x0001
	fgGreen     WORD = 0x0002
	fgCyan      WORD = 0x0003
	fgRed       WORD = 0x0004
	fgMagenta   WORD = 0x0005
	fgYellow    WORD = 0x0006
	fgWhite     WORD = 0x0007
	fgIntensity WORD = 0x0008
	fgMask      WORD = 0x000F

	bgBlack     WORD = 0x0000
	bgBlue      WORD = 0x0010
	bgGreen     WORD = 0x0020
	bgCyan      WORD = 0x0030
	bgRed       WORD = 0x0040
	bgMagenta   WORD = 0x0050
	bgYellow    WORD = 0x0060
	bgWhite     WORD = 0x0070
	bgIntensity WORD = 0x0080
	bgMask      WORD = 0x00F0
)

var codesWin = map[byte]WORD{
	0:  fgWhite | bgBlack,     //	"reset":
	1:  fgIntensity | fgWhite, //CpBold          = CodePair{1, 22}
	22: fgWhite,               //	UnBold:        // Just assume this means reset to white fg
	39: fgWhite,               //	"resetfg":
	49: fgWhite,               //	"resetbg":        // Just assume this means reset to white fg
	30: fgBlack,               //	"black":
	31: fgRed,                 //	"red":
	32: fgGreen,               //	"green":
	33: fgYellow,              //	"yellow":
	34: fgBlue,                //	"blue":
	35: fgMagenta,             //	"magenta":
	36: fgCyan,                //	"cyan":
	37: fgWhite,               //	"white":
	90: fgWhite,               //	"grey":
	40: bgBlack,               //	"bgBlack":
	41: bgRed,                 //	"bgRed":
	42: bgGreen,               //	"bgGreen":
	43: bgYellow,              //	"bgYellow":
	44: bgBlue,                //	"bgBlue":
	45: bgMagenta,             //	"bgMagenta":
	46: bgCyan,                //	"bgCyan":
	47: bgWhite,               //	"bgWhite":
}

// Return our writer so we can override Write()
func (ui *UI) OutputWriter() io.Writer {
	return &ColorWriter{os.Stdout, os.Stdout.Fd()}
}

// Return our writer so we can override Write()
func (ui *UI) ErrorWriter() io.Writer {
	return &ColorWriter{os.Stderr, os.Stderr.Fd()}
}

type ColorWriter struct {
	w  io.Writer
	fd uintptr
}

// Rough emulation of Ansi terminal codes.
// Parse the string, pick out the codes, and convert to
// calls to SetConsoleTextAttribute
func (cw *ColorWriter) Write(p []byte) (n int, err error) {
	var totalWritten = len(p)
	ctlStart := []byte{0x1b, '['}

	for nextIndex := 0; len(p) > 0; {

		var controlIndex int = -1
		// search for the next control code
		nextIndex = bytes.Index(p, ctlStart)
		if nextIndex == -1 {
			nextIndex = len(p)
		} else {
			controlIndex = nextIndex + 2
		}

		cw.w.Write(p[0:nextIndex])

		if controlIndex != -1 {
			// The control code is written as separate ascii digits. usually 2,
			// ending with 'm'.
			// Stop at 4 as a sanity check.
			controlCode := p[controlIndex] - '0'
			controlIndex++
			for i := 0; p[controlIndex] != 'm' && i < 4; i++ {
				controlCode *= 10
				controlCode += p[controlIndex] - '0'
				controlIndex++
			}
			if code, ok := codesWin[controlCode]; ok {
				setConsoleTextAttribute(cw.fd, code)
			}
			nextIndex = controlIndex + 1
		}
		p = p[nextIndex:]
	}
	return totalWritten, nil
}

// setConsoleTextAttribute sets the attributes of characters written to the
// console screen buffer by the WriteFile or WriteConsole function.
// See http://msdn.microsoft.com/en-us/library/windows/desktop/ms686047(v=vs.85).aspx.
func setConsoleTextAttribute(handle uintptr, attribute WORD) error {
	r1, r2, err := setConsoleTextAttributeProc.Call(handle, uintptr(attribute), 0)
	use(attribute)
	return checkError(r1, r2, err)
}

// checkError evaluates the results of a Windows API call and returns the error if it failed.
func checkError(r1, r2 uintptr, err error) error {
	// Windows APIs return non-zero to indicate success
	if r1 != 0 {
		return nil
	}

	// Return the error if provided, otherwise default to EINVAL
	if err != nil {
		return err
	}
	return syscall.EINVAL
}

// use is a no-op, but the compiler cannot see that it is.
// Calling use(p) ensures that p is kept live until that point.
func use(p interface{}) {}
<|endoftext|>


<commit_before>// Copyright 2016-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/vishvananda/netlink"

	"github.com/cilium/cilium/common"
	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/cgroups"
	"github.com/cilium/cilium/pkg/command/exec"
	"github.com/cilium/cilium/pkg/datapath/alignchecker"
	"github.com/cilium/cilium/pkg/datapath/iptables"
	"github.com/cilium/cilium/pkg/datapath/loader"
	"github.com/cilium/cilium/pkg/datapath/prefilter"
	"github.com/cilium/cilium/pkg/defaults"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/node"
	"github.com/cilium/cilium/pkg/option"
)

func (d *Daemon) compileBase() error {
	var args []string
	var mode string
	var ret error

	args = make([]string, initArgMax)

	// Lock so that endpoints cannot be built while we are compile base programs.
	d.compilationMutex.Lock()
	defer d.compilationMutex.Unlock()

	if err := d.writeNetdevHeader("./"); err != nil {
		log.WithError(err).Warn("Unable to write netdev header")
		return err
	}
	loader.Init(d.datapath, &d.nodeDiscovery.LocalConfig)

	scopedLog := log.WithField(logfields.XDPDevice, option.Config.DevicePreFilter)
	if option.Config.DevicePreFilter != "undefined" {
		if err := prefilter.ProbePreFilter(option.Config.DevicePreFilter, option.Config.ModePreFilter); err != nil {
			scopedLog.WithError(err).Warn("Turning off prefilter")
			option.Config.DevicePreFilter = "undefined"
		}
	}
	if option.Config.DevicePreFilter != "undefined" {
		if d.preFilter, ret = prefilter.NewPreFilter(); ret != nil {
			scopedLog.WithError(ret).Warn("Unable to init prefilter")
			return ret
		}

		if err := d.writePreFilterHeader("./"); err != nil {
			scopedLog.WithError(err).Warn("Unable to write prefilter header")
			return err
		}

		args[initArgDevicePreFilter] = option.Config.DevicePreFilter
		args[initArgModePreFilter] = option.Config.ModePreFilter
	}

	args[initArgLib] = option.Config.BpfDir
	args[initArgRundir] = option.Config.StateDir
	args[initArgCgroupRoot] = cgroups.GetCgroupRoot()
	args[initArgBpffsRoot] = bpf.GetMapRoot()

	if option.Config.EnableIPv4 {
		args[initArgIPv4NodeIP] = node.GetInternalIPv4().String()
	} else {
		args[initArgIPv4NodeIP] = "<nil>"
	}

	if option.Config.EnableIPv6 {
		args[initArgIPv6NodeIP] = node.GetIPv6().String()
	} else {
		args[initArgIPv6NodeIP] = "<nil>"
	}

	args[initArgMTU] = fmt.Sprintf("%d", d.mtuConfig.GetDeviceMTU())

	if option.Config.EnableIPSec {
		args[initArgIPSec] = "true"
	} else {
		args[initArgIPSec] = "false"
	}

	if !option.Config.InstallIptRules && option.Config.Masquerade {
		args[initArgMasquerade] = "true"
	} else {
		args[initArgMasquerade] = "false"
	}

	if option.Config.EnableHostReachableServices {
		args[initArgHostReachableServices] = "true"
		if option.Config.EnableHostServicesUDP {
			args[initArgHostReachableServicesUDP] = "true"
		} else {
			args[initArgHostReachableServicesUDP] = "false"
		}
	} else {
		args[initArgHostReachableServices] = "false"
		args[initArgHostReachableServicesUDP] = "false"
	}

	if option.Config.EncryptInterface != "" {
		args[initArgEncryptInterface] = option.Config.EncryptInterface
	}

	if option.Config.Device != "undefined" {
		_, err := netlink.LinkByName(option.Config.Device)
		if err != nil {
			log.WithError(err).WithField("device", option.Config.Device).Warn("Link does not exist")
			return err
		}

		if option.Config.IsLBEnabled() {
			if option.Config.Device != option.Config.LBInterface {
				//FIXME: allow different interfaces
				return fmt.Errorf("Unable to have an interface for LB mode different than snooping interface")
			}
			if err := d.setHostAddresses(); err != nil {
				return err
			}
			mode = "lb"
		} else {
			if option.Config.DatapathMode == option.DatapathModeIpvlan {
				mode = "ipvlan"
			} else {
				mode = "direct"
			}
		}

		args[initArgMode] = mode
		if option.Config.EnableNodePort &&
			strings.ToLower(option.Config.Tunnel) != "disabled" {
			args[initArgMode] = option.Config.Tunnel
		}
		args[initArgDevice] = option.Config.Device
	} else {
		if option.Config.IsLBEnabled() && strings.ToLower(option.Config.Tunnel) != "disabled" {
			//FIXME: allow LBMode in tunnel
			return fmt.Errorf("Unable to run LB mode with tunnel mode")
		}

		args[initArgMode] = option.Config.Tunnel

		if option.Config.IsFlannelMasterDeviceSet() {
			args[initArgMode] = "flannel"
			args[initArgDevice] = option.Config.FlannelMasterDevice
		}
	}

	if option.Config.EnableNodePort {
		args[initArgNodePort] = "true"
	}

	log.Info("Setting up base BPF datapath")

	prog := filepath.Join(option.Config.BpfDir, "init.sh")
	ctx, cancel := context.WithTimeout(context.Background(), defaults.ExecTimeout)
	defer cancel()
	cmd := exec.CommandContext(ctx, prog, args...)
	cmd.Env = bpf.Environment()
	if _, err := cmd.CombinedOutput(log, true); err != nil {
		return err
	}

	if canDisableDwarfRelocations {
		// Validate alignments of C and Go equivalent structs
		if err := alignchecker.CheckStructAlignments(defaults.AlignCheckerName); err != nil {
			log.WithError(err).Fatal("C and Go structs alignment check failed")
		}
	} else {
		log.Warning("Cannot check matching of C and Go common struct alignments due to old LLVM/clang version")
	}

	if !option.Config.IsFlannelMasterDeviceSet() {
		d.ipam.ReserveLocalRoutes()
	}

	if err := d.datapath.Node().NodeConfigurationChanged(d.nodeDiscovery.LocalConfig); err != nil {
		return err
	}

	iptablesManager := iptables.IptablesManager{}
	iptablesManager.Init()
	// Always remove masquerade rule and then re-add it if required
	iptablesManager.RemoveRules()
	if option.Config.InstallIptRules {
		if err := iptablesManager.InstallRules(option.Config.HostDevice); err != nil {
			return err
		}
	}
	// Reinstall proxy rules for any running proxies
	if d.l7Proxy != nil {
		d.l7Proxy.ReinstallRules()
	}

	log.Info("Setting sysctl net.core.bpf_jit_enable=1")
	log.Info("Setting sysctl net.ipv4.conf.all.rp_filter=0")
	log.Info("Setting sysctl net.ipv6.conf.all.disable_ipv6=0")

	return nil
}

func (d *Daemon) createNodeConfigHeaderfile() error {
	nodeConfigPath := option.Config.GetNodeConfigPath()
	f, err := os.Create(nodeConfigPath)
	if err != nil {
		log.WithError(err).WithField(logfields.Path, nodeConfigPath).Fatal("Failed to create node configuration file")
		return err
	}
	defer f.Close()

	if err = d.datapath.WriteNodeConfig(f, &d.nodeDiscovery.LocalConfig); err != nil {
		log.WithError(err).WithField(logfields.Path, nodeConfigPath).Fatal("Failed to write node configuration file")
		return err
	}
	return nil
}

func deleteHostDevice() {
	link, err := netlink.LinkByName(option.Config.HostDevice)
	if err != nil {
		log.WithError(err).Warningf("Unable to lookup host device %s. No old cilium_host interface exists", option.Config.HostDevice)
		return
	}

	if err := netlink.LinkDel(link); err != nil {
		log.WithError(err).Errorf("Unable to delete host device %s to change allocation CIDR", option.Config.HostDevice)
	}
}

// listFilterIfs returns a map of interfaces based on the given filter.
// The filter should take a link and, if found, return the index of that
// interface, if not found return -1.
func listFilterIfs(filter func(netlink.Link) int) (map[int]netlink.Link, error) {
	ifs, err := netlink.LinkList()
	if err != nil {
		return nil, err
	}
	vethLXCIdxs := map[int]netlink.Link{}
	for _, intf := range ifs {
		if idx := filter(intf); idx != -1 {
			vethLXCIdxs[idx] = intf
		}
	}
	return vethLXCIdxs, nil
}

// clearCiliumVeths checks all veths created by cilium and removes all that
// are considered a leftover from failed attempts to connect the container.
func (d *Daemon) clearCiliumVeths() error {
	log.Info("Removing stale endpoint interfaces")

	leftVeths, err := listFilterIfs(func(intf netlink.Link) int {
		// Filter by veth and return the index of the interface.
		if intf.Type() == "veth" {
			return intf.Attrs().Index
		}
		return -1
	})

	if err != nil {
		return fmt.Errorf("unable to retrieve host network interfaces: %s", err)
	}

	for _, v := range leftVeths {
		peerIndex := v.Attrs().ParentIndex
		parentVeth, found := leftVeths[peerIndex]
		if found && peerIndex != 0 && strings.HasPrefix(parentVeth.Attrs().Name, "lxc") {
			err := netlink.LinkDel(v)
			if err != nil {
				log.WithError(err).Warningf("Unable to delete stale veth device %s", v.Attrs().Name)
			}
		}
	}
	return nil
}

// Must be called with option.Config.EnablePolicyMU locked.
func (d *Daemon) writePreFilterHeader(dir string) error {
	headerPath := filepath.Join(dir, common.PreFilterHeaderFileName)
	log.WithField(logfields.Path, headerPath).Debug("writing configuration")
	f, err := os.Create(headerPath)
	if err != nil {
		return fmt.Errorf("failed to open file %s for writing: %s", headerPath, err)

	}
	defer f.Close()
	fw := bufio.NewWriter(f)
	fmt.Fprint(fw, "/*\n")
	fmt.Fprintf(fw, " * XDP device: %s\n", option.Config.DevicePreFilter)
	fmt.Fprintf(fw, " * XDP mode: %s\n", option.Config.ModePreFilter)
	fmt.Fprint(fw, " */\n\n")
	d.preFilter.WriteConfig(fw)
	return fw.Flush()
}

func (d *Daemon) writeNetdevHeader(dir string) error {
	headerPath := filepath.Join(dir, common.NetdevHeaderFileName)
	log.WithField(logfields.Path, headerPath).Debug("writing configuration")

	f, err := os.Create(headerPath)
	if err != nil {
		return fmt.Errorf("failed to open file %s for writing: %s", headerPath, err)

	}
	defer f.Close()

	if err := d.datapath.WriteNetdevConfig(f, d); err != nil {
		return err
	}
	return nil
}
<commit_msg>[daemon] - Change MTU source for cilium_host (Use the Route one)<commit_after>// Copyright 2016-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/vishvananda/netlink"

	"github.com/cilium/cilium/common"
	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/cgroups"
	"github.com/cilium/cilium/pkg/command/exec"
	"github.com/cilium/cilium/pkg/datapath/alignchecker"
	"github.com/cilium/cilium/pkg/datapath/iptables"
	"github.com/cilium/cilium/pkg/datapath/loader"
	"github.com/cilium/cilium/pkg/datapath/prefilter"
	"github.com/cilium/cilium/pkg/defaults"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/node"
	"github.com/cilium/cilium/pkg/option"
)

func (d *Daemon) compileBase() error {
	var args []string
	var mode string
	var ret error

	args = make([]string, initArgMax)

	// Lock so that endpoints cannot be built while we are compile base programs.
	d.compilationMutex.Lock()
	defer d.compilationMutex.Unlock()

	if err := d.writeNetdevHeader("./"); err != nil {
		log.WithError(err).Warn("Unable to write netdev header")
		return err
	}
	loader.Init(d.datapath, &d.nodeDiscovery.LocalConfig)

	scopedLog := log.WithField(logfields.XDPDevice, option.Config.DevicePreFilter)
	if option.Config.DevicePreFilter != "undefined" {
		if err := prefilter.ProbePreFilter(option.Config.DevicePreFilter, option.Config.ModePreFilter); err != nil {
			scopedLog.WithError(err).Warn("Turning off prefilter")
			option.Config.DevicePreFilter = "undefined"
		}
	}
	if option.Config.DevicePreFilter != "undefined" {
		if d.preFilter, ret = prefilter.NewPreFilter(); ret != nil {
			scopedLog.WithError(ret).Warn("Unable to init prefilter")
			return ret
		}

		if err := d.writePreFilterHeader("./"); err != nil {
			scopedLog.WithError(err).Warn("Unable to write prefilter header")
			return err
		}

		args[initArgDevicePreFilter] = option.Config.DevicePreFilter
		args[initArgModePreFilter] = option.Config.ModePreFilter
	}

	args[initArgLib] = option.Config.BpfDir
	args[initArgRundir] = option.Config.StateDir
	args[initArgCgroupRoot] = cgroups.GetCgroupRoot()
	args[initArgBpffsRoot] = bpf.GetMapRoot()

	if option.Config.EnableIPv4 {
		args[initArgIPv4NodeIP] = node.GetInternalIPv4().String()
	} else {
		args[initArgIPv4NodeIP] = "<nil>"
	}

	if option.Config.EnableIPv6 {
		args[initArgIPv6NodeIP] = node.GetIPv6().String()
	} else {
		args[initArgIPv6NodeIP] = "<nil>"
	}

	args[initArgMTU] = fmt.Sprintf("%d", d.mtuConfig.GetRouteMTU())

	if option.Config.EnableIPSec {
		args[initArgIPSec] = "true"
	} else {
		args[initArgIPSec] = "false"
	}

	if !option.Config.InstallIptRules && option.Config.Masquerade {
		args[initArgMasquerade] = "true"
	} else {
		args[initArgMasquerade] = "false"
	}

	if option.Config.EnableHostReachableServices {
		args[initArgHostReachableServices] = "true"
		if option.Config.EnableHostServicesUDP {
			args[initArgHostReachableServicesUDP] = "true"
		} else {
			args[initArgHostReachableServicesUDP] = "false"
		}
	} else {
		args[initArgHostReachableServices] = "false"
		args[initArgHostReachableServicesUDP] = "false"
	}

	if option.Config.EncryptInterface != "" {
		args[initArgEncryptInterface] = option.Config.EncryptInterface
	}

	if option.Config.Device != "undefined" {
		_, err := netlink.LinkByName(option.Config.Device)
		if err != nil {
			log.WithError(err).WithField("device", option.Config.Device).Warn("Link does not exist")
			return err
		}

		if option.Config.IsLBEnabled() {
			if option.Config.Device != option.Config.LBInterface {
				//FIXME: allow different interfaces
				return fmt.Errorf("Unable to have an interface for LB mode different than snooping interface")
			}
			if err := d.setHostAddresses(); err != nil {
				return err
			}
			mode = "lb"
		} else {
			if option.Config.DatapathMode == option.DatapathModeIpvlan {
				mode = "ipvlan"
			} else {
				mode = "direct"
			}
		}

		args[initArgMode] = mode
		if option.Config.EnableNodePort &&
			strings.ToLower(option.Config.Tunnel) != "disabled" {
			args[initArgMode] = option.Config.Tunnel
		}
		args[initArgDevice] = option.Config.Device
	} else {
		if option.Config.IsLBEnabled() && strings.ToLower(option.Config.Tunnel) != "disabled" {
			//FIXME: allow LBMode in tunnel
			return fmt.Errorf("Unable to run LB mode with tunnel mode")
		}

		args[initArgMode] = option.Config.Tunnel

		if option.Config.IsFlannelMasterDeviceSet() {
			args[initArgMode] = "flannel"
			args[initArgDevice] = option.Config.FlannelMasterDevice
		}
	}

	if option.Config.EnableNodePort {
		args[initArgNodePort] = "true"
	}

	log.Info("Setting up base BPF datapath")

	prog := filepath.Join(option.Config.BpfDir, "init.sh")
	ctx, cancel := context.WithTimeout(context.Background(), defaults.ExecTimeout)
	defer cancel()
	cmd := exec.CommandContext(ctx, prog, args...)
	cmd.Env = bpf.Environment()
	if _, err := cmd.CombinedOutput(log, true); err != nil {
		return err
	}

	if canDisableDwarfRelocations {
		// Validate alignments of C and Go equivalent structs
		if err := alignchecker.CheckStructAlignments(defaults.AlignCheckerName); err != nil {
			log.WithError(err).Fatal("C and Go structs alignment check failed")
		}
	} else {
		log.Warning("Cannot check matching of C and Go common struct alignments due to old LLVM/clang version")
	}

	if !option.Config.IsFlannelMasterDeviceSet() {
		d.ipam.ReserveLocalRoutes()
	}

	if err := d.datapath.Node().NodeConfigurationChanged(d.nodeDiscovery.LocalConfig); err != nil {
		return err
	}

	iptablesManager := iptables.IptablesManager{}
	iptablesManager.Init()
	// Always remove masquerade rule and then re-add it if required
	iptablesManager.RemoveRules()
	if option.Config.InstallIptRules {
		if err := iptablesManager.InstallRules(option.Config.HostDevice); err != nil {
			return err
		}
	}
	// Reinstall proxy rules for any running proxies
	if d.l7Proxy != nil {
		d.l7Proxy.ReinstallRules()
	}

	log.Info("Setting sysctl net.core.bpf_jit_enable=1")
	log.Info("Setting sysctl net.ipv4.conf.all.rp_filter=0")
	log.Info("Setting sysctl net.ipv6.conf.all.disable_ipv6=0")

	return nil
}

func (d *Daemon) createNodeConfigHeaderfile() error {
	nodeConfigPath := option.Config.GetNodeConfigPath()
	f, err := os.Create(nodeConfigPath)
	if err != nil {
		log.WithError(err).WithField(logfields.Path, nodeConfigPath).Fatal("Failed to create node configuration file")
		return err
	}
	defer f.Close()

	if err = d.datapath.WriteNodeConfig(f, &d.nodeDiscovery.LocalConfig); err != nil {
		log.WithError(err).WithField(logfields.Path, nodeConfigPath).Fatal("Failed to write node configuration file")
		return err
	}
	return nil
}

func deleteHostDevice() {
	link, err := netlink.LinkByName(option.Config.HostDevice)
	if err != nil {
		log.WithError(err).Warningf("Unable to lookup host device %s. No old cilium_host interface exists", option.Config.HostDevice)
		return
	}

	if err := netlink.LinkDel(link); err != nil {
		log.WithError(err).Errorf("Unable to delete host device %s to change allocation CIDR", option.Config.HostDevice)
	}
}

// listFilterIfs returns a map of interfaces based on the given filter.
// The filter should take a link and, if found, return the index of that
// interface, if not found return -1.
func listFilterIfs(filter func(netlink.Link) int) (map[int]netlink.Link, error) {
	ifs, err := netlink.LinkList()
	if err != nil {
		return nil, err
	}
	vethLXCIdxs := map[int]netlink.Link{}
	for _, intf := range ifs {
		if idx := filter(intf); idx != -1 {
			vethLXCIdxs[idx] = intf
		}
	}
	return vethLXCIdxs, nil
}

// clearCiliumVeths checks all veths created by cilium and removes all that
// are considered a leftover from failed attempts to connect the container.
func (d *Daemon) clearCiliumVeths() error {
	log.Info("Removing stale endpoint interfaces")

	leftVeths, err := listFilterIfs(func(intf netlink.Link) int {
		// Filter by veth and return the index of the interface.
		if intf.Type() == "veth" {
			return intf.Attrs().Index
		}
		return -1
	})

	if err != nil {
		return fmt.Errorf("unable to retrieve host network interfaces: %s", err)
	}

	for _, v := range leftVeths {
		peerIndex := v.Attrs().ParentIndex
		parentVeth, found := leftVeths[peerIndex]
		if found && peerIndex != 0 && strings.HasPrefix(parentVeth.Attrs().Name, "lxc") {
			err := netlink.LinkDel(v)
			if err != nil {
				log.WithError(err).Warningf("Unable to delete stale veth device %s", v.Attrs().Name)
			}
		}
	}
	return nil
}

// Must be called with option.Config.EnablePolicyMU locked.
func (d *Daemon) writePreFilterHeader(dir string) error {
	headerPath := filepath.Join(dir, common.PreFilterHeaderFileName)
	log.WithField(logfields.Path, headerPath).Debug("writing configuration")
	f, err := os.Create(headerPath)
	if err != nil {
		return fmt.Errorf("failed to open file %s for writing: %s", headerPath, err)

	}
	defer f.Close()
	fw := bufio.NewWriter(f)
	fmt.Fprint(fw, "/*\n")
	fmt.Fprintf(fw, " * XDP device: %s\n", option.Config.DevicePreFilter)
	fmt.Fprintf(fw, " * XDP mode: %s\n", option.Config.ModePreFilter)
	fmt.Fprint(fw, " */\n\n")
	d.preFilter.WriteConfig(fw)
	return fw.Flush()
}

func (d *Daemon) writeNetdevHeader(dir string) error {
	headerPath := filepath.Join(dir, common.NetdevHeaderFileName)
	log.WithField(logfields.Path, headerPath).Debug("writing configuration")

	f, err := os.Create(headerPath)
	if err != nil {
		return fmt.Errorf("failed to open file %s for writing: %s", headerPath, err)

	}
	defer f.Close()

	if err := d.datapath.WriteNetdevConfig(f, d); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package b2

import (
	"context"
	"hash"
	"io"
	"net/http"
	"path"

	"github.com/restic/restic/internal/backend"
	"github.com/restic/restic/internal/backend/sema"
	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"

	"github.com/cenkalti/backoff/v4"
	"github.com/kurin/blazer/b2"
)

// b2Backend is a backend which stores its data on Backblaze B2.
type b2Backend struct {
	client       *b2.Client
	bucket       *b2.Bucket
	cfg          Config
	listMaxItems int
	backend.Layout
	sem sema.Semaphore
}

// Billing happens in 1000 item granlarity, but we are more interested in reducing the number of network round trips
const defaultListMaxItems = 10 * 1000

// ensure statically that *b2Backend implements restic.Backend.
var _ restic.Backend = &b2Backend{}

func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Client, error) {
	opts := []b2.ClientOption{b2.Transport(rt)}

	c, err := b2.NewClient(ctx, cfg.AccountID, cfg.Key.Unwrap(), opts...)
	if err != nil {
		return nil, errors.Wrap(err, "b2.NewClient")
	}
	return c, nil
}

// Open opens a connection to the B2 service.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
	debug.Log("cfg %#v", cfg)

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	client, err := newClient(ctx, cfg, rt)
	if err != nil {
		return nil, err
	}

	bucket, err := client.Bucket(ctx, cfg.Bucket)
	if err != nil {
		return nil, errors.Wrap(err, "Bucket")
	}

	sem, err := sema.New(cfg.Connections)
	if err != nil {
		return nil, err
	}

	be := &b2Backend{
		client: client,
		bucket: bucket,
		cfg:    cfg,
		Layout: &backend.DefaultLayout{
			Join: path.Join,
			Path: cfg.Prefix,
		},
		listMaxItems: defaultListMaxItems,
		sem:          sem,
	}

	return be, nil
}

// Create opens a connection to the B2 service. If the bucket does not exist yet,
// it is created.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
	debug.Log("cfg %#v", cfg)

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	client, err := newClient(ctx, cfg, rt)
	if err != nil {
		return nil, err
	}

	attr := b2.BucketAttrs{
		Type: b2.Private,
	}
	bucket, err := client.NewBucket(ctx, cfg.Bucket, &attr)
	if err != nil {
		return nil, errors.Wrap(err, "NewBucket")
	}

	sem, err := sema.New(cfg.Connections)
	if err != nil {
		return nil, err
	}

	be := &b2Backend{
		client: client,
		bucket: bucket,
		cfg:    cfg,
		Layout: &backend.DefaultLayout{
			Join: path.Join,
			Path: cfg.Prefix,
		},
		listMaxItems: defaultListMaxItems,
		sem:          sem,
	}

	present, err := be.Test(ctx, restic.Handle{Type: restic.ConfigFile})
	if err != nil {
		return nil, err
	}

	if present {
		return nil, errors.New("config already exists")
	}

	return be, nil
}

// SetListMaxItems sets the number of list items to load per request.
func (be *b2Backend) SetListMaxItems(i int) {
	be.listMaxItems = i
}

func (be *b2Backend) Connections() uint {
	return be.cfg.Connections
}

// Location returns the location for the backend.
func (be *b2Backend) Location() string {
	return be.cfg.Bucket
}

// Hasher may return a hash function for calculating a content hash for the backend
func (be *b2Backend) Hasher() hash.Hash {
	return nil
}

// HasAtomicReplace returns whether Save() can atomically replace files
func (be *b2Backend) HasAtomicReplace() bool {
	return true
}

// IsNotExist returns true if the error is caused by a non-existing file.
func (be *b2Backend) IsNotExist(err error) bool {
	return b2.IsNotExist(errors.Cause(err))
}

// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
	return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}

func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
	debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
	if err := h.Valid(); err != nil {
		return nil, backoff.Permanent(err)
	}

	if offset < 0 {
		return nil, errors.New("offset is negative")
	}

	if length < 0 {
		return nil, errors.Errorf("invalid length %d", length)
	}

	ctx, cancel := context.WithCancel(ctx)

	be.sem.GetToken()

	name := be.Layout.Filename(h)
	obj := be.bucket.Object(name)

	if offset == 0 && length == 0 {
		rd := obj.NewReader(ctx)
		return be.sem.ReleaseTokenOnClose(rd, cancel), nil
	}

	// pass a negative length to NewRangeReader so that the remainder of the
	// file is read.
	if length == 0 {
		length = -1
	}

	rd := obj.NewRangeReader(ctx, offset, int64(length))
	return be.sem.ReleaseTokenOnClose(rd, cancel), nil
}

// Save stores data in the backend at the handle.
func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	if err := h.Valid(); err != nil {
		return backoff.Permanent(err)
	}

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	name := be.Filename(h)
	debug.Log("Save %v, name %v", h, name)
	obj := be.bucket.Object(name)

	// b2 always requires sha1 checksums for uploaded file parts
	w := obj.NewWriter(ctx)
	n, err := io.Copy(w, rd)
	debug.Log("  saved %d bytes, err %v", n, err)

	if err != nil {
		_ = w.Close()
		return errors.Wrap(err, "Copy")
	}

	// sanity check
	if n != rd.Length() {
		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", n, rd.Length())
	}
	return errors.Wrap(w.Close(), "Close")
}

// Stat returns information about a blob.
func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
	debug.Log("Stat %v", h)

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	name := be.Filename(h)
	obj := be.bucket.Object(name)
	info, err := obj.Attrs(ctx)
	if err != nil {
		debug.Log("Attrs() err %v", err)
		return restic.FileInfo{}, errors.Wrap(err, "Stat")
	}
	return restic.FileInfo{Size: info.Size, Name: h.Name}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (be *b2Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
	debug.Log("Test %v", h)

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	found := false
	name := be.Filename(h)
	obj := be.bucket.Object(name)
	info, err := obj.Attrs(ctx)
	if err == nil && info != nil && info.Status == b2.Uploaded {
		found = true
	}
	return found, nil
}

// Remove removes the blob with the given name and type.
func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
	debug.Log("Remove %v", h)

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	// the retry backend will also repeat the remove method up to 10 times
	for i := 0; i < 3; i++ {
		obj := be.bucket.Object(be.Filename(h))
		err := obj.Delete(ctx)
		if err == nil {
			// keep deleting until we are sure that no leftover file versions exist
			continue
		}
		// consider a file as removed if b2 informs us that it does not exist
		if b2.IsNotExist(err) {
			return nil
		}
		return errors.Wrap(err, "Delete")
	}

	return errors.New("failed to delete all file versions")
}

type semLocker struct {
	sema.Semaphore
}

func (sm *semLocker) Lock()   { sm.GetToken() }
func (sm *semLocker) Unlock() { sm.ReleaseToken() }

// List returns a channel that yields all names of blobs of type t.
func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
	debug.Log("List %v", t)

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	prefix, _ := be.Basedir(t)
	iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems), b2.ListLocker(&semLocker{be.sem}))

	for iter.Next() {
		obj := iter.Object()

		attrs, err := obj.Attrs(ctx)
		if err != nil {
			return err
		}

		fi := restic.FileInfo{
			Name: path.Base(obj.Name()),
			Size: attrs.Size,
		}

		if err := fn(fi); err != nil {
			return err
		}
	}
	if err := iter.Err(); err != nil {
		debug.Log("List: %v", err)
		return err
	}
	return nil
}

// Remove keys for a specified backend type.
func (be *b2Backend) removeKeys(ctx context.Context, t restic.FileType) error {
	debug.Log("removeKeys %v", t)
	return be.List(ctx, t, func(fi restic.FileInfo) error {
		return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name})
	})
}

// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *b2Backend) Delete(ctx context.Context) error {
	alltypes := []restic.FileType{
		restic.PackFile,
		restic.KeyFile,
		restic.LockFile,
		restic.SnapshotFile,
		restic.IndexFile}

	for _, t := range alltypes {
		err := be.removeKeys(ctx, t)
		if err != nil {
			return nil
		}
	}
	err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
	if err != nil && b2.IsNotExist(errors.Cause(err)) {
		err = nil
	}

	return err
}

// Close does nothing
func (be *b2Backend) Close() error { return nil }
<commit_msg>b2: cancel connection setup after a minute<commit_after>package b2

import (
	"context"
	"hash"
	"io"
	"net/http"
	"path"
	"time"

	"github.com/restic/restic/internal/backend"
	"github.com/restic/restic/internal/backend/sema"
	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"

	"github.com/cenkalti/backoff/v4"
	"github.com/kurin/blazer/b2"
)

// b2Backend is a backend which stores its data on Backblaze B2.
type b2Backend struct {
	client       *b2.Client
	bucket       *b2.Bucket
	cfg          Config
	listMaxItems int
	backend.Layout
	sem sema.Semaphore
}

// Billing happens in 1000 item granlarity, but we are more interested in reducing the number of network round trips
const defaultListMaxItems = 10 * 1000

// ensure statically that *b2Backend implements restic.Backend.
var _ restic.Backend = &b2Backend{}

func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Client, error) {
	opts := []b2.ClientOption{b2.Transport(rt)}

	// if the connection B2 fails, this can cause the client to hang
	// cancel the connection after a minute to at least provide some feedback to the user
	ctx, cancel := context.WithTimeout(ctx, time.Minute)
	defer cancel()
	c, err := b2.NewClient(ctx, cfg.AccountID, cfg.Key.Unwrap(), opts...)
	if err == context.DeadlineExceeded {
		return nil, errors.New("connection to B2 failed")
	} else if err != nil {
		return nil, errors.Wrap(err, "b2.NewClient")
	}
	return c, nil
}

// Open opens a connection to the B2 service.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
	debug.Log("cfg %#v", cfg)

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	client, err := newClient(ctx, cfg, rt)
	if err != nil {
		return nil, err
	}

	bucket, err := client.Bucket(ctx, cfg.Bucket)
	if err != nil {
		return nil, errors.Wrap(err, "Bucket")
	}

	sem, err := sema.New(cfg.Connections)
	if err != nil {
		return nil, err
	}

	be := &b2Backend{
		client: client,
		bucket: bucket,
		cfg:    cfg,
		Layout: &backend.DefaultLayout{
			Join: path.Join,
			Path: cfg.Prefix,
		},
		listMaxItems: defaultListMaxItems,
		sem:          sem,
	}

	return be, nil
}

// Create opens a connection to the B2 service. If the bucket does not exist yet,
// it is created.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
	debug.Log("cfg %#v", cfg)

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	client, err := newClient(ctx, cfg, rt)
	if err != nil {
		return nil, err
	}

	attr := b2.BucketAttrs{
		Type: b2.Private,
	}
	bucket, err := client.NewBucket(ctx, cfg.Bucket, &attr)
	if err != nil {
		return nil, errors.Wrap(err, "NewBucket")
	}

	sem, err := sema.New(cfg.Connections)
	if err != nil {
		return nil, err
	}

	be := &b2Backend{
		client: client,
		bucket: bucket,
		cfg:    cfg,
		Layout: &backend.DefaultLayout{
			Join: path.Join,
			Path: cfg.Prefix,
		},
		listMaxItems: defaultListMaxItems,
		sem:          sem,
	}

	present, err := be.Test(ctx, restic.Handle{Type: restic.ConfigFile})
	if err != nil {
		return nil, err
	}

	if present {
		return nil, errors.New("config already exists")
	}

	return be, nil
}

// SetListMaxItems sets the number of list items to load per request.
func (be *b2Backend) SetListMaxItems(i int) {
	be.listMaxItems = i
}

func (be *b2Backend) Connections() uint {
	return be.cfg.Connections
}

// Location returns the location for the backend.
func (be *b2Backend) Location() string {
	return be.cfg.Bucket
}

// Hasher may return a hash function for calculating a content hash for the backend
func (be *b2Backend) Hasher() hash.Hash {
	return nil
}

// HasAtomicReplace returns whether Save() can atomically replace files
func (be *b2Backend) HasAtomicReplace() bool {
	return true
}

// IsNotExist returns true if the error is caused by a non-existing file.
func (be *b2Backend) IsNotExist(err error) bool {
	return b2.IsNotExist(errors.Cause(err))
}

// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
	return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}

func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
	debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
	if err := h.Valid(); err != nil {
		return nil, backoff.Permanent(err)
	}

	if offset < 0 {
		return nil, errors.New("offset is negative")
	}

	if length < 0 {
		return nil, errors.Errorf("invalid length %d", length)
	}

	ctx, cancel := context.WithCancel(ctx)

	be.sem.GetToken()

	name := be.Layout.Filename(h)
	obj := be.bucket.Object(name)

	if offset == 0 && length == 0 {
		rd := obj.NewReader(ctx)
		return be.sem.ReleaseTokenOnClose(rd, cancel), nil
	}

	// pass a negative length to NewRangeReader so that the remainder of the
	// file is read.
	if length == 0 {
		length = -1
	}

	rd := obj.NewRangeReader(ctx, offset, int64(length))
	return be.sem.ReleaseTokenOnClose(rd, cancel), nil
}

// Save stores data in the backend at the handle.
func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	if err := h.Valid(); err != nil {
		return backoff.Permanent(err)
	}

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	name := be.Filename(h)
	debug.Log("Save %v, name %v", h, name)
	obj := be.bucket.Object(name)

	// b2 always requires sha1 checksums for uploaded file parts
	w := obj.NewWriter(ctx)
	n, err := io.Copy(w, rd)
	debug.Log("  saved %d bytes, err %v", n, err)

	if err != nil {
		_ = w.Close()
		return errors.Wrap(err, "Copy")
	}

	// sanity check
	if n != rd.Length() {
		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", n, rd.Length())
	}
	return errors.Wrap(w.Close(), "Close")
}

// Stat returns information about a blob.
func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
	debug.Log("Stat %v", h)

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	name := be.Filename(h)
	obj := be.bucket.Object(name)
	info, err := obj.Attrs(ctx)
	if err != nil {
		debug.Log("Attrs() err %v", err)
		return restic.FileInfo{}, errors.Wrap(err, "Stat")
	}
	return restic.FileInfo{Size: info.Size, Name: h.Name}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (be *b2Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
	debug.Log("Test %v", h)

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	found := false
	name := be.Filename(h)
	obj := be.bucket.Object(name)
	info, err := obj.Attrs(ctx)
	if err == nil && info != nil && info.Status == b2.Uploaded {
		found = true
	}
	return found, nil
}

// Remove removes the blob with the given name and type.
func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
	debug.Log("Remove %v", h)

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	// the retry backend will also repeat the remove method up to 10 times
	for i := 0; i < 3; i++ {
		obj := be.bucket.Object(be.Filename(h))
		err := obj.Delete(ctx)
		if err == nil {
			// keep deleting until we are sure that no leftover file versions exist
			continue
		}
		// consider a file as removed if b2 informs us that it does not exist
		if b2.IsNotExist(err) {
			return nil
		}
		return errors.Wrap(err, "Delete")
	}

	return errors.New("failed to delete all file versions")
}

type semLocker struct {
	sema.Semaphore
}

func (sm *semLocker) Lock()   { sm.GetToken() }
func (sm *semLocker) Unlock() { sm.ReleaseToken() }

// List returns a channel that yields all names of blobs of type t.
func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
	debug.Log("List %v", t)

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	prefix, _ := be.Basedir(t)
	iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems), b2.ListLocker(&semLocker{be.sem}))

	for iter.Next() {
		obj := iter.Object()

		attrs, err := obj.Attrs(ctx)
		if err != nil {
			return err
		}

		fi := restic.FileInfo{
			Name: path.Base(obj.Name()),
			Size: attrs.Size,
		}

		if err := fn(fi); err != nil {
			return err
		}
	}
	if err := iter.Err(); err != nil {
		debug.Log("List: %v", err)
		return err
	}
	return nil
}

// Remove keys for a specified backend type.
func (be *b2Backend) removeKeys(ctx context.Context, t restic.FileType) error {
	debug.Log("removeKeys %v", t)
	return be.List(ctx, t, func(fi restic.FileInfo) error {
		return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name})
	})
}

// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *b2Backend) Delete(ctx context.Context) error {
	alltypes := []restic.FileType{
		restic.PackFile,
		restic.KeyFile,
		restic.LockFile,
		restic.SnapshotFile,
		restic.IndexFile}

	for _, t := range alltypes {
		err := be.removeKeys(ctx, t)
		if err != nil {
			return nil
		}
	}
	err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
	if err != nil && b2.IsNotExist(errors.Cause(err)) {
		err = nil
	}

	return err
}

// Close does nothing
func (be *b2Backend) Close() error { return nil }
<|endoftext|>


<commit_before>package metadata

import (
	"fmt"

	"github.com/freneticmonkey/migrate/go/test"
	"github.com/freneticmonkey/migrate/go/util"
)

// Metadata This struct stores the identification information for each table
// and table field in the target database.  This data is used to match the
// schema of the target database to the YAML schema
type Metadata struct {
	MDID       int64  `db:"mdid, autoincrement, primarykey" json:"mdid"`
	DB         int    `db:"db" json:"db"`
	PropertyID string `db:"property_id" json:"property_id"`
	ParentID   string `db:"parent_id" json:"parent_id"`
	Type       string `db:"type" json:"type"`
	Name       string `db:"name" json:"name"`
	Exists     bool   `db:"exists" json:"exists"`
}

// Insert Insert the Metadata into the Management DB
func (m *Metadata) Insert() error {
	m.DB = targetDBID
	if len(m.PropertyID) == 0 {
		return fmt.Errorf("Inserting empty Metadata")
	}
	if err := configured(); err != nil {
		return err
	}

	err := mgmtDb.Insert(m)
	if usingCache {
		if err == nil {
			cache = append(cache, *m)
		}
	}
	return err
}

// Update Update the Metadata in the Management DB
func (m *Metadata) Update() (err error) {
	if err = configured(); err != nil {
		return err
	}
	_, err = mgmtDb.Update(m)

	if err == nil {
		if usingCache {
			for i := range cache {
				if cache[i].MDID == m.MDID {
					cache[i] = *m
				}
			}
			cache = append(cache, *m)
		}
	}
	return err
}

// Delete Remove the Metadata from the database
func (m *Metadata) Delete() (err error) {
	if err := configured(); err != nil {
		return err
	}
	_, err = mgmtDb.Delete(m)
	if usingCache {
		for i := range cache {
			if cache[i].MDID == m.MDID {
				cache = append(cache[:i], cache[i+1:]...)
			}
		}
	}
	return err
}

// IsTable Returns if there is a value for ParentID. If empty the property is a table.
func (m *Metadata) IsTable() bool {
	return len(m.ParentID) == 0
}

// OnCreate Check if the Metadata is known to the database yet.  Depends on MDID being set != 0
func (m *Metadata) OnCreate() error {
	// If this Metadata hasn't been inserted into the database yet, insert it
	if m.MDID == 0 {

		if err := configured(); err != nil {
			return err
		}

		return m.Insert()
	}
	return nil
}

// ToDBRow Used to convert the Metadata into a unit test DBRow
func (m Metadata) ToDBRow() test.DBRow {
	return test.DBRow{
		m.MDID,
		m.DB,
		m.PropertyID,
		m.ParentID,
		m.Type,
		m.Name,
		m.Exists,
	}
}

// Load Uses the valud of MDID to load from the Management DB
func Load(mdid int64) (m *Metadata, err error) {
	var md Metadata

	if usingCache {

		for i := range cache {
			if cache[i].MDID == mdid {
				return &md, nil
			}
		}

	} else {

		if err = configured(); err != nil {
			return m, err
		}
		query := fmt.Sprintf("SELECT * FROM `metadata` WHERE mdid=%d", mdid)
		err = mgmtDb.SelectOne(&md, query)

		if err == nil {
			m = &md
		}
	}
	return m, err
}

// TableRegistered Returns a boolean indicating that the Table named 'name' is
// registered in the Metadata table
func TableRegistered(name string) (reg bool, err error) {
	if err = configured(); err != nil {
		return false, err
	}

	query := fmt.Sprintf("SELECT count(*) from metadata WHERE name=\"%s\" and type=\"Table\"", name)
	count, err := mgmtDb.SelectInt(query)
	return count > 0, err
}

// LoadAllTableMetadata Load all of the Metadata rows for a table with the
// matching Property and Database IDs
func LoadAllTableMetadata(name string) (md []Metadata, err error) {
	var tblMd Metadata

	tblMd, err = GetTableByName(name)

	if err != nil {
		return md, err
	}

	if usingCache {

		for _, meta := range cache {
			if meta.Name == name || meta.ParentID == tblMd.PropertyID {
				md = append(md, meta)
			}
		}

	} else {

		query := fmt.Sprintf("select * from metadata WHERE name = \"%s\" OR parent_id = \"%s\" AND db=%d", tblMd.Name, tblMd.PropertyID, targetDBID)
		_, err = mgmtDb.Select(&md, query)

	}

	util.ErrorCheckf(err, "There was a problem retrieving Metadata for Table with Name: [%s] and PropertyID: [%s]", name, tblMd.PropertyID)
	return md, err
}

// GetTableByName Get a Table metadata object from the database by name
func GetTableByName(name string) (md Metadata, err error) {

	// Find a Table (Tables don't have a ParentID)
	return GetByName(name, "")
}

// GetByName Get a metadata object from the database with name
func GetByName(name string, parentID string) (md Metadata, err error) {
	if err = configured(); err != nil {
		return md, err
	}

	if usingCache {

		for _, md := range cache {
			if md.Name == name && md.ParentID == parentID {
				return md, nil
			}
		}

	} else {
		query := fmt.Sprintf("SELECT * FROM metadata WHERE name=\"%s\"", name)

		if len(parentID) > 0 {
			query += fmt.Sprintf(" AND parent_id=\"%s\"", parentID)
		}

		err = mgmtDb.SelectOne(&md, query)
	}

	return md, err
}

// UpdateCache Build a localstore of the Metadata Management DB for the target DB
func UpdateCache() error {
	query := fmt.Sprintf("SELECT * FROM metadata WHERE db = %d", targetDBID)

	_, err := mgmtDb.Select(&cache, query)
	return err
}
<commit_msg>Added additional validation to Metadata creation to ensure orphaned Metadata records are not created<commit_after>package metadata

import (
	"fmt"

	"github.com/freneticmonkey/migrate/go/test"
	"github.com/freneticmonkey/migrate/go/util"
)

// Metadata This struct stores the identification information for each table
// and table field in the target database.  This data is used to match the
// schema of the target database to the YAML schema
type Metadata struct {
	MDID       int64  `db:"mdid, autoincrement, primarykey" json:"mdid"`
	DB         int    `db:"db" json:"db"`
	PropertyID string `db:"property_id" json:"property_id"`
	ParentID   string `db:"parent_id" json:"parent_id"`
	Type       string `db:"type" json:"type"`
	Name       string `db:"name" json:"name"`
	Exists     bool   `db:"exists" json:"exists"`
}

// Insert Insert the Metadata into the Management DB
func (m *Metadata) Insert() error {
	m.DB = targetDBID
	if len(m.PropertyID) == 0 {
		return fmt.Errorf("Inserting empty Metadata")
	}
	if len(m.ParentID) == 0 && m.Type != "Table" {
		return fmt.Errorf("Inserting Table field Metadata with no ParentID")
	}
	if err := configured(); err != nil {
		return err
	}

	err := mgmtDb.Insert(m)
	if usingCache {
		if err == nil {
			cache = append(cache, *m)
		}
	}
	return err
}

// Update Update the Metadata in the Management DB
func (m *Metadata) Update() (err error) {
	if err = configured(); err != nil {
		return err
	}
	_, err = mgmtDb.Update(m)

	if err == nil {
		if usingCache {
			for i := range cache {
				if cache[i].MDID == m.MDID {
					cache[i] = *m
				}
			}
			cache = append(cache, *m)
		}
	}
	return err
}

// Delete Remove the Metadata from the database
func (m *Metadata) Delete() (err error) {
	if err := configured(); err != nil {
		return err
	}
	_, err = mgmtDb.Delete(m)
	if usingCache {
		for i := range cache {
			if cache[i].MDID == m.MDID {
				cache = append(cache[:i], cache[i+1:]...)
			}
		}
	}
	return err
}

// IsTable Returns if there is a value for ParentID. If empty the property is a table.
func (m *Metadata) IsTable() bool {
	return len(m.ParentID) == 0
}

// OnCreate Check if the Metadata is known to the database yet.  Depends on MDID being set != 0
func (m *Metadata) OnCreate() error {
	// If this Metadata hasn't been inserted into the database yet, insert it
	if m.MDID == 0 {

		if err := configured(); err != nil {
			return err
		}

		return m.Insert()
	}
	return nil
}

// ToDBRow Used to convert the Metadata into a unit test DBRow
func (m Metadata) ToDBRow() test.DBRow {
	return test.DBRow{
		m.MDID,
		m.DB,
		m.PropertyID,
		m.ParentID,
		m.Type,
		m.Name,
		m.Exists,
	}
}

// Load Uses the valud of MDID to load from the Management DB
func Load(mdid int64) (m *Metadata, err error) {
	var md Metadata

	if usingCache {

		for i := range cache {
			if cache[i].MDID == mdid {
				return &md, nil
			}
		}

	} else {

		if err = configured(); err != nil {
			return m, err
		}
		query := fmt.Sprintf("SELECT * FROM `metadata` WHERE mdid=%d", mdid)
		err = mgmtDb.SelectOne(&md, query)

		if err == nil {
			m = &md
		}
	}
	return m, err
}

// TableRegistered Returns a boolean indicating that the Table named 'name' is
// registered in the Metadata table
func TableRegistered(name string) (reg bool, err error) {
	if err = configured(); err != nil {
		return false, err
	}

	query := fmt.Sprintf("SELECT count(*) from metadata WHERE name=\"%s\" and type=\"Table\"", name)
	count, err := mgmtDb.SelectInt(query)
	return count > 0, err
}

// LoadAllTableMetadata Load all of the Metadata rows for a table with the
// matching Property and Database IDs
func LoadAllTableMetadata(name string) (md []Metadata, err error) {
	var tblMd Metadata

	tblMd, err = GetTableByName(name)

	if err != nil {
		return md, err
	}

	if usingCache {

		for _, meta := range cache {
			if meta.Name == name || meta.ParentID == tblMd.PropertyID {
				md = append(md, meta)
			}
		}

	} else {

		query := fmt.Sprintf("select * from metadata WHERE name = \"%s\" OR parent_id = \"%s\" AND db=%d", tblMd.Name, tblMd.PropertyID, targetDBID)
		_, err = mgmtDb.Select(&md, query)

	}

	util.ErrorCheckf(err, "There was a problem retrieving Metadata for Table with Name: [%s] and PropertyID: [%s]", name, tblMd.PropertyID)
	return md, err
}

// GetTableByName Get a Table metadata object from the database by name
func GetTableByName(name string) (md Metadata, err error) {

	// Find a Table (Tables don't have a ParentID)
	return GetByName(name, "")
}

// GetByName Get a metadata object from the database with name
func GetByName(name string, parentID string) (md Metadata, err error) {
	if err = configured(); err != nil {
		return md, err
	}

	if usingCache {

		for _, md := range cache {
			if md.Name == name && md.ParentID == parentID {
				return md, nil
			}
		}

	} else {
		query := fmt.Sprintf("SELECT * FROM metadata WHERE name=\"%s\"", name)

		if len(parentID) > 0 {
			query += fmt.Sprintf(" AND parent_id=\"%s\"", parentID)
		}

		err = mgmtDb.SelectOne(&md, query)
	}

	return md, err
}

// UpdateCache Build a localstore of the Metadata Management DB for the target DB
func UpdateCache() error {
	query := fmt.Sprintf("SELECT * FROM metadata WHERE db = %d", targetDBID)

	_, err := mgmtDb.Select(&cache, query)
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/hashicorp/consul/api"
	"github.com/nu7hatch/gouuid"

	"code.cloudfoundry.org/auctioneer"
	"code.cloudfoundry.org/auctioneer/auctionmetricemitterdelegate"
	"code.cloudfoundry.org/auctioneer/auctionrunnerdelegate"
	"code.cloudfoundry.org/auctioneer/handlers"
	"code.cloudfoundry.org/bbs"
	"code.cloudfoundry.org/cfhttp"
	"code.cloudfoundry.org/cflager"
	"code.cloudfoundry.org/consuladapter"
	"code.cloudfoundry.org/debugserver"
	"code.cloudfoundry.org/locket"
	"code.cloudfoundry.org/rep"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/localip"

	"code.cloudfoundry.org/auction/auctionrunner"
	"code.cloudfoundry.org/auction/auctiontypes"
	"code.cloudfoundry.org/clock"
	"github.com/cloudfoundry/dropsonde"
	"github.com/cloudfoundry/gunk/workpool"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
)

var communicationTimeout = flag.Duration(
	"communicationTimeout",
	10*time.Second,
	"Timeout applied to all HTTP requests.",
)

var cellStateTimeout = flag.Duration(
	"cellStateTimeout",
	1*time.Second,
	"Timeout applied to HTTP requests to the Cell State endpoint.",
)

var consulCluster = flag.String(
	"consulCluster",
	"",
	"comma-separated list of consul server addresses (ip:port)",
)

var dropsondePort = flag.Int(
	"dropsondePort",
	3457,
	"port the local metron agent is listening on",
)

var lockTTL = flag.Duration(
	"lockTTL",
	locket.LockTTL,
	"TTL for service lock",
)

var lockRetryInterval = flag.Duration(
	"lockRetryInterval",
	locket.RetryInterval,
	"interval to wait before retrying a failed lock acquisition",
)

var listenAddr = flag.String(
	"listenAddr",
	"0.0.0.0:9016",
	"host:port to serve auction and LRP stop requests on",
)

var bbsAddress = flag.String(
	"bbsAddress",
	"",
	"Address to the BBS Server",
)

var bbsCACert = flag.String(
	"bbsCACert",
	"",
	"path to certificate authority cert used for mutually authenticated TLS BBS communication",
)

var bbsClientCert = flag.String(
	"bbsClientCert",
	"",
	"path to client cert used for mutually authenticated TLS BBS communication",
)

var bbsClientKey = flag.String(
	"bbsClientKey",
	"",
	"path to client key used for mutually authenticated TLS BBS communication",
)

var bbsClientSessionCacheSize = flag.Int(
	"bbsClientSessionCacheSize",
	0,
	"Capacity of the ClientSessionCache option on the TLS configuration. If zero, golang's default will be used",
)

var bbsMaxIdleConnsPerHost = flag.Int(
	"bbsMaxIdleConnsPerHost",
	0,
	"Controls the maximum number of idle (keep-alive) connctions per host. If zero, golang's default will be used",
)

var auctionRunnerWorkers = flag.Int(
	"auctionRunnerWorkers",
	1000,
	"Max concurrency for cell operations in the auction runner",
)

var startingContainerWeight = flag.Float64(
	"startingContainerWeight",
	0.25,
	"Factor to bias against cells with starting containers (0.0 - 1.0)",
)

const (
	auctionRunnerTimeout = 10 * time.Second
	dropsondeOrigin      = "auctioneer"
	serverProtocol       = "http"
)

func main() {
	debugserver.AddFlags(flag.CommandLine)
	cflager.AddFlags(flag.CommandLine)
	flag.Parse()

	cfhttp.Initialize(*communicationTimeout)

	logger, reconfigurableSink := cflager.New("auctioneer")
	initializeDropsonde(logger)

	if err := validateBBSAddress(); err != nil {
		logger.Fatal("invalid-bbs-address", err)
	}

	consulClient, err := consuladapter.NewClientFromUrl(*consulCluster)
	if err != nil {
		logger.Fatal("new-client-failed", err)
	}

	port, err := strconv.Atoi(strings.Split(*listenAddr, ":")[1])
	if err != nil {
		logger.Fatal("invalid-port", err)
	}

	clock := clock.NewClock()
	auctioneerServiceClient := auctioneer.NewServiceClient(consulClient, clock)

	auctionRunner := initializeAuctionRunner(logger, *cellStateTimeout,
		initializeBBSClient(logger), *startingContainerWeight)
	auctionServer := initializeAuctionServer(logger, auctionRunner)
	lockMaintainer := initializeLockMaintainer(logger, auctioneerServiceClient, port)
	registrationRunner := initializeRegistrationRunner(logger, consulClient, clock, port)

	members := grouper.Members{
		{"lock-maintainer", lockMaintainer},
		{"auction-runner", auctionRunner},
		{"auction-server", auctionServer},
		{"registration-runner", registrationRunner},
	}

	if dbgAddr := debugserver.DebugAddress(flag.CommandLine); dbgAddr != "" {
		members = append(grouper.Members{
			{"debug-server", debugserver.Runner(dbgAddr, reconfigurableSink)},
		}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)

	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started")

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
}

func initializeAuctionRunner(logger lager.Logger, cellStateTimeout time.Duration, bbsClient bbs.InternalClient, startingContainerWeight float64) auctiontypes.AuctionRunner {
	httpClient := cfhttp.NewClient()
	stateClient := cfhttp.NewCustomTimeoutClient(cellStateTimeout)
	repClientFactory := rep.NewClientFactory(httpClient, stateClient)

	delegate := auctionrunnerdelegate.New(repClientFactory, bbsClient, logger)
	metricEmitter := auctionmetricemitterdelegate.New()
	workPool, err := workpool.NewWorkPool(*auctionRunnerWorkers)
	if err != nil {
		logger.Fatal("failed-to-construct-auction-runner-workpool", err, lager.Data{"num-workers": *auctionRunnerWorkers}) // should never happen
	}

	return auctionrunner.New(
		logger,
		delegate,
		metricEmitter,
		clock.NewClock(),
		workPool,
		startingContainerWeight,
	)
}

func initializeDropsonde(logger lager.Logger) {
	dropsondeDestination := fmt.Sprint("localhost:", *dropsondePort)
	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed to initialize dropsonde: %v", err)
	}
}

func initializeAuctionServer(logger lager.Logger, runner auctiontypes.AuctionRunner) ifrit.Runner {
	return http_server.New(*listenAddr, handlers.New(runner, logger))
}

func initializeRegistrationRunner(logger lager.Logger, consulClient consuladapter.Client, clock clock.Clock, port int) ifrit.Runner {
	registration := &api.AgentServiceRegistration{
		Name: "auctioneer",
		Port: port,
		Check: &api.AgentServiceCheck{
			TTL: "3s",
		},
	}
	return locket.NewRegistrationRunner(logger, registration, consulClient, locket.RetryInterval, clock)
}

func initializeLockMaintainer(logger lager.Logger, serviceClient auctioneer.ServiceClient, port int) ifrit.Runner {
	uuid, err := uuid.NewV4()
	if err != nil {
		logger.Fatal("Couldn't generate uuid", err)
	}

	localIP, err := localip.LocalIP()
	if err != nil {
		logger.Fatal("Couldn't determine local IP", err)
	}

	address := fmt.Sprintf("%s://%s:%d", serverProtocol, localIP, port)
	auctioneerPresence := auctioneer.NewPresence(uuid.String(), address)

	lockMaintainer, err := serviceClient.NewAuctioneerLockRunner(logger, auctioneerPresence, *lockRetryInterval, *lockTTL)
	if err != nil {
		logger.Fatal("Couldn't create lock maintainer", err)
	}

	return lockMaintainer
}

func validateBBSAddress() error {
	if *bbsAddress == "" {
		return errors.New("bbsAddress is required")
	}
	return nil
}

func initializeBBSClient(logger lager.Logger) bbs.InternalClient {
	bbsURL, err := url.Parse(*bbsAddress)
	if err != nil {
		logger.Fatal("Invalid BBS URL", err)
	}

	if bbsURL.Scheme != "https" {
		return bbs.NewClient(*bbsAddress)
	}

	bbsClient, err := bbs.NewSecureClient(*bbsAddress, *bbsCACert, *bbsClientCert, *bbsClientKey, *bbsClientSessionCacheSize, *bbsMaxIdleConnsPerHost)
	if err != nil {
		logger.Fatal("Failed to configure secure BBS client", err)
	}
	return bbsClient
}
<commit_msg>Move localip to code.cloudfoundry.org<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/hashicorp/consul/api"
	"github.com/nu7hatch/gouuid"

	"code.cloudfoundry.org/auctioneer"
	"code.cloudfoundry.org/auctioneer/auctionmetricemitterdelegate"
	"code.cloudfoundry.org/auctioneer/auctionrunnerdelegate"
	"code.cloudfoundry.org/auctioneer/handlers"
	"code.cloudfoundry.org/bbs"
	"code.cloudfoundry.org/cfhttp"
	"code.cloudfoundry.org/cflager"
	"code.cloudfoundry.org/consuladapter"
	"code.cloudfoundry.org/debugserver"
	"code.cloudfoundry.org/localip"
	"code.cloudfoundry.org/locket"
	"code.cloudfoundry.org/rep"
	"github.com/pivotal-golang/lager"

	"code.cloudfoundry.org/auction/auctionrunner"
	"code.cloudfoundry.org/auction/auctiontypes"
	"code.cloudfoundry.org/clock"
	"github.com/cloudfoundry/dropsonde"
	"github.com/cloudfoundry/gunk/workpool"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
)

var communicationTimeout = flag.Duration(
	"communicationTimeout",
	10*time.Second,
	"Timeout applied to all HTTP requests.",
)

var cellStateTimeout = flag.Duration(
	"cellStateTimeout",
	1*time.Second,
	"Timeout applied to HTTP requests to the Cell State endpoint.",
)

var consulCluster = flag.String(
	"consulCluster",
	"",
	"comma-separated list of consul server addresses (ip:port)",
)

var dropsondePort = flag.Int(
	"dropsondePort",
	3457,
	"port the local metron agent is listening on",
)

var lockTTL = flag.Duration(
	"lockTTL",
	locket.LockTTL,
	"TTL for service lock",
)

var lockRetryInterval = flag.Duration(
	"lockRetryInterval",
	locket.RetryInterval,
	"interval to wait before retrying a failed lock acquisition",
)

var listenAddr = flag.String(
	"listenAddr",
	"0.0.0.0:9016",
	"host:port to serve auction and LRP stop requests on",
)

var bbsAddress = flag.String(
	"bbsAddress",
	"",
	"Address to the BBS Server",
)

var bbsCACert = flag.String(
	"bbsCACert",
	"",
	"path to certificate authority cert used for mutually authenticated TLS BBS communication",
)

var bbsClientCert = flag.String(
	"bbsClientCert",
	"",
	"path to client cert used for mutually authenticated TLS BBS communication",
)

var bbsClientKey = flag.String(
	"bbsClientKey",
	"",
	"path to client key used for mutually authenticated TLS BBS communication",
)

var bbsClientSessionCacheSize = flag.Int(
	"bbsClientSessionCacheSize",
	0,
	"Capacity of the ClientSessionCache option on the TLS configuration. If zero, golang's default will be used",
)

var bbsMaxIdleConnsPerHost = flag.Int(
	"bbsMaxIdleConnsPerHost",
	0,
	"Controls the maximum number of idle (keep-alive) connctions per host. If zero, golang's default will be used",
)

var auctionRunnerWorkers = flag.Int(
	"auctionRunnerWorkers",
	1000,
	"Max concurrency for cell operations in the auction runner",
)

var startingContainerWeight = flag.Float64(
	"startingContainerWeight",
	0.25,
	"Factor to bias against cells with starting containers (0.0 - 1.0)",
)

const (
	auctionRunnerTimeout = 10 * time.Second
	dropsondeOrigin      = "auctioneer"
	serverProtocol       = "http"
)

func main() {
	debugserver.AddFlags(flag.CommandLine)
	cflager.AddFlags(flag.CommandLine)
	flag.Parse()

	cfhttp.Initialize(*communicationTimeout)

	logger, reconfigurableSink := cflager.New("auctioneer")
	initializeDropsonde(logger)

	if err := validateBBSAddress(); err != nil {
		logger.Fatal("invalid-bbs-address", err)
	}

	consulClient, err := consuladapter.NewClientFromUrl(*consulCluster)
	if err != nil {
		logger.Fatal("new-client-failed", err)
	}

	port, err := strconv.Atoi(strings.Split(*listenAddr, ":")[1])
	if err != nil {
		logger.Fatal("invalid-port", err)
	}

	clock := clock.NewClock()
	auctioneerServiceClient := auctioneer.NewServiceClient(consulClient, clock)

	auctionRunner := initializeAuctionRunner(logger, *cellStateTimeout,
		initializeBBSClient(logger), *startingContainerWeight)
	auctionServer := initializeAuctionServer(logger, auctionRunner)
	lockMaintainer := initializeLockMaintainer(logger, auctioneerServiceClient, port)
	registrationRunner := initializeRegistrationRunner(logger, consulClient, clock, port)

	members := grouper.Members{
		{"lock-maintainer", lockMaintainer},
		{"auction-runner", auctionRunner},
		{"auction-server", auctionServer},
		{"registration-runner", registrationRunner},
	}

	if dbgAddr := debugserver.DebugAddress(flag.CommandLine); dbgAddr != "" {
		members = append(grouper.Members{
			{"debug-server", debugserver.Runner(dbgAddr, reconfigurableSink)},
		}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)

	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started")

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
}

func initializeAuctionRunner(logger lager.Logger, cellStateTimeout time.Duration, bbsClient bbs.InternalClient, startingContainerWeight float64) auctiontypes.AuctionRunner {
	httpClient := cfhttp.NewClient()
	stateClient := cfhttp.NewCustomTimeoutClient(cellStateTimeout)
	repClientFactory := rep.NewClientFactory(httpClient, stateClient)

	delegate := auctionrunnerdelegate.New(repClientFactory, bbsClient, logger)
	metricEmitter := auctionmetricemitterdelegate.New()
	workPool, err := workpool.NewWorkPool(*auctionRunnerWorkers)
	if err != nil {
		logger.Fatal("failed-to-construct-auction-runner-workpool", err, lager.Data{"num-workers": *auctionRunnerWorkers}) // should never happen
	}

	return auctionrunner.New(
		logger,
		delegate,
		metricEmitter,
		clock.NewClock(),
		workPool,
		startingContainerWeight,
	)
}

func initializeDropsonde(logger lager.Logger) {
	dropsondeDestination := fmt.Sprint("localhost:", *dropsondePort)
	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed to initialize dropsonde: %v", err)
	}
}

func initializeAuctionServer(logger lager.Logger, runner auctiontypes.AuctionRunner) ifrit.Runner {
	return http_server.New(*listenAddr, handlers.New(runner, logger))
}

func initializeRegistrationRunner(logger lager.Logger, consulClient consuladapter.Client, clock clock.Clock, port int) ifrit.Runner {
	registration := &api.AgentServiceRegistration{
		Name: "auctioneer",
		Port: port,
		Check: &api.AgentServiceCheck{
			TTL: "3s",
		},
	}
	return locket.NewRegistrationRunner(logger, registration, consulClient, locket.RetryInterval, clock)
}

func initializeLockMaintainer(logger lager.Logger, serviceClient auctioneer.ServiceClient, port int) ifrit.Runner {
	uuid, err := uuid.NewV4()
	if err != nil {
		logger.Fatal("Couldn't generate uuid", err)
	}

	localIP, err := localip.LocalIP()
	if err != nil {
		logger.Fatal("Couldn't determine local IP", err)
	}

	address := fmt.Sprintf("%s://%s:%d", serverProtocol, localIP, port)
	auctioneerPresence := auctioneer.NewPresence(uuid.String(), address)

	lockMaintainer, err := serviceClient.NewAuctioneerLockRunner(logger, auctioneerPresence, *lockRetryInterval, *lockTTL)
	if err != nil {
		logger.Fatal("Couldn't create lock maintainer", err)
	}

	return lockMaintainer
}

func validateBBSAddress() error {
	if *bbsAddress == "" {
		return errors.New("bbsAddress is required")
	}
	return nil
}

func initializeBBSClient(logger lager.Logger) bbs.InternalClient {
	bbsURL, err := url.Parse(*bbsAddress)
	if err != nil {
		logger.Fatal("Invalid BBS URL", err)
	}

	if bbsURL.Scheme != "https" {
		return bbs.NewClient(*bbsAddress)
	}

	bbsClient, err := bbs.NewSecureClient(*bbsAddress, *bbsCACert, *bbsClientCert, *bbsClientKey, *bbsClientSessionCacheSize, *bbsMaxIdleConnsPerHost)
	if err != nil {
		logger.Fatal("Failed to configure secure BBS client", err)
	}
	return bbsClient
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/darkcrux/gopherduty"
	"github.com/hashicorp/consul/api"
	"github.com/nlopes/slack"
	"gopkg.in/gomail.v2"
	"time"
)

// AlertHandlers are responsible for alerting to some external endpoint
// when given an alert (email, pagerduty, etc)
type AlertHandler interface {
	Alert(datacenter string, alert *AlertState)
}

type StdoutHandler struct {
	LogLevel string `mapstructure:"log_level"`
	logger   *log.Logger
}

func (handler StdoutHandler) Alert(datacenter string, alert *AlertState) {
	text := []string{alert.Message}
	if alert.Details != "" {
		text = append(text, strings.Split(alert.Details, "\n")...)
	}
	for _, line := range text {
		switch strings.ToLower(handler.LogLevel) {
		case "panic":
			handler.logger.Panic(line)
		case "fatal":
			handler.logger.Fatal(line)
		case "error":
			handler.logger.Error(line)
		case "warn", "warning":
			handler.logger.Warn(line)
		case "info":
			handler.logger.Info(line)
		case "debug":
			handler.logger.Debug(line)
		}
	}
}

type EmailHandler struct {
	Recipients []string `mapstructure:"recipients"`
	MaxRetries int      `mapstructure:"max_retries"`
}

func (handler EmailHandler) Alert(datacenter string, alert *AlertState) {
	for _, recipient := range handler.Recipients {
		// Get the mail server to use for this recipient
		records, err := net.LookupMX(strings.Split(recipient, "@")[1])
		if err != nil {
			log.Error("Error looking up email server: ", err)
			continue
		}

		m := gomail.NewMessage()
		m.SetAddressHeader("From", "consul-alerting@noreply.com", "Consul Alerting")
		m.SetAddressHeader("To", recipient, "")

		m.SetHeader("Subject", alert.Message)
		m.SetBody("text/plain", alert.Details)

		d := gomail.NewPlainDialer(records[0].Host, 25, "", "")

		tries := 0
		for tries <= handler.MaxRetries {
			if err := d.DialAndSend(m); err != nil {
				log.Error("Error sending alert email: ", err)
				log.Error("Retrying email in 5s...")
				time.Sleep(5 * time.Second)
				tries++
			} else {
				break
			}
		}
	}
}

type PagerdutyHandler struct {
	ServiceKey string `mapstructure:"service_key"`
	MaxRetries int    `mapstructure:"max_retries"`
}

func (handler PagerdutyHandler) Alert(datacenter string, alert *AlertState) {
	client := gopherduty.NewClient(handler.ServiceKey)
	client.MaxRetry = handler.MaxRetries

	// This key needs to be unique to the datacenter and service/node we're alerting on
	incidentKey := datacenter + "-" + alert.Service + "-" + alert.Tag + "-" + alert.Node

	if alert.Status != api.HealthPassing {
		client.Trigger(incidentKey, alert.Message, "", "", alert.Details)
	} else {
		client.Resolve(incidentKey, alert.Message, alert.Details)
	}
}

type SlackHandler struct {
	Token       string `mapstructure:"api_token"`
	ChannelName string `mapstructure:"channel_name"`
	MaxRetries  int    `mapstructure:"max_retries"`
}

const slackMessageFormat = `
*%s*
%s
`

func (handler SlackHandler) Alert(datacenter string, alert *AlertState) {
	api := slack.New(handler.Token)
	message := fmt.Sprintf(slackMessageFormat, alert.Message, alert.Details)
	tries := 0

	for tries <= handler.MaxRetries {
		_, _, err := api.PostMessage(handler.ChannelName, message, slack.PostMessageParameters{})

		if err != nil {
			log.Errorf("Error sending alert to Slack (channel: %s): %s", handler.ChannelName, err)
			log.Errorf("Retrying alert to slack in 5s...")
			time.Sleep(5 * time.Second)
		} else {
			break
		}

		tries++
	}
}
<commit_msg>Log PagerDuty API errors when they occur.<commit_after>package main

import (
	"fmt"
	"net"
	"strings"

	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/darkcrux/gopherduty"
	"github.com/hashicorp/consul/api"
	"github.com/nlopes/slack"
	"gopkg.in/gomail.v2"
)

// AlertHandlers are responsible for alerting to some external endpoint
// when given an alert (email, pagerduty, etc)
type AlertHandler interface {
	Alert(datacenter string, alert *AlertState)
}

type StdoutHandler struct {
	LogLevel string `mapstructure:"log_level"`
	logger   *log.Logger
}

func (handler StdoutHandler) Alert(datacenter string, alert *AlertState) {
	text := []string{alert.Message}
	if alert.Details != "" {
		text = append(text, strings.Split(alert.Details, "\n")...)
	}
	for _, line := range text {
		switch strings.ToLower(handler.LogLevel) {
		case "panic":
			handler.logger.Panic(line)
		case "fatal":
			handler.logger.Fatal(line)
		case "error":
			handler.logger.Error(line)
		case "warn", "warning":
			handler.logger.Warn(line)
		case "info":
			handler.logger.Info(line)
		case "debug":
			handler.logger.Debug(line)
		}
	}
}

type EmailHandler struct {
	Recipients []string `mapstructure:"recipients"`
	MaxRetries int      `mapstructure:"max_retries"`
}

func (handler EmailHandler) Alert(datacenter string, alert *AlertState) {
	for _, recipient := range handler.Recipients {
		// Get the mail server to use for this recipient
		records, err := net.LookupMX(strings.Split(recipient, "@")[1])
		if err != nil {
			log.Error("Error looking up email server: ", err)
			continue
		}

		m := gomail.NewMessage()
		m.SetAddressHeader("From", "consul-alerting@noreply.com", "Consul Alerting")
		m.SetAddressHeader("To", recipient, "")

		m.SetHeader("Subject", alert.Message)
		m.SetBody("text/plain", alert.Details)

		d := gomail.NewPlainDialer(records[0].Host, 25, "", "")

		tries := 0
		for tries <= handler.MaxRetries {
			if err := d.DialAndSend(m); err != nil {
				log.Error("Error sending alert email: ", err)
				log.Error("Retrying email in 5s...")
				time.Sleep(5 * time.Second)
				tries++
			} else {
				break
			}
		}
	}
}

type PagerdutyHandler struct {
	ServiceKey string `mapstructure:"service_key"`
	MaxRetries int    `mapstructure:"max_retries"`
}

func (handler PagerdutyHandler) Alert(datacenter string, alert *AlertState) {
	client := gopherduty.NewClient(handler.ServiceKey)
	client.MaxRetry = handler.MaxRetries

	// This key needs to be unique to the datacenter and service/node we're alerting on
	incidentKey := datacenter + "-" + alert.Service + "-" + alert.Tag + "-" + alert.Node

	var resp *gopherduty.PagerDutyResponse
	if alert.Status != api.HealthPassing {
		resp = client.Trigger(incidentKey, alert.Message, "", "", alert.Details)
	} else {
		resp = client.Resolve(incidentKey, alert.Message, alert.Details)
	}

	for _, err := range resp.Errors {
		log.Errorf("Error sending alert to PagerDuty: %v (details: %v, message: %v)", err, alert.Details, alert.Message)
	}
}

type SlackHandler struct {
	Token       string `mapstructure:"api_token"`
	ChannelName string `mapstructure:"channel_name"`
	MaxRetries  int    `mapstructure:"max_retries"`
}

const slackMessageFormat = `
*%s*
%s
`

func (handler SlackHandler) Alert(datacenter string, alert *AlertState) {
	api := slack.New(handler.Token)
	message := fmt.Sprintf(slackMessageFormat, alert.Message, alert.Details)
	tries := 0

	for tries <= handler.MaxRetries {
		_, _, err := api.PostMessage(handler.ChannelName, message, slack.PostMessageParameters{})

		if err != nil {
			log.Errorf("Error sending alert to Slack (channel: %s): %s", handler.ChannelName, err)
			log.Errorf("Retrying alert to slack in 5s...")
			time.Sleep(5 * time.Second)
		} else {
			break
		}

		tries++
	}
}
<|endoftext|>