<commit_before>//
// Author:: Zachary Schneider (<schneider@boundary.com>)
// Copyright:: Copyright (c) 2015 Boundary, Inc.
// License:: Apache License, Version 2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"fmt"
	"log"

	"github.com/hashicorp/consul/api"
	"github.com/jwaldrip/odin/cli"
	"github.com/hashicorp/consul/lib"
)

var Role = cli.NewSubCommand("role", "Role operations", roleRun)

func init() {
	Role.DefineParams("action")

	Role.SetLongDescription(`
Interact with cascade roles

Actions:
  list - list local roles
  listAll - list all nodes and roles
  find <role> - list nodes with role
  set <roles> - set local roles (replaces current)
  append <roles> - append roles to local set
  `)
}

func roleRun(c cli.Command) {
	switch c.Param("action").String() {
	case "list":
		roleList(c)
	case "set":
		roleSet(c)
	case "listAll":
		roleListAll(c)
	case "find":
		roleFind(c)
	case "append":
		roleAppend(c)
	default:
		cli.ShowUsage(c)
	}
}

func roleListAll(_ cli.Command) {

	roles, err := allNodeRoles()
	if err != nil {
		log.Fatalln("err: ", err)
	}

	for k, v := range roles {
		printRole(k, v)
	}
}

func roleList(_ cli.Command) {

	nodeRoles, err := allNodeRoles()
	if err != nil {
		log.Fatalln("err: ", err)
	}

	myKey, err := selfKey()
	if err != nil {
		log.Fatalln("err: ", err)
	}
	printRole(myKey, nodeRoles[myKey])
}

func roleFind(c cli.Command) {
	cmdRoles := c.Args().Strings()
	if len(cmdRoles) == 0 {
		log.Fatalln("Must specify a role to find")
	}
	if len(cmdRoles) != 1 {
		// maybe we could support multiple but i don't think it's necessary
		log.Fatalln("One role only is supported for `find` command")
	}
	role := cmdRoles[0]
	allRoles, err := allNodeRoles()
	if err != nil {
		log.Fatalln("err: ", err)
	}

	fmt.Printf("All nodes containing role %s:\n\n", role)
	for node, r := range allRoles {
		if lib.StrContains(r, role) {
			printRole(node, r)
		}
	}
}

func roleSet(c cli.Command) {
	roleActualSet(c.Args().Strings(), c)
}

func roleActualSet(roles []string, c cli.Command) {
	client, _ := api.NewClient(api.DefaultConfig())
	agent := client.Agent()

	reg := &api.AgentServiceRegistration{
		Name: "cascade",
		Tags: roles,
	}

	if err := agent.ServiceRegister(reg); err != nil {
		log.Fatalln("err: ", err)
	}

	roleList(c)
}

func roleAppend(c cli.Command) {
	nodeRoles, err := allNodeRoles()
	if err != nil {
		log.Fatalln("err: ", err)
	}

	myKey, err := selfKey()
	if err != nil {
		log.Fatalln("err: ", err)
	}

	var finalSet []string
	for _, role := range nodeRoles[myKey] {
		finalSet = append(finalSet, role)
	}

	for _, role := range c.Args().Strings() {
		if !lib.StrContains(finalSet, role) {
			finalSet = append(finalSet, role)
		}
	}

	roleActualSet(finalSet, c)
}

func allNodeRoles() (map[string][]string, error) {
	roleMap := make(map[string][]string)
	client, _ := api.NewClient(api.DefaultConfig())
	catalog := client.Catalog()
	cascadeServices, _, err := catalog.Service("cascade", "", nil)
	if err != nil {
		return nil, err
	}

	for _, service := range cascadeServices {
		roleMap[ makeKey(service.Node, service.Address) ] = service.ServiceTags
	}
	return roleMap, nil
}

func selfKey() (string, error) {
	client, _ := api.NewClient(api.DefaultConfig())
	agent := client.Agent()

	self, err := agent.Self()

	if err != nil {
		return "", err
	}

	return makeKey(self["Config"]["NodeName"].(string), self["Config"]["AdvertiseAddr"].(string)), nil
}

func makeKey(hostName string, hostAddr string) (string) {
	return fmt.Sprintf("%s (%s)", hostName, hostAddr)
}

func printRole(key string, roles []string) {
	fmt.Println(key + ":")
	for _, role := range roles {
		fmt.Println("  -", role)
	}
}
<commit_msg>reduce dep footprint<commit_after>//
// Author:: Zachary Schneider (<schneider@boundary.com>)
// Copyright:: Copyright (c) 2015 Boundary, Inc.
// License:: Apache License, Version 2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"fmt"
	"log"

	"github.com/hashicorp/consul/api"
	"github.com/jwaldrip/odin/cli"
)

var Role = cli.NewSubCommand("role", "Role operations", roleRun)

func init() {
	Role.DefineParams("action")

	Role.SetLongDescription(`
Interact with cascade roles

Actions:
  list - list local roles
  listAll - list all nodes and roles
  find <role> - list nodes with role
  set <roles> - set local roles (replaces current)
  append <roles> - append roles to local set
  `)
}

func roleRun(c cli.Command) {
	switch c.Param("action").String() {
	case "list":
		roleList(c)
	case "set":
		roleSet(c)
	case "listAll":
		roleListAll(c)
	case "find":
		roleFind(c)
	case "append":
		roleAppend(c)
	default:
		cli.ShowUsage(c)
	}
}

func roleListAll(_ cli.Command) {

	roles, err := allNodeRoles()
	if err != nil {
		log.Fatalln("err: ", err)
	}

	for k, v := range roles {
		printRole(k, v)
	}
}

func roleList(_ cli.Command) {

	nodeRoles, err := allNodeRoles()
	if err != nil {
		log.Fatalln("err: ", err)
	}

	myKey, err := selfKey()
	if err != nil {
		log.Fatalln("err: ", err)
	}
	printRole(myKey, nodeRoles[myKey])
}

func roleFind(c cli.Command) {
	cmdRoles := c.Args().Strings()
	if len(cmdRoles) == 0 {
		log.Fatalln("Must specify a role to find")
	}
	if len(cmdRoles) != 1 {
		// maybe we could support multiple but i don't think it's necessary
		log.Fatalln("One role only is supported for `find` command")
	}
	role := cmdRoles[0]
	allRoles, err := allNodeRoles()
	if err != nil {
		log.Fatalln("err: ", err)
	}

	fmt.Printf("All nodes containing role %s:\n\n", role)
	for node, r := range allRoles {
		if StrContains(r, role) {
			printRole(node, r)
		}
	}
}

func roleSet(c cli.Command) {
	roleActualSet(c.Args().Strings(), c)
}

func roleActualSet(roles []string, c cli.Command) {
	client, _ := api.NewClient(api.DefaultConfig())
	agent := client.Agent()

	reg := &api.AgentServiceRegistration{
		Name: "cascade",
		Tags: roles,
	}

	if err := agent.ServiceRegister(reg); err != nil {
		log.Fatalln("err: ", err)
	}

	roleList(c)
}

func roleAppend(c cli.Command) {
	nodeRoles, err := allNodeRoles()
	if err != nil {
		log.Fatalln("err: ", err)
	}

	myKey, err := selfKey()
	if err != nil {
		log.Fatalln("err: ", err)
	}

	var finalSet []string
	for _, role := range nodeRoles[myKey] {
		finalSet = append(finalSet, role)
	}

	for _, role := range c.Args().Strings() {
		if !StrContains(finalSet, role) {
			finalSet = append(finalSet, role)
		}
	}

	roleActualSet(finalSet, c)
}

func allNodeRoles() (map[string][]string, error) {
	roleMap := make(map[string][]string)
	client, _ := api.NewClient(api.DefaultConfig())
	catalog := client.Catalog()
	cascadeServices, _, err := catalog.Service("cascade", "", nil)
	if err != nil {
		return nil, err
	}

	for _, service := range cascadeServices {
		roleMap[ makeKey(service.Node, service.Address) ] = service.ServiceTags
	}
	return roleMap, nil
}

func selfKey() (string, error) {
	client, _ := api.NewClient(api.DefaultConfig())
	agent := client.Agent()

	self, err := agent.Self()

	if err != nil {
		return "", err
	}

	return makeKey(self["Config"]["NodeName"].(string), self["Config"]["AdvertiseAddr"].(string)), nil
}

func makeKey(hostName string, hostAddr string) (string) {
	return fmt.Sprintf("%s (%s)", hostName, hostAddr)
}

func printRole(key string, roles []string) {
	fmt.Println(key + ":")
	for _, role := range roles {
		fmt.Println("  -", role)
	}
}

// StrContains checks if a list contains a string
func StrContains(l []string, s string) bool {
	for _, v := range l {
		if v == s {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/holizz/greyhound"
	"log"
	"net/http"
	"os"
	"time"
)

type stringslice []string

func (s *stringslice) String() string {
	return "a"
}

func (s *stringslice) Set(in string) (err error) {
	*s = append(*s, in)
	return
}

func main() {
	ignore := stringslice([]string{})
	port := flag.Int("p", 3000, "port number to listen on")
	dir := flag.String("d", ".", "directory to serve")
	timeout := flag.Duration("t", time.Second * 5, "timeout in milliseconds")
	flag.Var(&ignore, "i", "ignore errors matching this string")
	flag.Parse()

	if len(flag.Args()) > 0 {
		flag.Usage()
		os.Exit(0)
	}

	phpHandler, err := greyhound.NewPhpHandler(*dir, *timeout, ignore)
	defer phpHandler.Close()
	if err != nil {
		log.Fatalln(err)
	}

	fallbackHandler := greyhound.NewFallbackHandler(*dir, ".php", phpHandler)

	http.Handle("/", fallbackHandler)

	fmt.Printf("Listening on :%d\n", *port)
	log.Fatalln(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil))
}
<commit_msg>Point out that multiple -i options are allowed<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/holizz/greyhound"
	"log"
	"net/http"
	"os"
	"time"
)

type stringslice []string

func (s *stringslice) String() string {
	return "a"
}

func (s *stringslice) Set(in string) (err error) {
	*s = append(*s, in)
	return
}

func main() {
	ignore := stringslice([]string{})
	port := flag.Int("p", 3000, "port number to listen on")
	dir := flag.String("d", ".", "directory to serve")
	timeout := flag.Duration("t", time.Second * 5, "timeout in milliseconds")
	flag.Var(&ignore, "i", "ignore errors matching this string (multiple allowed)")
	flag.Parse()

	if len(flag.Args()) > 0 {
		flag.Usage()
		os.Exit(0)
	}

	phpHandler, err := greyhound.NewPhpHandler(*dir, *timeout, ignore)
	defer phpHandler.Close()
	if err != nil {
		log.Fatalln(err)
	}

	fallbackHandler := greyhound.NewFallbackHandler(*dir, ".php", phpHandler)

	http.Handle("/", fallbackHandler)

	fmt.Printf("Listening on :%d\n", *port)
	log.Fatalln(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil))
}
<|endoftext|>


<commit_before>package dnsserver

import (
	"context"
	"crypto/tls"
	"errors"
	"fmt"
	"net"

	"github.com/coredns/caddy"
	"github.com/coredns/coredns/pb"
	"github.com/coredns/coredns/plugin/pkg/reuseport"
	"github.com/coredns/coredns/plugin/pkg/transport"

	"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc"
	"github.com/miekg/dns"
	"github.com/opentracing/opentracing-go"
	"google.golang.org/grpc"
	"google.golang.org/grpc/peer"
)

// ServergRPC represents an instance of a DNS-over-gRPC server.
type ServergRPC struct {
	*Server
	grpcServer *grpc.Server
	listenAddr net.Addr
	tlsConfig  *tls.Config
}

// NewServergRPC returns a new CoreDNS GRPC server and compiles all plugin in to it.
func NewServergRPC(addr string, group []*Config) (*ServergRPC, error) {
	s, err := NewServer(addr, group)
	if err != nil {
		return nil, err
	}
	// The *tls* plugin must make sure that multiple conflicting
	// TLS configuration returns an error: it can only be specified once.
	var tlsConfig *tls.Config
	for _, conf := range s.zones {
		// Should we error if some configs *don't* have TLS?
		tlsConfig = conf.TLSConfig
	}

	return &ServergRPC{Server: s, tlsConfig: tlsConfig}, nil
}

// Compile-time check to ensure Server implements the caddy.GracefulServer interface
var _ caddy.GracefulServer = &Server{}

// Serve implements caddy.TCPServer interface.
func (s *ServergRPC) Serve(l net.Listener) error {
	s.m.Lock()
	s.listenAddr = l.Addr()
	s.m.Unlock()

	if s.Tracer() != nil {
		onlyIfParent := func(parentSpanCtx opentracing.SpanContext, method string, req, resp interface{}) bool {
			return parentSpanCtx != nil
		}
		intercept := otgrpc.OpenTracingServerInterceptor(s.Tracer(), otgrpc.IncludingSpans(onlyIfParent))
		s.grpcServer = grpc.NewServer(grpc.UnaryInterceptor(intercept))
	} else {
		s.grpcServer = grpc.NewServer()
	}

	pb.RegisterDnsServiceServer(s.grpcServer, s)

	if s.tlsConfig != nil {
		l = tls.NewListener(l, s.tlsConfig)
	}
	return s.grpcServer.Serve(l)
}

// ServePacket implements caddy.UDPServer interface.
func (s *ServergRPC) ServePacket(p net.PacketConn) error { return nil }

// Listen implements caddy.TCPServer interface.
func (s *ServergRPC) Listen() (net.Listener, error) {

	l, err := reuseport.Listen("tcp", s.Addr[len(transport.GRPC+"://"):])
	if err != nil {
		return nil, err
	}
	return l, nil
}

// ListenPacket implements caddy.UDPServer interface.
func (s *ServergRPC) ListenPacket() (net.PacketConn, error) { return nil, nil }

// OnStartupComplete lists the sites served by this server
// and any relevant information, assuming Quiet is false.
func (s *ServergRPC) OnStartupComplete() {
	if Quiet {
		return
	}

	out := startUpZones(transport.GRPC+"://", s.Addr, s.zones)
	if out != "" {
		fmt.Print(out)
	}
}

// Stop stops the server. It blocks until the server is
// totally stopped.
func (s *ServergRPC) Stop() (err error) {
	s.m.Lock()
	defer s.m.Unlock()
	if s.grpcServer != nil {
		s.grpcServer.GracefulStop()
	}
	return
}

// Query is the main entry-point into the gRPC server. From here we call ServeDNS like
// any normal server. We use a custom responseWriter to pick up the bytes we need to write
// back to the client as a protobuf.
func (s *ServergRPC) Query(ctx context.Context, in *pb.DnsPacket) (*pb.DnsPacket, error) {
	msg := new(dns.Msg)
	err := msg.Unpack(in.Msg)
	if err != nil {
		return nil, err
	}

	p, ok := peer.FromContext(ctx)
	if !ok {
		return nil, errors.New("no peer in gRPC context")
	}

	a, ok := p.Addr.(*net.TCPAddr)
	if !ok {
		return nil, fmt.Errorf("no TCP peer in gRPC context: %v", p.Addr)
	}

	w := &gRPCresponse{localAddr: s.listenAddr, remoteAddr: a, Msg: msg}

	dnsCtx := context.WithValue(ctx, Key{}, s.Server)
	dnsCtx = context.WithValue(dnsCtx, LoopKey{}, 0)
	s.ServeDNS(dnsCtx, w, msg)

	packed, err := w.Msg.Pack()
	if err != nil {
		return nil, err
	}

	return &pb.DnsPacket{Msg: packed}, nil
}

// Shutdown stops the server (non gracefully).
func (s *ServergRPC) Shutdown() error {
	if s.grpcServer != nil {
		s.grpcServer.Stop()
	}
	return nil
}

type gRPCresponse struct {
	localAddr  net.Addr
	remoteAddr net.Addr
	Msg        *dns.Msg
}

// Write is the hack that makes this work. It does not actually write the message
// but returns the bytes we need to write in r. We can then pick this up in Query
// and write a proper protobuf back to the client.
func (r *gRPCresponse) Write(b []byte) (int, error) {
	r.Msg = new(dns.Msg)
	return len(b), r.Msg.Unpack(b)
}

// These methods implement the dns.ResponseWriter interface from Go DNS.
func (r *gRPCresponse) Close() error              { return nil }
func (r *gRPCresponse) TsigStatus() error         { return nil }
func (r *gRPCresponse) TsigTimersOnly(b bool)     {}
func (r *gRPCresponse) Hijack()                   {}
func (r *gRPCresponse) LocalAddr() net.Addr       { return r.localAddr }
func (r *gRPCresponse) RemoteAddr() net.Addr      { return r.remoteAddr }
func (r *gRPCresponse) WriteMsg(m *dns.Msg) error { r.Msg = m; return nil }
<commit_msg>Enable HTTP/2 in grpc service (#4842)<commit_after>package dnsserver

import (
	"context"
	"crypto/tls"
	"errors"
	"fmt"
	"net"

	"github.com/coredns/caddy"
	"github.com/coredns/coredns/pb"
	"github.com/coredns/coredns/plugin/pkg/reuseport"
	"github.com/coredns/coredns/plugin/pkg/transport"

	"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc"
	"github.com/miekg/dns"
	"github.com/opentracing/opentracing-go"
	"google.golang.org/grpc"
	"google.golang.org/grpc/peer"
)

// ServergRPC represents an instance of a DNS-over-gRPC server.
type ServergRPC struct {
	*Server
	grpcServer *grpc.Server
	listenAddr net.Addr
	tlsConfig  *tls.Config
}

// NewServergRPC returns a new CoreDNS GRPC server and compiles all plugin in to it.
func NewServergRPC(addr string, group []*Config) (*ServergRPC, error) {
	s, err := NewServer(addr, group)
	if err != nil {
		return nil, err
	}
	// The *tls* plugin must make sure that multiple conflicting
	// TLS configuration returns an error: it can only be specified once.
	var tlsConfig *tls.Config
	for _, conf := range s.zones {
		// Should we error if some configs *don't* have TLS?
		tlsConfig = conf.TLSConfig
	}
	// http/2 is required when using gRPC. We need to specify it in next protos
	// or the upgrade won't happen.
	if tlsConfig != nil {
		tlsConfig.NextProtos = []string{"h2"}
	}

	return &ServergRPC{Server: s, tlsConfig: tlsConfig}, nil
}

// Compile-time check to ensure Server implements the caddy.GracefulServer interface
var _ caddy.GracefulServer = &Server{}

// Serve implements caddy.TCPServer interface.
func (s *ServergRPC) Serve(l net.Listener) error {
	s.m.Lock()
	s.listenAddr = l.Addr()
	s.m.Unlock()

	if s.Tracer() != nil {
		onlyIfParent := func(parentSpanCtx opentracing.SpanContext, method string, req, resp interface{}) bool {
			return parentSpanCtx != nil
		}
		intercept := otgrpc.OpenTracingServerInterceptor(s.Tracer(), otgrpc.IncludingSpans(onlyIfParent))
		s.grpcServer = grpc.NewServer(grpc.UnaryInterceptor(intercept))
	} else {
		s.grpcServer = grpc.NewServer()
	}

	pb.RegisterDnsServiceServer(s.grpcServer, s)

	if s.tlsConfig != nil {
		l = tls.NewListener(l, s.tlsConfig)
	}
	return s.grpcServer.Serve(l)
}

// ServePacket implements caddy.UDPServer interface.
func (s *ServergRPC) ServePacket(p net.PacketConn) error { return nil }

// Listen implements caddy.TCPServer interface.
func (s *ServergRPC) Listen() (net.Listener, error) {

	l, err := reuseport.Listen("tcp", s.Addr[len(transport.GRPC+"://"):])
	if err != nil {
		return nil, err
	}
	return l, nil
}

// ListenPacket implements caddy.UDPServer interface.
func (s *ServergRPC) ListenPacket() (net.PacketConn, error) { return nil, nil }

// OnStartupComplete lists the sites served by this server
// and any relevant information, assuming Quiet is false.
func (s *ServergRPC) OnStartupComplete() {
	if Quiet {
		return
	}

	out := startUpZones(transport.GRPC+"://", s.Addr, s.zones)
	if out != "" {
		fmt.Print(out)
	}
}

// Stop stops the server. It blocks until the server is
// totally stopped.
func (s *ServergRPC) Stop() (err error) {
	s.m.Lock()
	defer s.m.Unlock()
	if s.grpcServer != nil {
		s.grpcServer.GracefulStop()
	}
	return
}

// Query is the main entry-point into the gRPC server. From here we call ServeDNS like
// any normal server. We use a custom responseWriter to pick up the bytes we need to write
// back to the client as a protobuf.
func (s *ServergRPC) Query(ctx context.Context, in *pb.DnsPacket) (*pb.DnsPacket, error) {
	msg := new(dns.Msg)
	err := msg.Unpack(in.Msg)
	if err != nil {
		return nil, err
	}

	p, ok := peer.FromContext(ctx)
	if !ok {
		return nil, errors.New("no peer in gRPC context")
	}

	a, ok := p.Addr.(*net.TCPAddr)
	if !ok {
		return nil, fmt.Errorf("no TCP peer in gRPC context: %v", p.Addr)
	}

	w := &gRPCresponse{localAddr: s.listenAddr, remoteAddr: a, Msg: msg}

	dnsCtx := context.WithValue(ctx, Key{}, s.Server)
	dnsCtx = context.WithValue(dnsCtx, LoopKey{}, 0)
	s.ServeDNS(dnsCtx, w, msg)

	packed, err := w.Msg.Pack()
	if err != nil {
		return nil, err
	}

	return &pb.DnsPacket{Msg: packed}, nil
}

// Shutdown stops the server (non gracefully).
func (s *ServergRPC) Shutdown() error {
	if s.grpcServer != nil {
		s.grpcServer.Stop()
	}
	return nil
}

type gRPCresponse struct {
	localAddr  net.Addr
	remoteAddr net.Addr
	Msg        *dns.Msg
}

// Write is the hack that makes this work. It does not actually write the message
// but returns the bytes we need to write in r. We can then pick this up in Query
// and write a proper protobuf back to the client.
func (r *gRPCresponse) Write(b []byte) (int, error) {
	r.Msg = new(dns.Msg)
	return len(b), r.Msg.Unpack(b)
}

// These methods implement the dns.ResponseWriter interface from Go DNS.
func (r *gRPCresponse) Close() error              { return nil }
func (r *gRPCresponse) TsigStatus() error         { return nil }
func (r *gRPCresponse) TsigTimersOnly(b bool)     {}
func (r *gRPCresponse) Hijack()                   {}
func (r *gRPCresponse) LocalAddr() net.Addr       { return r.localAddr }
func (r *gRPCresponse) RemoteAddr() net.Addr      { return r.remoteAddr }
func (r *gRPCresponse) WriteMsg(m *dns.Msg) error { r.Msg = m; return nil }
<|endoftext|>


<commit_before>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package attrs bundles some common interface attributes and provides
// helpers to generate the appropriate OpenConfig and ATETopology.
//
// The use of this package in new tests is discouraged.  Legacy tests using this package
// will be migrated to use testbed topology helpers.
package attrs

import (
	"fmt"

	"github.com/openconfig/featureprofiles/internal/deviations"
	"github.com/openconfig/ondatra"
	oc "github.com/openconfig/ondatra/telemetry"
	"github.com/openconfig/ygot/ygot"
)

// Attributes bundles some common attributes for devices and/or interfaces.
// It provides helpers to generate appropriate configuration for OpenConfig
// and for an ATETopology.  All fields are optional; only those that are
// non-empty will be set when configuring an interface.
type Attributes struct {
	IPv4    string
	IPv6    string
	MAC     string
	Name    string // Interface name, only applied to ATE ports.
	Desc    string // Description, only applied to DUT interfaces.
	IPv4Len uint8  // Prefix length for IPv4.
	IPv6Len uint8  // Prefix length for IPv6.
	MTU     uint16
}

// IPv4CIDR constructs the IPv4 CIDR notation with the given prefix
// length, e.g. "192.0.2.1/30".
func (a *Attributes) IPv4CIDR() string {
	return fmt.Sprintf("%s/%d", a.IPv4, a.IPv4Len)
}

// IPv6CIDR constructs the IPv6 CIDR notation with the given prefix
// length, e.g. "2001:db8::1/126".
func (a *Attributes) IPv6CIDR() string {
	return fmt.Sprintf("%s/%d", a.IPv6, a.IPv6Len)
}

// ConfigInterface configures an OpenConfig interface with these attributes.
func (a *Attributes) ConfigInterface(intf *oc.Interface) *oc.Interface {
	if a.Desc != "" {
		intf.Description = ygot.String(a.Desc)
	}
	intf.Type = oc.IETFInterfaces_InterfaceType_ethernetCsmacd
	if *deviations.InterfaceEnabled {
		intf.Enabled = ygot.Bool(true)
	}
	if a.MTU > 0 && !*deviations.OmitL2MTU {
		intf.Mtu = ygot.Uint16(a.MTU + 14)
	}
	e := intf.GetOrCreateEthernet()
	if a.MAC != "" {
		e.MacAddress = ygot.String(a.MAC)
	}

	s := intf.GetOrCreateSubinterface(0)
	if a.IPv4 != "" {
		s4 := s.GetOrCreateIpv4()
		if *deviations.InterfaceEnabled {
			s4.Enabled = ygot.Bool(true)
		}
		if a.MTU > 0 {
			s4.Mtu = ygot.Uint16(a.MTU)
		}
		a4 := s4.GetOrCreateAddress(a.IPv4)
		if a.IPv4Len > 0 {
			a4.PrefixLength = ygot.Uint8(a.IPv4Len)
		}
	}

	if a.IPv6 != "" {
		s6 := s.GetOrCreateIpv6()
		if a.MTU > 0 {
			s6.Mtu = ygot.Uint32(uint32(a.MTU))
		}
		if *deviations.InterfaceEnabled {
			s6.Enabled = ygot.Bool(true)
		}
		a6 := s6.GetOrCreateAddress(a.IPv6)
		if a.IPv6Len > 0 {
			a6.PrefixLength = ygot.Uint8(a.IPv6Len)
		}
	}
	return intf
}

// NewInterface returns a new *oc.Interface configured with these attributes
func (a *Attributes) NewInterface(name string) *oc.Interface {
	return a.ConfigInterface(&oc.Interface{Name: ygot.String(name)})
}

// AddToATE adds a new interface to an ATETopology with these attributes.
func (a *Attributes) AddToATE(top *ondatra.ATETopology, ap *ondatra.Port, peer *Attributes) *ondatra.Interface {
	i := top.AddInterface(a.Name).WithPort(ap)
	if a.MTU > 0 {
		i.Ethernet().WithMTU(a.MTU)
	}
	if a.IPv4 != "" {
		i.IPv4().
			WithAddress(a.IPv4CIDR()).
			WithDefaultGateway(peer.IPv4)
	}
	if a.IPv6 != "" {
		i.IPv6().
			WithAddress(a.IPv6CIDR()).
			WithDefaultGateway(peer.IPv6)
	}
	return i
}
<commit_msg>AddToOTG added (#605)<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package attrs bundles some common interface attributes and provides
// helpers to generate the appropriate OpenConfig and ATETopology.
//
// The use of this package in new tests is discouraged.  Legacy tests using this package
// will be migrated to use testbed topology helpers.
package attrs

import (
	"fmt"

	"github.com/open-traffic-generator/snappi/gosnappi"
	"github.com/openconfig/featureprofiles/internal/deviations"
	"github.com/openconfig/ondatra"
	oc "github.com/openconfig/ondatra/telemetry"
	"github.com/openconfig/ygot/ygot"
)

// Attributes bundles some common attributes for devices and/or interfaces.
// It provides helpers to generate appropriate configuration for OpenConfig
// and for an ATETopology.  All fields are optional; only those that are
// non-empty will be set when configuring an interface.
type Attributes struct {
	IPv4    string
	IPv6    string
	MAC     string
	Name    string // Interface name, only applied to ATE ports.
	Desc    string // Description, only applied to DUT interfaces.
	IPv4Len uint8  // Prefix length for IPv4.
	IPv6Len uint8  // Prefix length for IPv6.
	MTU     uint16
}

// IPv4CIDR constructs the IPv4 CIDR notation with the given prefix
// length, e.g. "192.0.2.1/30".
func (a *Attributes) IPv4CIDR() string {
	return fmt.Sprintf("%s/%d", a.IPv4, a.IPv4Len)
}

// IPv6CIDR constructs the IPv6 CIDR notation with the given prefix
// length, e.g. "2001:db8::1/126".
func (a *Attributes) IPv6CIDR() string {
	return fmt.Sprintf("%s/%d", a.IPv6, a.IPv6Len)
}

// ConfigInterface configures an OpenConfig interface with these attributes.
func (a *Attributes) ConfigInterface(intf *oc.Interface) *oc.Interface {
	if a.Desc != "" {
		intf.Description = ygot.String(a.Desc)
	}
	intf.Type = oc.IETFInterfaces_InterfaceType_ethernetCsmacd
	if *deviations.InterfaceEnabled {
		intf.Enabled = ygot.Bool(true)
	}
	if a.MTU > 0 && !*deviations.OmitL2MTU {
		intf.Mtu = ygot.Uint16(a.MTU + 14)
	}
	e := intf.GetOrCreateEthernet()
	if a.MAC != "" {
		e.MacAddress = ygot.String(a.MAC)
	}

	s := intf.GetOrCreateSubinterface(0)
	if a.IPv4 != "" {
		s4 := s.GetOrCreateIpv4()
		if *deviations.InterfaceEnabled {
			s4.Enabled = ygot.Bool(true)
		}
		if a.MTU > 0 {
			s4.Mtu = ygot.Uint16(a.MTU)
		}
		a4 := s4.GetOrCreateAddress(a.IPv4)
		if a.IPv4Len > 0 {
			a4.PrefixLength = ygot.Uint8(a.IPv4Len)
		}
	}

	if a.IPv6 != "" {
		s6 := s.GetOrCreateIpv6()
		if a.MTU > 0 {
			s6.Mtu = ygot.Uint32(uint32(a.MTU))
		}
		if *deviations.InterfaceEnabled {
			s6.Enabled = ygot.Bool(true)
		}
		a6 := s6.GetOrCreateAddress(a.IPv6)
		if a.IPv6Len > 0 {
			a6.PrefixLength = ygot.Uint8(a.IPv6Len)
		}
	}
	return intf
}

// NewInterface returns a new *oc.Interface configured with these attributes
func (a *Attributes) NewInterface(name string) *oc.Interface {
	return a.ConfigInterface(&oc.Interface{Name: ygot.String(name)})
}

// AddToATE adds a new interface to an ATETopology with these attributes.
func (a *Attributes) AddToATE(top *ondatra.ATETopology, ap *ondatra.Port, peer *Attributes) *ondatra.Interface {
	i := top.AddInterface(a.Name).WithPort(ap)
	if a.MTU > 0 {
		i.Ethernet().WithMTU(a.MTU)
	}
	if a.IPv4 != "" {
		i.IPv4().
			WithAddress(a.IPv4CIDR()).
			WithDefaultGateway(peer.IPv4)
	}
	if a.IPv6 != "" {
		i.IPv6().
			WithAddress(a.IPv6CIDR()).
			WithDefaultGateway(peer.IPv6)
	}
	return i
}

// AddToOTG adds basic elements to a gosnappi configuration
func (a *Attributes) AddToOTG(top gosnappi.Config, ap *ondatra.Port, peer *Attributes) {
	top.Ports().Add().SetName(ap.ID())
	dev := top.Devices().Add().SetName(a.Name)
	eth := dev.Ethernets().Add().SetName(a.Name + ".Eth")
	eth.SetPortName(ap.ID()).SetMac(a.MAC)

	if a.MTU > 0 {
		eth.SetMtu(int32(a.MTU))
	}
	if a.IPv4 != "" {
		ip := eth.Ipv4Addresses().Add().SetName(dev.Name() + ".IPv4")
		ip.SetAddress(a.IPv4).SetGateway(peer.IPv4).SetPrefix(int32(a.IPv4Len))
	}
	if a.IPv6 != "" {
		ip := eth.Ipv4Addresses().Add().SetName(dev.Name() + ".IPv6")
		ip.SetAddress(a.IPv6).SetGateway(peer.IPv6).SetPrefix(int32(a.IPv6Len))
	}
}
<|endoftext|>


<commit_before>// local build script file, similar to a makefile or collection of bash scripts in other projects

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"os"
	"os/exec"
	"strings"

	"github.com/urfave/cli/v2"
)

var packages = []string{"cli", "altsrc"}

func main() {
	app := cli.NewApp()

	app.Name = "builder"
	app.Usage = "Generates a new urfave/cli build!"

	app.Commands = cli.Commands{
		{
			Name:   "vet",
			Action: VetActionFunc,
		},
		{
			Name:   "test",
			Action: TestActionFunc,
		},
		{
			Name:   "gfmrun",
			Action: GfmrunActionFunc,
		},
		{
			Name:   "toc",
			Action: TocActionFunc,
		},
		{
			Name:   "check-binary-size",
			Action: checkBinarySizeActionFunc,
		},
	}

	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}

func runCmd(arg string, args ...string) error {
	cmd := exec.Command(arg, args...)

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}

func VetActionFunc(_ *cli.Context) error {
	return runCmd("go", "vet")
}

func TestActionFunc(c *cli.Context) error {
	for _, pkg := range packages {
		var packageName string

		if pkg == "cli" {
			packageName = "github.com/urfave/cli/v2"
		} else {
			packageName = fmt.Sprintf("github.com/urfave/cli/v2/%s", pkg)
		}

		coverProfile := fmt.Sprintf("--coverprofile=%s.coverprofile", pkg)

		err := runCmd("go", "test", "-v", coverProfile, packageName)
		if err != nil {
			return err
		}
	}

	return testCleanup()
}

func testCleanup() error {
	var out bytes.Buffer

	for _, pkg := range packages {
		file, err := os.Open(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}

		b, err := ioutil.ReadAll(file)
		if err != nil {
			return err
		}

		out.Write(b)
		err = file.Close()
		if err != nil {
			return err
		}

		err = os.Remove(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}
	}

	outFile, err := os.Create("coverage.txt")
	if err != nil {
		return err
	}

	_, err = out.WriteTo(outFile)
	if err != nil {
		return err
	}

	err = outFile.Close()
	if err != nil {
		return err
	}

	return nil
}

func GfmrunActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()

	var counter int
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "package main") {
			counter++
		}
	}

	err = file.Close()
	if err != nil {
		return err
	}

	err = scanner.Err()
	if err != nil {
		return err
	}

	return runCmd("gfmrun", "-c", fmt.Sprint(counter), "-s", filename)
}

func TocActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	err := runCmd("markdown-toc", "-i", filename)
	if err != nil {
		return err
	}

	err = runCmd("git", "diff", "--exit-code")
	if err != nil {
		return err
	}

	return nil
}

// checkBinarySizeActionFunc checks the size of an example binary to ensure that we are keeping size down
// this was originally inspired by https://github.com/urfave/cli/issues/1055, and followed up on as a part
// of https://github.com/urfave/cli/issues/1057
func checkBinarySizeActionFunc(c *cli.Context) (err error) {
	const (
		sourceFilePath       = "./internal/example/example.go"
		builtFilePath        = "./internal/example/built-example"
		desiredMinBinarySize = 4.7
		desiredMaxBinarySize = 5.0
		badNewsEmoji         = "🚨"
		goodNewsEmoji        = "✨"
		checksPassedEmoji    = "✅"
		mbStringFormatter    = "%.1fMB"
	)

	// build example binary
	err = runCmd("go", "build", "-o", builtFilePath, sourceFilePath)
	if err != nil {
		return err
	}

	// get file into
	fileInfo, err := os.Stat(builtFilePath)
	if err != nil {
		return err
	}

	// get human readable size, in MB with one decimal place.
	// example output is: 35.2MB. (note: this simply an example)
	// that output is much easier to reason about than the `35223432`
	// that you would see output without the rounding
	fileSize := fileInfo.Size()
	fileSizeInMB := float64(fileSize) / float64(1000000)
	roundedFileSize := math.Round(fileSizeInMB*10) / 10
	roundedFileSizeString := fmt.Sprintf(mbStringFormatter, roundedFileSize)

	// check against bounds
	isLessThanDesiredMin := roundedFileSize < desiredMinBinarySize
	isMoreThanDesiredMax := roundedFileSize > desiredMaxBinarySize
	desiredMinSizeString := fmt.Sprintf(mbStringFormatter, desiredMinBinarySize)
	desiredMaxSizeString := fmt.Sprintf(mbStringFormatter, desiredMaxBinarySize)

	// show guidance
	fmt.Println(fmt.Sprintf("\n%s is the current binary size", roundedFileSizeString))
	// show guidance for min size
	if isLessThanDesiredMin {
		fmt.Println(fmt.Sprintf("  %s %s is the target min size", goodNewsEmoji, desiredMinSizeString))
		fmt.Println("") // visual spacing
		fmt.Println("     The binary is smaller than the target min size, which is great news!")
		fmt.Println("     That means that whatever you've done is shrinking the binary size.")
		fmt.Println("     You'll want to go into ./internal/build/build.go and decrease")
		fmt.Println("     the desiredMinBinarySize, and also probably decrease the ")
		fmt.Println("     desiredMaxBinarySize by the same amount. That will ensure that")
		fmt.Println("     future PRs will enforce the newly shrunk binary sizes.")
		fmt.Println("") // visual spacing
		os.Exit(1)
	} else {
		fmt.Println(fmt.Sprintf("  %s %s is the target min size", checksPassedEmoji, desiredMinSizeString))
	}
	// show guidance for max size
	if isMoreThanDesiredMax {
		fmt.Println(fmt.Sprintf("  %s %s is the target max size", badNewsEmoji, desiredMaxSizeString))
		fmt.Println("") // visual spacing
		os.Exit(1)
	} else {
		fmt.Println(fmt.Sprintf("  %s %s is the target max size", checksPassedEmoji, desiredMaxSizeString))
	}

	return nil
}
<commit_msg>max guidance<commit_after>// local build script file, similar to a makefile or collection of bash scripts in other projects

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"os"
	"os/exec"
	"strings"

	"github.com/urfave/cli/v2"
)

var packages = []string{"cli", "altsrc"}

func main() {
	app := cli.NewApp()

	app.Name = "builder"
	app.Usage = "Generates a new urfave/cli build!"

	app.Commands = cli.Commands{
		{
			Name:   "vet",
			Action: VetActionFunc,
		},
		{
			Name:   "test",
			Action: TestActionFunc,
		},
		{
			Name:   "gfmrun",
			Action: GfmrunActionFunc,
		},
		{
			Name:   "toc",
			Action: TocActionFunc,
		},
		{
			Name:   "check-binary-size",
			Action: checkBinarySizeActionFunc,
		},
	}

	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}

func runCmd(arg string, args ...string) error {
	cmd := exec.Command(arg, args...)

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}

func VetActionFunc(_ *cli.Context) error {
	return runCmd("go", "vet")
}

func TestActionFunc(c *cli.Context) error {
	for _, pkg := range packages {
		var packageName string

		if pkg == "cli" {
			packageName = "github.com/urfave/cli/v2"
		} else {
			packageName = fmt.Sprintf("github.com/urfave/cli/v2/%s", pkg)
		}

		coverProfile := fmt.Sprintf("--coverprofile=%s.coverprofile", pkg)

		err := runCmd("go", "test", "-v", coverProfile, packageName)
		if err != nil {
			return err
		}
	}

	return testCleanup()
}

func testCleanup() error {
	var out bytes.Buffer

	for _, pkg := range packages {
		file, err := os.Open(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}

		b, err := ioutil.ReadAll(file)
		if err != nil {
			return err
		}

		out.Write(b)
		err = file.Close()
		if err != nil {
			return err
		}

		err = os.Remove(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}
	}

	outFile, err := os.Create("coverage.txt")
	if err != nil {
		return err
	}

	_, err = out.WriteTo(outFile)
	if err != nil {
		return err
	}

	err = outFile.Close()
	if err != nil {
		return err
	}

	return nil
}

func GfmrunActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()

	var counter int
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "package main") {
			counter++
		}
	}

	err = file.Close()
	if err != nil {
		return err
	}

	err = scanner.Err()
	if err != nil {
		return err
	}

	return runCmd("gfmrun", "-c", fmt.Sprint(counter), "-s", filename)
}

func TocActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	err := runCmd("markdown-toc", "-i", filename)
	if err != nil {
		return err
	}

	err = runCmd("git", "diff", "--exit-code")
	if err != nil {
		return err
	}

	return nil
}

// checkBinarySizeActionFunc checks the size of an example binary to ensure that we are keeping size down
// this was originally inspired by https://github.com/urfave/cli/issues/1055, and followed up on as a part
// of https://github.com/urfave/cli/issues/1057
func checkBinarySizeActionFunc(c *cli.Context) (err error) {
	const (
		sourceFilePath       = "./internal/example/example.go"
		builtFilePath        = "./internal/example/built-example"
		desiredMinBinarySize = 4.5
		desiredMaxBinarySize = 5.0
		badNewsEmoji         = "🚨"
		goodNewsEmoji        = "✨"
		checksPassedEmoji    = "✅"
		mbStringFormatter    = "%.1fMB"
	)

	// build example binary
	err = runCmd("go", "build", "-o", builtFilePath, sourceFilePath)
	if err != nil {
		return err
	}

	// get file into
	fileInfo, err := os.Stat(builtFilePath)
	if err != nil {
		return err
	}

	// get human readable size, in MB with one decimal place.
	// example output is: 35.2MB. (note: this simply an example)
	// that output is much easier to reason about than the `35223432`
	// that you would see output without the rounding
	fileSize := fileInfo.Size()
	fileSizeInMB := float64(fileSize) / float64(1000000)
	roundedFileSize := math.Round(fileSizeInMB*10) / 10
	roundedFileSizeString := fmt.Sprintf(mbStringFormatter, roundedFileSize)

	// check against bounds
	isLessThanDesiredMin := roundedFileSize < desiredMinBinarySize
	isMoreThanDesiredMax := roundedFileSize > desiredMaxBinarySize
	desiredMinSizeString := fmt.Sprintf(mbStringFormatter, desiredMinBinarySize)
	desiredMaxSizeString := fmt.Sprintf(mbStringFormatter, desiredMaxBinarySize)

	// show guidance
	fmt.Println(fmt.Sprintf("\n%s is the current binary size", roundedFileSizeString))
	// show guidance for min size
	if isLessThanDesiredMin {
		fmt.Println(fmt.Sprintf("  %s %s is the target min size", goodNewsEmoji, desiredMinSizeString))
		fmt.Println("") // visual spacing
		fmt.Println("     The binary is smaller than the target min size, which is great news!")
		fmt.Println("     That means that your changes are shrinking the binary size.")
		fmt.Println("     You'll want to go into ./internal/build/build.go and decrease")
		fmt.Println("     the desiredMinBinarySize, and also probably decrease the ")
		fmt.Println("     desiredMaxBinarySize by the same amount. That will ensure that")
		fmt.Println("     future PRs will enforce the newly shrunk binary sizes.")
		fmt.Println("") // visual spacing
		os.Exit(1)
	} else {
		fmt.Println(fmt.Sprintf("  %s %s is the target min size", checksPassedEmoji, desiredMinSizeString))
	}
	// show guidance for max size
	if isMoreThanDesiredMax {
		fmt.Println(fmt.Sprintf("  %s %s is the target max size", badNewsEmoji, desiredMaxSizeString))
		fmt.Println("") // visual spacing
		fmt.Println("     The binary is larger than the target max size.")
		fmt.Println("     That means that your changes are increasing the binary size.")
		fmt.Println("     The first thing you'll want to do is ask your yourself")
		fmt.Println("     Is this change worth increasing the binary size?")
		fmt.Println("     Larger binary sizes for this package can dissuade its use.")
		fmt.Println("     If this change is worth the increase, then we can up the")
		fmt.Println("     desired max binary size. To do that you'll want to go into")
		fmt.Println("     ./internal/build/build.go and increase the desiredMaxBinarySize,")
		fmt.Println("     and increase the desiredMinBinarySize by the same amount.")
		fmt.Println("") // visual spacing
		os.Exit(1)
	} else {
		fmt.Println(fmt.Sprintf("  %s %s is the target max size", checksPassedEmoji, desiredMaxSizeString))
	}

	return nil
}
<|endoftext|>


<commit_before>package internal

import (
	"bytes"
	"fmt"
	"path/filepath"
	"runtime"
)

type CallerError struct {
	PC    []uintptr
	Cause error
}

func NewCallerError(cause error) error {
	pc := make([]uintptr, 20)
	n := runtime.Callers(2, pc)
	return CallerError{
		PC:    pc[:n],
		Cause: cause,
	}
}

func (err CallerError) Error() string {
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "%v\n", err.Cause)
	for _, pc := range err.PC {
		f := runtime.FuncForPC(pc-1)
		name := f.Name()
		file, line := f.FileLine(pc-1)
		entry := f.Entry()
		fmt.Fprintf(&buf, "\t%s %s:%d (+%#x)\n", name, filepath.Base(file), line, pc - entry)
	}
	return buf.String()
}
<commit_msg>Run go fmt.<commit_after>package internal

import (
	"bytes"
	"fmt"
	"path/filepath"
	"runtime"
)

type CallerError struct {
	PC    []uintptr
	Cause error
}

func NewCallerError(cause error) error {
	pc := make([]uintptr, 20)
	n := runtime.Callers(2, pc)
	return CallerError{
		PC:    pc[:n],
		Cause: cause,
	}
}

func (err CallerError) Error() string {
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "%v\n", err.Cause)
	for _, pc := range err.PC {
		f := runtime.FuncForPC(pc - 1)
		name := f.Name()
		file, line := f.FileLine(pc - 1)
		entry := f.Entry()
		fmt.Fprintf(&buf, "\t%s %s:%d (+%#x)\n", name, filepath.Base(file), line, pc-entry)
	}
	return buf.String()
}
<|endoftext|>


<commit_before>// Copyright 2017 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package clock manages game timers.
package clock

import (
	"sync"
	"time"
)

var (
	lastNow int64

	// lastSystemTime is the last system time in the previous Update.
	// lastSystemTime indicates the logical time in the game, so this can be bigger than the curren time.
	lastSystemTime int64

	currentFPS  float64
	currentTPS  float64
	lastUpdated int64
	fpsCount    = 0
	tpsCount    = 0

	m sync.Mutex
)

func init() {
	n := now()
	lastNow = n
	lastSystemTime = n
	lastUpdated = n
}

func CurrentFPS() float64 {
	m.Lock()
	v := currentFPS
	m.Unlock()
	return v
}

func CurrentTPS() float64 {
	m.Lock()
	v := currentTPS
	m.Unlock()
	return v
}

func min(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}

func calcCountFromTPS(tps int64, now int64) int {
	if tps == 0 {
		return 0
	}
	if tps < 0 {
		panic("clock: tps must >= 0")
	}

	diff := now - lastSystemTime
	if diff < 0 {
		return 0
	}

	count := 0
	syncWithSystemClock := false

	// When TPS is big (e.g. 300), the time gap can be small and diff might always exceeds the gap.
	// To avoid this, prepare another gap assuming TPS was 60 and use the bigger one (#1443).
	tooBigGap := int64(time.Second) * 5 / min(int64(tps), 60)
	if diff > tooBigGap {
		// The previous time is too old.
		// Let's force to sync the game time with the system clock.
		syncWithSystemClock = true
	} else {
		count = int(diff * tps / int64(time.Second))
	}

	// Stabilize the count.
	// Without this adjustment, count can be unstable like 0, 2, 0, 2, ...
	if count == 0 && (int64(time.Second)/tps/2) < diff {
		count = 1
	}
	if count == 2 && (int64(time.Second)/tps*3/2) > diff {
		count = 1
	}

	if syncWithSystemClock {
		lastSystemTime = now
	} else {
		lastSystemTime += int64(count) * int64(time.Second) / tps
	}

	return count
}

func updateFPSAndTPS(now int64, count int) {
	fpsCount++
	tpsCount += count
	if now < lastUpdated {
		panic("clock: lastUpdated must be older than now")
	}
	if time.Second > time.Duration(now-lastUpdated) {
		return
	}
	currentFPS = float64(fpsCount) * float64(time.Second) / float64(now-lastUpdated)
	currentTPS = float64(tpsCount) * float64(time.Second) / float64(now-lastUpdated)
	lastUpdated = now
	fpsCount = 0
	tpsCount = 0
}

const UncappedTPS = -1

// Update updates the inner clock state and returns an integer value
// indicating how many times the game should update based on given tps.
// tps represents TPS (ticks per second).
// If tps is UncappedTPS, Update always returns 1.
// If tps <= 0 and not UncappedTPS, Update always returns 0.
//
// Update is expected to be called per frame.
func Update(tps int) int {
	m.Lock()
	defer m.Unlock()

	n := now()
	if lastNow > n {
		// This ensures that now() must be monotonic (#875).
		panic("clock: lastNow must be older than n")
	}
	lastNow = n

	c := 0
	if tps == UncappedTPS {
		c = 1
	} else if tps > 0 {
		c = calcCountFromTPS(int64(tps), n)
	}
	updateFPSAndTPS(n, c)

	return c
}
<commit_msg>clock: Add comment<commit_after>// Copyright 2017 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package clock manages game timers.
package clock

import (
	"sync"
	"time"
)

var (
	lastNow int64

	// lastSystemTime is the last system time in the previous Update.
	// lastSystemTime indicates the logical time in the game, so this can be bigger than the curren time.
	lastSystemTime int64

	currentFPS  float64
	currentTPS  float64
	lastUpdated int64
	fpsCount    = 0
	tpsCount    = 0

	m sync.Mutex
)

func init() {
	n := now()
	lastNow = n
	lastSystemTime = n
	lastUpdated = n
}

func CurrentFPS() float64 {
	m.Lock()
	v := currentFPS
	m.Unlock()
	return v
}

func CurrentTPS() float64 {
	m.Lock()
	v := currentTPS
	m.Unlock()
	return v
}

func min(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}

func calcCountFromTPS(tps int64, now int64) int {
	if tps == 0 {
		return 0
	}
	if tps < 0 {
		panic("clock: tps must >= 0")
	}

	diff := now - lastSystemTime
	if diff < 0 {
		return 0
	}

	count := 0
	syncWithSystemClock := false

	// When TPS is big (e.g. 300), the time gap can be small and diff might always exceeds the gap.
	// To avoid this, prepare another gap assuming TPS was 60 and use the bigger one (#1443).
	tooBigGap := int64(time.Second) * 5 / min(int64(tps), 60)
	if diff > tooBigGap {
		// The previous time is too old.
		// Let's force to sync the game time with the system clock.
		syncWithSystemClock = true
	} else {
		count = int(diff * tps / int64(time.Second))
	}

	// Stabilize the count.
	// Without this adjustment, count can be unstable like 0, 2, 0, 2, ...
	// TODO: Brush up this logic so that this will work with any FPS. Now this works only when FPS = TPS.
	if count == 0 && (int64(time.Second)/tps/2) < diff {
		count = 1
	}
	if count == 2 && (int64(time.Second)/tps*3/2) > diff {
		count = 1
	}

	if syncWithSystemClock {
		lastSystemTime = now
	} else {
		lastSystemTime += int64(count) * int64(time.Second) / tps
	}

	return count
}

func updateFPSAndTPS(now int64, count int) {
	fpsCount++
	tpsCount += count
	if now < lastUpdated {
		panic("clock: lastUpdated must be older than now")
	}
	if time.Second > time.Duration(now-lastUpdated) {
		return
	}
	currentFPS = float64(fpsCount) * float64(time.Second) / float64(now-lastUpdated)
	currentTPS = float64(tpsCount) * float64(time.Second) / float64(now-lastUpdated)
	lastUpdated = now
	fpsCount = 0
	tpsCount = 0
}

const UncappedTPS = -1

// Update updates the inner clock state and returns an integer value
// indicating how many times the game should update based on given tps.
// tps represents TPS (ticks per second).
// If tps is UncappedTPS, Update always returns 1.
// If tps <= 0 and not UncappedTPS, Update always returns 0.
//
// Update is expected to be called per frame.
func Update(tps int) int {
	m.Lock()
	defer m.Unlock()

	n := now()
	if lastNow > n {
		// This ensures that now() must be monotonic (#875).
		panic("clock: lastNow must be older than n")
	}
	lastNow = n

	c := 0
	if tps == UncappedTPS {
		c = 1
	} else if tps > 0 {
		c = calcCountFromTPS(int64(tps), n)
	}
	updateFPSAndTPS(n, c)

	return c
}
<|endoftext|>


<commit_before>// Copyright 2017 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package clock manages game timers.
package clock

import (
	"sync"
	"time"
)

var (
	lastNow int64

	// lastSystemTime is the last system time in the previous Update.
	// lastSystemTime indicates the logical time in the game, so this can be bigger than the curren time.
	lastSystemTime int64

	currentFPS  float64
	currentTPS  float64
	lastUpdated int64
	fpsCount    = 0
	tpsCount    = 0

	m sync.Mutex
)

func init() {
	n := now()
	lastNow = n
	lastSystemTime = n
	lastUpdated = n
}

func CurrentFPS() float64 {
	m.Lock()
	v := currentFPS
	m.Unlock()
	return v
}

func CurrentTPS() float64 {
	m.Lock()
	v := currentTPS
	m.Unlock()
	return v
}

func calcCountFromTPS(tps int64, now int64) int {
	if tps == 0 {
		return 0
	}
	if tps < 0 {
		panic("clock: tps must >= 0")
	}

	diff := now - lastSystemTime
	if diff < 0 {
		return 0
	}

	count := 0
	syncWithSystemClock := false

	if diff > int64(time.Second)*5/60 {
		// The previous time is too old.
		// Let's force to sync the game time with the system clock.
		syncWithSystemClock = true
	} else {
		count = int(diff * tps / int64(time.Second))
	}

	// Stabilize the count.
	// Without this adjustment, count can be unstable like 0, 2, 0, 2, ...
	if count == 0 && (int64(time.Second)/tps/2) < diff {
		count = 1
	}
	if count == 2 && (int64(time.Second)/tps*3/2) > diff {
		count = 1
	}

	if syncWithSystemClock {
		lastSystemTime = now
	} else {
		lastSystemTime += int64(count) * int64(time.Second) / tps
	}

	return count
}

func updateFPSAndTPS(now int64, count int) {
	fpsCount++
	tpsCount += count
	if now < lastUpdated {
		panic("clock: lastUpdated must be older than now")
	}
	if time.Second > time.Duration(now-lastUpdated) {
		return
	}
	currentFPS = float64(fpsCount) * float64(time.Second) / float64(now-lastUpdated)
	currentTPS = float64(tpsCount) * float64(time.Second) / float64(now-lastUpdated)
	lastUpdated = now
	fpsCount = 0
	tpsCount = 0
}

const UncappedTPS = -1

// Update updates the inner clock state and returns an integer value
// indicating how many times the game should update based on given tps.
// tps represents TPS (ticks per second).
// If tps is UncappedTPS, Update always returns 1.
// If tps <= 0 and not UncappedTPS, Update always returns 0.
//
// Update is expected to be called per frame.
func Update(tps int) int {
	m.Lock()
	defer m.Unlock()

	n := now()
	if lastNow > n {
		// This ensures that now() must be monotonic (#875).
		panic("clock: lastNow must be older than n")
	}
	lastNow = n

	c := 0
	if tps == UncappedTPS {
		c = 1
	} else if tps > 0 {
		c = calcCountFromTPS(int64(tps), n)
	}
	updateFPSAndTPS(n, c)

	return c
}
<commit_msg>clock: Bug fix: The game was frozen when TPS is <= 4<commit_after>// Copyright 2017 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package clock manages game timers.
package clock

import (
	"sync"
	"time"
)

var (
	lastNow int64

	// lastSystemTime is the last system time in the previous Update.
	// lastSystemTime indicates the logical time in the game, so this can be bigger than the curren time.
	lastSystemTime int64

	currentFPS  float64
	currentTPS  float64
	lastUpdated int64
	fpsCount    = 0
	tpsCount    = 0

	m sync.Mutex
)

func init() {
	n := now()
	lastNow = n
	lastSystemTime = n
	lastUpdated = n
}

func CurrentFPS() float64 {
	m.Lock()
	v := currentFPS
	m.Unlock()
	return v
}

func CurrentTPS() float64 {
	m.Lock()
	v := currentTPS
	m.Unlock()
	return v
}

func calcCountFromTPS(tps int64, now int64) int {
	if tps == 0 {
		return 0
	}
	if tps < 0 {
		panic("clock: tps must >= 0")
	}

	diff := now - lastSystemTime
	if diff < 0 {
		return 0
	}

	count := 0
	syncWithSystemClock := false

	if diff > int64(time.Second)*5/int64(tps) {
		// The previous time is too old.
		// Let's force to sync the game time with the system clock.
		syncWithSystemClock = true
	} else {
		count = int(diff * tps / int64(time.Second))
	}

	// Stabilize the count.
	// Without this adjustment, count can be unstable like 0, 2, 0, 2, ...
	if count == 0 && (int64(time.Second)/tps/2) < diff {
		count = 1
	}
	if count == 2 && (int64(time.Second)/tps*3/2) > diff {
		count = 1
	}

	if syncWithSystemClock {
		lastSystemTime = now
	} else {
		lastSystemTime += int64(count) * int64(time.Second) / tps
	}

	return count
}

func updateFPSAndTPS(now int64, count int) {
	fpsCount++
	tpsCount += count
	if now < lastUpdated {
		panic("clock: lastUpdated must be older than now")
	}
	if time.Second > time.Duration(now-lastUpdated) {
		return
	}
	currentFPS = float64(fpsCount) * float64(time.Second) / float64(now-lastUpdated)
	currentTPS = float64(tpsCount) * float64(time.Second) / float64(now-lastUpdated)
	lastUpdated = now
	fpsCount = 0
	tpsCount = 0
}

const UncappedTPS = -1

// Update updates the inner clock state and returns an integer value
// indicating how many times the game should update based on given tps.
// tps represents TPS (ticks per second).
// If tps is UncappedTPS, Update always returns 1.
// If tps <= 0 and not UncappedTPS, Update always returns 0.
//
// Update is expected to be called per frame.
func Update(tps int) int {
	m.Lock()
	defer m.Unlock()

	n := now()
	if lastNow > n {
		// This ensures that now() must be monotonic (#875).
		panic("clock: lastNow must be older than n")
	}
	lastNow = n

	c := 0
	if tps == UncappedTPS {
		c = 1
	} else if tps > 0 {
		c = calcCountFromTPS(int64(tps), n)
	}
	updateFPSAndTPS(n, c)

	return c
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing log files
// and extracting new log lines to be passed into the virtual machines.
package tailer

// For regular files, mtail gets notified on modifications (i.e. appends) to
// log files that are being watched, in order to read the new lines. Log files
// can also be rotated, so mtail is also notified of creates in the log file
// directory.

import (
	"context"
	"expvar"
	"html/template"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"

	"github.com/google/mtail/internal/logline"
	"github.com/google/mtail/internal/watcher"
)

var (
	// logCount records the number of logs that are being tailed
	logCount = expvar.NewInt("log_count")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	lines chan<- *logline.LogLine // Logfile lines being emitted.
	w     watcher.Watcher
	ctx   context.Context

	handlesMu sync.RWMutex     // protects `handles'
	handles   map[string]*File // File handles for each pathname.

	globPatternsMu sync.RWMutex        // protects `globPatterns'
	globPatterns   map[string]struct{} // glob patterns to match newly created files in dir paths against

	runDone chan struct{} // Signals termination of the run goroutine.

	eventsHandle int // record the handle with which to add new log files to the watcher

	oneShot bool
}

// OneShot puts the tailer in one-shot mode.
func OneShot(t *Tailer) error {
	t.oneShot = true
	return nil
}

// Context sets the context of the tailer
func Context(ctx context.Context) func(*Tailer) error {
	return func(t *Tailer) error {
		t.ctx = ctx
		return nil
	}
}

// New creates a new Tailer.
func New(lines chan<- *logline.LogLine, w watcher.Watcher, options ...func(*Tailer) error) (*Tailer, error) {
	if lines == nil {
		return nil, errors.New("can't create tailer without lines channel")
	}
	if w == nil {
		return nil, errors.New("can't create tailer without W")
	}
	t := &Tailer{
		lines:        lines,
		w:            w,
		handles:      make(map[string]*File),
		globPatterns: make(map[string]struct{}),
		runDone:      make(chan struct{}),
	}
	if err := t.SetOption(options...); err != nil {
		return nil, err
	}
	handle, eventsChan := t.w.Events()
	t.eventsHandle = handle
	go t.run(eventsChan)
	return t, nil
}

// SetOption takes one or more option functions and applies them in order to Tailer.
func (t *Tailer) SetOption(options ...func(*Tailer) error) error {
	for _, option := range options {
		if err := option(t); err != nil {
			return err
		}
	}
	return nil
}

// setHandle sets a file handle under it's pathname
func (t *Tailer) setHandle(pathname string, f *File) error {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return errors.Wrapf(err, "Failed to lookup abspath of %q", pathname)
	}
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	t.handles[absPath] = f
	return nil
}

// handleForPath retrives a file handle for a pathname.
func (t *Tailer) handleForPath(pathname string) (*File, bool) {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		glog.V(2).Infof("Couldn't resolve path %q: %s", pathname, err)
		return nil, false
	}
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	fd, ok := t.handles[absPath]
	return fd, ok
}

func (t *Tailer) hasHandle(pathname string) bool {
	_, ok := t.handleForPath(pathname)
	return ok
}

// AddPattern adds a pattern to the list of patterns to filter filenames against.
func (t *Tailer) AddPattern(pattern string) error {
	absPath, err := filepath.Abs(pattern)
	if err != nil {
		glog.V(2).Infof("Couldn't canonicalize path %q: %s", pattern, err)
		return err
	}
	glog.V(2).Infof("AddPattern: %s", absPath)
	t.globPatternsMu.Lock()
	t.globPatterns[absPath] = struct{}{}
	t.globPatternsMu.Unlock()
	return nil
}

// TailPattern registers a pattern to be tailed.  If pattern is a plain
// file then it is watched for updates and opened.  If pattern is a glob, then
// all paths that match the glob are opened and watched, and the directories
// containing those matches, if any, are watched.
func (t *Tailer) TailPattern(pattern string) error {
	if err := t.AddPattern(pattern); err != nil {
		return err
	}
	// Add a watch on the containing directory, so we know when a rotation
	// occurs or something shows up that matches this pattern.
	if err := t.watchDirname(pattern); err != nil {
		return err
	}
	matches, err := filepath.Glob(pattern)
	if err != nil {
		return err
	}
	glog.V(1).Infof("glob matches: %v", matches)
	// Error if there are no matches, but if they show up later, they'll get picked up by the directory watch set above.
	if len(matches) == 0 {
		return errors.Errorf("No matches for pattern %q", pattern)
	}
	for _, pathname := range matches {
		err := t.TailPath(pathname)
		if err != nil {
			return errors.Wrapf(err, "attempting to tail %q", pathname)
		}
	}
	return nil
}

// TailPath registers a filesystem pathname to be tailed.
func (t *Tailer) TailPath(pathname string) error {
	if t.hasHandle(pathname) {
		glog.V(2).Infof("already watching %q", pathname)
		return nil
	}
	if err := t.w.Add(pathname, t.eventsHandle); err != nil {
		return err
	}
	// New file at start of program, seek to EOF.
	return t.openLogPath(pathname, false)
}

// handleLogEvent is dispatched when an Event is received, causing the tailer
// to read all available bytes from an already-opened file and send each log
// line onto lines channel.  Because we handle rotations and truncates when
// reaching EOF in the file reader itself, we don't care what the signal is
// from the filewatcher.
func (t *Tailer) handleLogEvent(pathname string) {
	glog.V(2).Infof("handleLogUpdate %s", pathname)
	fd, ok := t.handleForPath(pathname)
	if !ok {
		glog.V(1).Infof("No file handle found for %q, but is being watched", pathname)
		// We want to open files we have watches on in case the file was
		// unreadable before now; but we have to copmare against the glob to be
		// sure we don't just add all the files in a watched directory as they
		// get modified.
		t.handleCreateGlob(pathname)
		return
	}
	doFollow(t.ctx, fd)
}

// doFollow performs the Follow on an existing file descriptor, logging any errors
func doFollow(ctx context.Context, fd *File) {
	err := fd.Follow(ctx)
	if err != nil && err != io.EOF {
		glog.Info(err)
	}
}

// watchDirname adds the directory containing a path to be watched.
func (t *Tailer) watchDirname(pathname string) error {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return err
	}
	d := filepath.Dir(absPath)
	return t.w.Add(d, t.eventsHandle)
}

// openLogPath opens a log file named by pathname.
func (t *Tailer) openLogPath(pathname string, seekToStart bool) error {
	glog.V(2).Infof("openlogPath %s %v", pathname, seekToStart)
	if err := t.watchDirname(pathname); err != nil {
		return err
	}
	f, err := NewFile(pathname, t.lines, seekToStart || t.oneShot)
	if err != nil {
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			glog.V(1).Infof("pathname %q doesn't exist (yet?)", pathname)
			return nil
		}
		return err
	}
	glog.V(2).Infof("Adding a file watch on %q", f.Pathname)
	if err := t.w.Add(f.Pathname, t.eventsHandle); err != nil {
		return err
	}
	if err := t.setHandle(pathname, f); err != nil {
		return err
	}
	if err := f.Read(t.ctx); err != nil && err != io.EOF {
		return err
	}
	glog.Infof("Tailing %s", f.Pathname)
	logCount.Add(1)
	return nil
}

// handleCreateGlob matches the pathname against the glob patterns and starts tailing the file.
func (t *Tailer) handleCreateGlob(pathname string) {
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()

	for pattern := range t.globPatterns {
		matched, err := filepath.Match(pattern, pathname)
		if err != nil {
			glog.Warningf("Unexpected bad pattern %q not detected earlier", pattern)
			continue
		}
		if !matched {
			glog.V(2).Infof("%q did not match pattern %q", pathname, pattern)
			continue
		}
		glog.V(1).Infof("New file %q matched existing glob %q", pathname, pattern)
		// If this file was just created, read from the start of the file.
		if err := t.openLogPath(pathname, true); err != nil {
			glog.Infof("Failed to tail new file %q: %s", pathname, err)
		}
		glog.V(2).Infof("started tailing %q", pathname)
		return
	}
	glog.V(2).Infof("did not start tailing %q", pathname)
}

// run the main event loop for the Tailer.  It receives notification of
// log file changes from the watcher channel, and dispatches the log event
// handler.
func (t *Tailer) run(events <-chan watcher.Event) {
	defer close(t.runDone)

	for e := range events {
		glog.V(2).Infof("Event type %#v", e)
		t.handleLogEvent(e.Pathname)
	}
	glog.Infof("Closing lines channel.")
	close(t.lines)
	glog.Infof("Shutting down tailer.")
}

// Close signals termination to the watcher.
func (t *Tailer) Close() error {
	if err := t.w.Close(); err != nil {
		return err
	}
	<-t.runDone
	return nil
}

const tailerTemplate = `

Log Tailer

Patterns

{{range $name, $val := $.Patterns}}

{{$name}}

{{end}}

Log files watched

pathname
errors
rotations
truncations
lines read

{{range $name, $val := $.Handles}}

{{$name}}

{{index $.Errors $name}}
{{index $.Rotations $name}}
{{index $.Truncs $name}}
{{index $.Lines $name}}

{{end}}

`

// WriteStatusHTML emits the Tailer's state in HTML format to the io.Writer w.
func (t *Tailer) WriteStatusHTML(w io.Writer) error {
	tpl, err := template.New("tailer").Parse(tailerTemplate)
	if err != nil {
		return err
	}
	t.handlesMu.RLock()
	defer t.handlesMu.RUnlock()
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()
	data := struct {
		Handles   map[string]*File
		Patterns  map[string]struct{}
		Rotations map[string]string
		Lines     map[string]string
		Errors    map[string]string
		Truncs    map[string]string
	}{
		t.handles,
		t.globPatterns,
		make(map[string]string),
		make(map[string]string),
		make(map[string]string),
		make(map[string]string),
	}
	for _, pair := range []struct {
		v *expvar.Map
		m map[string]string
	}{
		{logErrors, data.Errors},
		{logRotations, data.Rotations},
		{logTruncs, data.Truncs},
		{lineCount, data.Lines},
	} {
		pair.v.Do(func(kv expvar.KeyValue) {
			pair.m[kv.Key] = kv.Value.String()
		})
	}
	return tpl.Execute(w, data)
}

// Gc removes file handles that have had no reads for 24h or more.
func (t *Tailer) Gc() error {
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	for k, v := range t.handles {
		if time.Since(v.LastRead) > (time.Hour * 24) {
			if err := t.w.Remove(v.Pathname); err != nil {
				glog.Info(err)
			}
			if err := v.Close(t.ctx); err != nil {
				glog.Info(err)
			}
			delete(t.handles, k)
		}
	}
	return nil
}

// StartExpiryLoop runs a permanent goroutine to expire metrics every duration.
func (t *Tailer) StartGcLoop(duration time.Duration) {
	if duration <= 0 {
		glog.Info("Log handle expiration disabled")
		return
	}
	go func() {
		glog.Infof("Starting log handle expiry loop every %s", duration.String())
		ticker := time.NewTicker(duration)
		for range ticker.C {
			if err := t.Gc(); err != nil {
				glog.Info(err)
			}
		}
	}()
}
<commit_msg>Trace spans in the tailer.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing log files
// and extracting new log lines to be passed into the virtual machines.
package tailer

// For regular files, mtail gets notified on modifications (i.e. appends) to
// log files that are being watched, in order to read the new lines. Log files
// can also be rotated, so mtail is also notified of creates in the log file
// directory.

import (
	"context"
	"expvar"
	"html/template"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"go.opencensus.io/trace"

	"github.com/google/mtail/internal/logline"
	"github.com/google/mtail/internal/watcher"
)

var (
	// logCount records the number of logs that are being tailed
	logCount = expvar.NewInt("log_count")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	lines chan<- *logline.LogLine // Logfile lines being emitted.
	w     watcher.Watcher
	ctx   context.Context

	handlesMu sync.RWMutex     // protects `handles'
	handles   map[string]*File // File handles for each pathname.

	globPatternsMu sync.RWMutex        // protects `globPatterns'
	globPatterns   map[string]struct{} // glob patterns to match newly created files in dir paths against

	runDone chan struct{} // Signals termination of the run goroutine.

	eventsHandle int // record the handle with which to add new log files to the watcher

	oneShot bool
}

// OneShot puts the tailer in one-shot mode.
func OneShot(t *Tailer) error {
	t.oneShot = true
	return nil
}

// Context sets the context of the tailer
func Context(ctx context.Context) func(*Tailer) error {
	return func(t *Tailer) error {
		t.ctx = ctx
		return nil
	}
}

// New creates a new Tailer.
func New(lines chan<- *logline.LogLine, w watcher.Watcher, options ...func(*Tailer) error) (*Tailer, error) {
	if lines == nil {
		return nil, errors.New("can't create tailer without lines channel")
	}
	if w == nil {
		return nil, errors.New("can't create tailer without W")
	}
	t := &Tailer{
		lines:        lines,
		w:            w,
		handles:      make(map[string]*File),
		globPatterns: make(map[string]struct{}),
		runDone:      make(chan struct{}),
	}
	if err := t.SetOption(options...); err != nil {
		return nil, err
	}
	handle, eventsChan := t.w.Events()
	t.eventsHandle = handle
	go t.run(eventsChan)
	return t, nil
}

// SetOption takes one or more option functions and applies them in order to Tailer.
func (t *Tailer) SetOption(options ...func(*Tailer) error) error {
	for _, option := range options {
		if err := option(t); err != nil {
			return err
		}
	}
	return nil
}

// setHandle sets a file handle under it's pathname
func (t *Tailer) setHandle(pathname string, f *File) error {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return errors.Wrapf(err, "Failed to lookup abspath of %q", pathname)
	}
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	t.handles[absPath] = f
	return nil
}

// handleForPath retrives a file handle for a pathname.
func (t *Tailer) handleForPath(pathname string) (*File, bool) {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		glog.V(2).Infof("Couldn't resolve path %q: %s", pathname, err)
		return nil, false
	}
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	fd, ok := t.handles[absPath]
	return fd, ok
}

func (t *Tailer) hasHandle(pathname string) bool {
	_, ok := t.handleForPath(pathname)
	return ok
}

// AddPattern adds a pattern to the list of patterns to filter filenames against.
func (t *Tailer) AddPattern(pattern string) error {
	absPath, err := filepath.Abs(pattern)
	if err != nil {
		glog.V(2).Infof("Couldn't canonicalize path %q: %s", pattern, err)
		return err
	}
	glog.V(2).Infof("AddPattern: %s", absPath)
	t.globPatternsMu.Lock()
	t.globPatterns[absPath] = struct{}{}
	t.globPatternsMu.Unlock()
	return nil
}

// TailPattern registers a pattern to be tailed.  If pattern is a plain
// file then it is watched for updates and opened.  If pattern is a glob, then
// all paths that match the glob are opened and watched, and the directories
// containing those matches, if any, are watched.
func (t *Tailer) TailPattern(pattern string) error {
	if err := t.AddPattern(pattern); err != nil {
		return err
	}
	// Add a watch on the containing directory, so we know when a rotation
	// occurs or something shows up that matches this pattern.
	if err := t.watchDirname(pattern); err != nil {
		return err
	}
	matches, err := filepath.Glob(pattern)
	if err != nil {
		return err
	}
	glog.V(1).Infof("glob matches: %v", matches)
	// Error if there are no matches, but if they show up later, they'll get picked up by the directory watch set above.
	if len(matches) == 0 {
		return errors.Errorf("No matches for pattern %q", pattern)
	}
	for _, pathname := range matches {
		err := t.TailPath(pathname)
		if err != nil {
			return errors.Wrapf(err, "attempting to tail %q", pathname)
		}
	}
	return nil
}

// TailPath registers a filesystem pathname to be tailed.
func (t *Tailer) TailPath(pathname string) error {
	if t.hasHandle(pathname) {
		glog.V(2).Infof("already watching %q", pathname)
		return nil
	}
	if err := t.w.Add(pathname, t.eventsHandle); err != nil {
		return err
	}
	// New file at start of program, seek to EOF.
	return t.openLogPath(pathname, false)
}

// handleLogEvent is dispatched when an Event is received, causing the tailer
// to read all available bytes from an already-opened file and send each log
// line onto lines channel.  Because we handle rotations and truncates when
// reaching EOF in the file reader itself, we don't care what the signal is
// from the filewatcher.
func (t *Tailer) handleLogEvent(ctx context.Context, pathname string) {
	glog.V(2).Infof("handleLogUpdate %s", pathname)
	fd, ok := t.handleForPath(pathname)
	if !ok {
		glog.V(1).Infof("No file handle found for %q, but is being watched", pathname)
		// We want to open files we have watches on in case the file was
		// unreadable before now; but we have to copmare against the glob to be
		// sure we don't just add all the files in a watched directory as they
		// get modified.
		t.handleCreateGlob(ctx, pathname)
		return
	}
	doFollow(ctx, fd)
}

// doFollow performs the Follow on an existing file descriptor, logging any errors
func doFollow(ctx context.Context, fd *File) {
	err := fd.Follow(ctx)
	if err != nil && err != io.EOF {
		glog.Info(err)
	}
}

// watchDirname adds the directory containing a path to be watched.
func (t *Tailer) watchDirname(pathname string) error {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return err
	}
	d := filepath.Dir(absPath)
	return t.w.Add(d, t.eventsHandle)
}

// openLogPath opens a log file named by pathname.
func (t *Tailer) openLogPath(pathname string, seekToStart bool) error {
	glog.V(2).Infof("openlogPath %s %v", pathname, seekToStart)
	if err := t.watchDirname(pathname); err != nil {
		return err
	}
	f, err := NewFile(pathname, t.lines, seekToStart || t.oneShot)
	if err != nil {
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			glog.V(1).Infof("pathname %q doesn't exist (yet?)", pathname)
			return nil
		}
		return err
	}
	glog.V(2).Infof("Adding a file watch on %q", f.Pathname)
	if err := t.w.Add(f.Pathname, t.eventsHandle); err != nil {
		return err
	}
	if err := t.setHandle(pathname, f); err != nil {
		return err
	}
	if err := f.Read(t.ctx); err != nil && err != io.EOF {
		return err
	}
	glog.Infof("Tailing %s", f.Pathname)
	logCount.Add(1)
	return nil
}

// handleCreateGlob matches the pathname against the glob patterns and starts tailing the file.
func (t *Tailer) handleCreateGlob(ctx context.Context, pathname string) {
	ctx, span := trace.StartSpan(ctx, "handleCreateGlob")
	defer span.End()
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()

	for pattern := range t.globPatterns {
		matched, err := filepath.Match(pattern, pathname)
		if err != nil {
			glog.Warningf("Unexpected bad pattern %q not detected earlier", pattern)
			continue
		}
		if !matched {
			glog.V(2).Infof("%q did not match pattern %q", pathname, pattern)
			continue
		}
		glog.V(1).Infof("New file %q matched existing glob %q", pathname, pattern)
		// If this file was just created, read from the start of the file.
		if err := t.openLogPath(pathname, true); err != nil {
			glog.Infof("Failed to tail new file %q: %s", pathname, err)
		}
		glog.V(2).Infof("started tailing %q", pathname)
		return
	}
	glog.V(2).Infof("did not start tailing %q", pathname)
}

// run the main event loop for the Tailer.  It receives notification of
// log file changes from the watcher channel, and dispatches the log event
// handler.
func (t *Tailer) run(events <-chan watcher.Event) {
	defer close(t.runDone)

	for e := range events {
		ctx, span := trace.StartSpan(t.ctx, "tailer.run")
		glog.V(2).Infof("Event type %#v", e)
		t.handleLogEvent(ctx, e.Pathname)
		span.End()
	}
	glog.Infof("Closing lines channel.")
	close(t.lines)
	glog.Infof("Shutting down tailer.")
}

// Close signals termination to the watcher.
func (t *Tailer) Close() error {
	if err := t.w.Close(); err != nil {
		return err
	}
	<-t.runDone
	return nil
}

const tailerTemplate = `

Log Tailer

Patterns

{{range $name, $val := $.Patterns}}

{{$name}}

{{end}}

Log files watched

pathname
errors
rotations
truncations
lines read

{{range $name, $val := $.Handles}}

{{$name}}

{{index $.Errors $name}}
{{index $.Rotations $name}}
{{index $.Truncs $name}}
{{index $.Lines $name}}

{{end}}

`

// WriteStatusHTML emits the Tailer's state in HTML format to the io.Writer w.
func (t *Tailer) WriteStatusHTML(w io.Writer) error {
	tpl, err := template.New("tailer").Parse(tailerTemplate)
	if err != nil {
		return err
	}
	t.handlesMu.RLock()
	defer t.handlesMu.RUnlock()
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()
	data := struct {
		Handles   map[string]*File
		Patterns  map[string]struct{}
		Rotations map[string]string
		Lines     map[string]string
		Errors    map[string]string
		Truncs    map[string]string
	}{
		t.handles,
		t.globPatterns,
		make(map[string]string),
		make(map[string]string),
		make(map[string]string),
		make(map[string]string),
	}
	for _, pair := range []struct {
		v *expvar.Map
		m map[string]string
	}{
		{logErrors, data.Errors},
		{logRotations, data.Rotations},
		{logTruncs, data.Truncs},
		{lineCount, data.Lines},
	} {
		pair.v.Do(func(kv expvar.KeyValue) {
			pair.m[kv.Key] = kv.Value.String()
		})
	}
	return tpl.Execute(w, data)
}

// Gc removes file handles that have had no reads for 24h or more.
func (t *Tailer) Gc() error {
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	for k, v := range t.handles {
		if time.Since(v.LastRead) > (time.Hour * 24) {
			if err := t.w.Remove(v.Pathname); err != nil {
				glog.Info(err)
			}
			if err := v.Close(t.ctx); err != nil {
				glog.Info(err)
			}
			delete(t.handles, k)
		}
	}
	return nil
}

// StartExpiryLoop runs a permanent goroutine to expire metrics every duration.
func (t *Tailer) StartGcLoop(duration time.Duration) {
	if duration <= 0 {
		glog.Info("Log handle expiration disabled")
		return
	}
	go func() {
		glog.Infof("Starting log handle expiry loop every %s", duration.String())
		ticker := time.NewTicker(duration)
		for range ticker.C {
			if err := t.Gc(); err != nil {
				glog.Info(err)
			}
		}
	}()
}
<|endoftext|>


<commit_before>// +build linux

package plugin

import (
	"encoding/json"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/daemon/initlayer"
	"github.com/docker/docker/libcontainerd"
	"github.com/docker/docker/pkg/mount"
	"github.com/docker/docker/pkg/plugins"
	"github.com/docker/docker/pkg/stringid"
	"github.com/docker/docker/plugin/v2"
	"github.com/opencontainers/go-digest"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
)

func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) error {
	p.Rootfs = filepath.Join(pm.config.Root, p.PluginObj.ID, "rootfs")
	if p.IsEnabled() && !force {
		return fmt.Errorf("plugin %s is already enabled", p.Name())
	}
	spec, err := p.InitSpec(pm.config.ExecRoot)
	if err != nil {
		return err
	}

	c.restart = true
	c.exitChan = make(chan bool)

	pm.mu.Lock()
	pm.cMap[p] = c
	pm.mu.Unlock()

	var propRoot string
	if p.PropagatedMount != "" {
		propRoot = filepath.Join(filepath.Dir(p.Rootfs), "propagated-mount")

		if err := os.MkdirAll(propRoot, 0755); err != nil {
			logrus.Errorf("failed to create PropagatedMount directory at %s: %v", propRoot, err)
		}

		if err := mount.MakeRShared(propRoot); err != nil {
			return errors.Wrap(err, "error setting up propagated mount dir")
		}

		if err := mount.Mount(propRoot, p.PropagatedMount, "none", "rbind"); err != nil {
			return errors.Wrap(err, "error creating mount for propagated mount")
		}
	}

	if err := initlayer.Setup(filepath.Join(pm.config.Root, p.PluginObj.ID, rootFSFileName), 0, 0); err != nil {
		return errors.WithStack(err)
	}

	if err := pm.containerdClient.Create(p.GetID(), "", "", specs.Spec(*spec), attachToLog(p.GetID())); err != nil {
		if p.PropagatedMount != "" {
			if err := mount.Unmount(p.PropagatedMount); err != nil {
				logrus.Warnf("Could not unmount %s: %v", p.PropagatedMount, err)
			}
			if err := mount.Unmount(propRoot); err != nil {
				logrus.Warnf("Could not unmount %s: %v", propRoot, err)
			}
		}
		return errors.WithStack(err)
	}

	return pm.pluginPostStart(p, c)
}

func (pm *Manager) pluginPostStart(p *v2.Plugin, c *controller) error {
	sockAddr := filepath.Join(pm.config.ExecRoot, p.GetID(), p.GetSocket())
	client, err := plugins.NewClientWithTimeout("unix://"+sockAddr, nil, c.timeoutInSecs)
	if err != nil {
		c.restart = false
		shutdownPlugin(p, c, pm.containerdClient)
		return errors.WithStack(err)
	}

	p.SetPClient(client)

	maxRetries := 3
	var retries int
	for {
		time.Sleep(3 * time.Second)
		retries++

		if retries > maxRetries {
			logrus.Debugf("error net dialing plugin: %v", err)
			c.restart = false
			// While restoring plugins, we need to explicitly set the state to disabled
			pm.config.Store.SetState(p, false)
			shutdownPlugin(p, c, pm.containerdClient)
			return err
		}

		// net dial into the unix socket to see if someone's listening.
		conn, err := net.Dial("unix", sockAddr)
		if err == nil {
			conn.Close()
			break
		}
	}
	pm.config.Store.SetState(p, true)
	pm.config.Store.CallHandler(p)

	return pm.save(p)
}

func (pm *Manager) restore(p *v2.Plugin) error {
	if err := pm.containerdClient.Restore(p.GetID(), attachToLog(p.GetID())); err != nil {
		return err
	}

	if pm.config.LiveRestoreEnabled {
		c := &controller{}
		if pids, _ := pm.containerdClient.GetPidsForContainer(p.GetID()); len(pids) == 0 {
			// plugin is not running, so follow normal startup procedure
			return pm.enable(p, c, true)
		}

		c.exitChan = make(chan bool)
		c.restart = true
		pm.mu.Lock()
		pm.cMap[p] = c
		pm.mu.Unlock()
		return pm.pluginPostStart(p, c)
	}

	return nil
}

func shutdownPlugin(p *v2.Plugin, c *controller, containerdClient libcontainerd.Client) {
	pluginID := p.GetID()

	err := containerdClient.Signal(pluginID, int(syscall.SIGTERM))
	if err != nil {
		logrus.Errorf("Sending SIGTERM to plugin failed with error: %v", err)
	} else {
		select {
		case <-c.exitChan:
			logrus.Debug("Clean shutdown of plugin")
		case <-time.After(time.Second * 10):
			logrus.Debug("Force shutdown plugin")
			if err := containerdClient.Signal(pluginID, int(syscall.SIGKILL)); err != nil {
				logrus.Errorf("Sending SIGKILL to plugin failed with error: %v", err)
			}
		}
	}
}

func (pm *Manager) disable(p *v2.Plugin, c *controller) error {
	if !p.IsEnabled() {
		return fmt.Errorf("plugin %s is already disabled", p.Name())
	}

	c.restart = false
	shutdownPlugin(p, c, pm.containerdClient)
	pm.config.Store.SetState(p, false)
	return pm.save(p)
}

// Shutdown stops all plugins and called during daemon shutdown.
func (pm *Manager) Shutdown() {
	plugins := pm.config.Store.GetAll()
	for _, p := range plugins {
		pm.mu.RLock()
		c := pm.cMap[p]
		pm.mu.RUnlock()

		if pm.config.LiveRestoreEnabled && p.IsEnabled() {
			logrus.Debug("Plugin active when liveRestore is set, skipping shutdown")
			continue
		}
		if pm.containerdClient != nil && p.IsEnabled() {
			c.restart = false
			shutdownPlugin(p, c, pm.containerdClient)
		}
	}
}

func (pm *Manager) upgradePlugin(p *v2.Plugin, configDigest digest.Digest, blobsums []digest.Digest, tmpRootFSDir string, privileges *types.PluginPrivileges) (err error) {
	config, err := pm.setupNewPlugin(configDigest, blobsums, privileges)
	if err != nil {
		return err
	}

	pdir := filepath.Join(pm.config.Root, p.PluginObj.ID)
	orig := filepath.Join(pdir, "rootfs")
	backup := orig + "-old"
	if err := os.Rename(orig, backup); err != nil {
		return err
	}

	defer func() {
		if err != nil {
			if rmErr := os.RemoveAll(orig); rmErr != nil && !os.IsNotExist(rmErr) {
				logrus.WithError(rmErr).WithField("dir", backup).Error("error cleaning up after failed upgrade")
				return
			}

			if err := os.Rename(backup, orig); err != nil {
				err = errors.Wrap(err, "error restoring old plugin root on upgrade failure")
			}
			if rmErr := os.RemoveAll(tmpRootFSDir); rmErr != nil && !os.IsNotExist(rmErr) {
				logrus.WithError(rmErr).WithField("plugin", p.Name()).Errorf("error cleaning up plugin upgrade dir: %s", tmpRootFSDir)
			}
		} else {
			if rmErr := os.RemoveAll(backup); rmErr != nil && !os.IsNotExist(rmErr) {
				logrus.WithError(rmErr).WithField("dir", backup).Error("error cleaning up old plugin root after successful upgrade")
			}

			p.Config = configDigest
			p.Blobsums = blobsums
		}
	}()

	if err := os.Rename(tmpRootFSDir, orig); err != nil {
		return errors.Wrap(err, "error upgrading")
	}

	p.PluginObj.Config = config
	err = pm.save(p)
	return errors.Wrap(err, "error saving upgraded plugin config")
}

func (pm *Manager) setupNewPlugin(configDigest digest.Digest, blobsums []digest.Digest, privileges *types.PluginPrivileges) (types.PluginConfig, error) {
	configRC, err := pm.blobStore.Get(configDigest)
	if err != nil {
		return types.PluginConfig{}, err
	}
	defer configRC.Close()

	var config types.PluginConfig
	dec := json.NewDecoder(configRC)
	if err := dec.Decode(&config); err != nil {
		return types.PluginConfig{}, errors.Wrapf(err, "failed to parse config")
	}
	if dec.More() {
		return types.PluginConfig{}, errors.New("invalid config json")
	}

	requiredPrivileges, err := computePrivileges(config)
	if err != nil {
		return types.PluginConfig{}, err
	}
	if privileges != nil {
		if err := validatePrivileges(requiredPrivileges, *privileges); err != nil {
			return types.PluginConfig{}, err
		}
	}

	return config, nil
}

// createPlugin creates a new plugin. take lock before calling.
func (pm *Manager) createPlugin(name string, configDigest digest.Digest, blobsums []digest.Digest, rootFSDir string, privileges *types.PluginPrivileges) (p *v2.Plugin, err error) {
	if err := pm.config.Store.validateName(name); err != nil { // todo: this check is wrong. remove store
		return nil, err
	}

	config, err := pm.setupNewPlugin(configDigest, blobsums, privileges)
	if err != nil {
		return nil, err
	}

	p = &v2.Plugin{
		PluginObj: types.Plugin{
			Name:   name,
			ID:     stringid.GenerateRandomID(),
			Config: config,
		},
		Config:   configDigest,
		Blobsums: blobsums,
	}
	p.InitEmptySettings()

	pdir := filepath.Join(pm.config.Root, p.PluginObj.ID)
	if err := os.MkdirAll(pdir, 0700); err != nil {
		return nil, errors.Wrapf(err, "failed to mkdir %v", pdir)
	}

	defer func() {
		if err != nil {
			os.RemoveAll(pdir)
		}
	}()

	if err := os.Rename(rootFSDir, filepath.Join(pdir, rootFSFileName)); err != nil {
		return nil, errors.Wrap(err, "failed to rename rootfs")
	}

	if err := pm.save(p); err != nil {
		return nil, err
	}

	pm.config.Store.Add(p) // todo: remove

	return p, nil
}
<commit_msg>fix cleanup logic if restoring plugin fails<commit_after>// +build linux

package plugin

import (
	"encoding/json"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/daemon/initlayer"
	"github.com/docker/docker/libcontainerd"
	"github.com/docker/docker/pkg/mount"
	"github.com/docker/docker/pkg/plugins"
	"github.com/docker/docker/pkg/stringid"
	"github.com/docker/docker/plugin/v2"
	"github.com/opencontainers/go-digest"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
)

func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) error {
	p.Rootfs = filepath.Join(pm.config.Root, p.PluginObj.ID, "rootfs")
	if p.IsEnabled() && !force {
		return fmt.Errorf("plugin %s is already enabled", p.Name())
	}
	spec, err := p.InitSpec(pm.config.ExecRoot)
	if err != nil {
		return err
	}

	c.restart = true
	c.exitChan = make(chan bool)

	pm.mu.Lock()
	pm.cMap[p] = c
	pm.mu.Unlock()

	var propRoot string
	if p.PropagatedMount != "" {
		propRoot = filepath.Join(filepath.Dir(p.Rootfs), "propagated-mount")

		if err := os.MkdirAll(propRoot, 0755); err != nil {
			logrus.Errorf("failed to create PropagatedMount directory at %s: %v", propRoot, err)
		}

		if err := mount.MakeRShared(propRoot); err != nil {
			return errors.Wrap(err, "error setting up propagated mount dir")
		}

		if err := mount.Mount(propRoot, p.PropagatedMount, "none", "rbind"); err != nil {
			return errors.Wrap(err, "error creating mount for propagated mount")
		}
	}

	if err := initlayer.Setup(filepath.Join(pm.config.Root, p.PluginObj.ID, rootFSFileName), 0, 0); err != nil {
		return errors.WithStack(err)
	}

	if err := pm.containerdClient.Create(p.GetID(), "", "", specs.Spec(*spec), attachToLog(p.GetID())); err != nil {
		if p.PropagatedMount != "" {
			if err := mount.Unmount(p.PropagatedMount); err != nil {
				logrus.Warnf("Could not unmount %s: %v", p.PropagatedMount, err)
			}
			if err := mount.Unmount(propRoot); err != nil {
				logrus.Warnf("Could not unmount %s: %v", propRoot, err)
			}
		}
		return errors.WithStack(err)
	}

	return pm.pluginPostStart(p, c)
}

func (pm *Manager) pluginPostStart(p *v2.Plugin, c *controller) error {
	sockAddr := filepath.Join(pm.config.ExecRoot, p.GetID(), p.GetSocket())
	client, err := plugins.NewClientWithTimeout("unix://"+sockAddr, nil, c.timeoutInSecs)
	if err != nil {
		c.restart = false
		shutdownPlugin(p, c, pm.containerdClient)
		return errors.WithStack(err)
	}

	p.SetPClient(client)

	maxRetries := 3
	var retries int
	for {
		time.Sleep(3 * time.Second)
		retries++

		if retries > maxRetries {
			logrus.Debugf("error net dialing plugin: %v", err)
			c.restart = false
			// While restoring plugins, we need to explicitly set the state to disabled
			pm.config.Store.SetState(p, false)
			shutdownPlugin(p, c, pm.containerdClient)
			return err
		}

		// net dial into the unix socket to see if someone's listening.
		conn, err := net.Dial("unix", sockAddr)
		if err == nil {
			conn.Close()
			break
		}
	}
	pm.config.Store.SetState(p, true)
	pm.config.Store.CallHandler(p)

	return pm.save(p)
}

func (pm *Manager) restore(p *v2.Plugin) error {
	if err := pm.containerdClient.Restore(p.GetID(), attachToLog(p.GetID())); err != nil {
		return err
	}

	if pm.config.LiveRestoreEnabled {
		c := &controller{}
		if pids, _ := pm.containerdClient.GetPidsForContainer(p.GetID()); len(pids) == 0 {
			// plugin is not running, so follow normal startup procedure
			return pm.enable(p, c, true)
		}

		c.exitChan = make(chan bool)
		c.restart = true
		pm.mu.Lock()
		pm.cMap[p] = c
		pm.mu.Unlock()
		return pm.pluginPostStart(p, c)
	}

	return nil
}

func shutdownPlugin(p *v2.Plugin, c *controller, containerdClient libcontainerd.Client) {
	pluginID := p.GetID()

	err := containerdClient.Signal(pluginID, int(syscall.SIGTERM))
	if err != nil {
		logrus.Errorf("Sending SIGTERM to plugin failed with error: %v", err)
	} else {
		select {
		case <-c.exitChan:
			logrus.Debug("Clean shutdown of plugin")
		case <-time.After(time.Second * 10):
			logrus.Debug("Force shutdown plugin")
			if err := containerdClient.Signal(pluginID, int(syscall.SIGKILL)); err != nil {
				logrus.Errorf("Sending SIGKILL to plugin failed with error: %v", err)
			}
		}
	}
}

func (pm *Manager) disable(p *v2.Plugin, c *controller) error {
	if !p.IsEnabled() {
		return fmt.Errorf("plugin %s is already disabled", p.Name())
	}

	c.restart = false
	shutdownPlugin(p, c, pm.containerdClient)
	pm.config.Store.SetState(p, false)
	return pm.save(p)
}

// Shutdown stops all plugins and called during daemon shutdown.
func (pm *Manager) Shutdown() {
	plugins := pm.config.Store.GetAll()
	for _, p := range plugins {
		pm.mu.RLock()
		c := pm.cMap[p]
		pm.mu.RUnlock()

		if pm.config.LiveRestoreEnabled && p.IsEnabled() {
			logrus.Debug("Plugin active when liveRestore is set, skipping shutdown")
			continue
		}
		if pm.containerdClient != nil && p.IsEnabled() {
			c.restart = false
			shutdownPlugin(p, c, pm.containerdClient)
		}
	}
}

func (pm *Manager) upgradePlugin(p *v2.Plugin, configDigest digest.Digest, blobsums []digest.Digest, tmpRootFSDir string, privileges *types.PluginPrivileges) (err error) {
	config, err := pm.setupNewPlugin(configDigest, blobsums, privileges)
	if err != nil {
		return err
	}

	pdir := filepath.Join(pm.config.Root, p.PluginObj.ID)
	orig := filepath.Join(pdir, "rootfs")
	backup := orig + "-old"
	if err := os.Rename(orig, backup); err != nil {
		return err
	}

	defer func() {
		if err != nil {
			if rmErr := os.RemoveAll(orig); rmErr != nil && !os.IsNotExist(rmErr) {
				logrus.WithError(rmErr).WithField("dir", backup).Error("error cleaning up after failed upgrade")
				return
			}
			if mvErr := os.Rename(backup, orig); mvErr != nil {
				err = errors.Wrap(mvErr, "error restoring old plugin root on upgrade failure")
			}
			if rmErr := os.RemoveAll(tmpRootFSDir); rmErr != nil && !os.IsNotExist(rmErr) {
				logrus.WithError(rmErr).WithField("plugin", p.Name()).Errorf("error cleaning up plugin upgrade dir: %s", tmpRootFSDir)
			}
		} else {
			if rmErr := os.RemoveAll(backup); rmErr != nil && !os.IsNotExist(rmErr) {
				logrus.WithError(rmErr).WithField("dir", backup).Error("error cleaning up old plugin root after successful upgrade")
			}

			p.Config = configDigest
			p.Blobsums = blobsums
		}
	}()

	if err := os.Rename(tmpRootFSDir, orig); err != nil {
		return errors.Wrap(err, "error upgrading")
	}

	p.PluginObj.Config = config
	err = pm.save(p)
	return errors.Wrap(err, "error saving upgraded plugin config")
}

func (pm *Manager) setupNewPlugin(configDigest digest.Digest, blobsums []digest.Digest, privileges *types.PluginPrivileges) (types.PluginConfig, error) {
	configRC, err := pm.blobStore.Get(configDigest)
	if err != nil {
		return types.PluginConfig{}, err
	}
	defer configRC.Close()

	var config types.PluginConfig
	dec := json.NewDecoder(configRC)
	if err := dec.Decode(&config); err != nil {
		return types.PluginConfig{}, errors.Wrapf(err, "failed to parse config")
	}
	if dec.More() {
		return types.PluginConfig{}, errors.New("invalid config json")
	}

	requiredPrivileges, err := computePrivileges(config)
	if err != nil {
		return types.PluginConfig{}, err
	}
	if privileges != nil {
		if err := validatePrivileges(requiredPrivileges, *privileges); err != nil {
			return types.PluginConfig{}, err
		}
	}

	return config, nil
}

// createPlugin creates a new plugin. take lock before calling.
func (pm *Manager) createPlugin(name string, configDigest digest.Digest, blobsums []digest.Digest, rootFSDir string, privileges *types.PluginPrivileges) (p *v2.Plugin, err error) {
	if err := pm.config.Store.validateName(name); err != nil { // todo: this check is wrong. remove store
		return nil, err
	}

	config, err := pm.setupNewPlugin(configDigest, blobsums, privileges)
	if err != nil {
		return nil, err
	}

	p = &v2.Plugin{
		PluginObj: types.Plugin{
			Name:   name,
			ID:     stringid.GenerateRandomID(),
			Config: config,
		},
		Config:   configDigest,
		Blobsums: blobsums,
	}
	p.InitEmptySettings()

	pdir := filepath.Join(pm.config.Root, p.PluginObj.ID)
	if err := os.MkdirAll(pdir, 0700); err != nil {
		return nil, errors.Wrapf(err, "failed to mkdir %v", pdir)
	}

	defer func() {
		if err != nil {
			os.RemoveAll(pdir)
		}
	}()

	if err := os.Rename(rootFSDir, filepath.Join(pdir, rootFSFileName)); err != nil {
		return nil, errors.Wrap(err, "failed to rename rootfs")
	}

	if err := pm.save(p); err != nil {
		return nil, err
	}

	pm.config.Store.Add(p) // todo: remove

	return p, nil
}
<|endoftext|>