<commit_before>package nat

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	ma "github.com/jbenet/go-multiaddr"
	manet "github.com/jbenet/go-multiaddr-net"

	nat "github.com/fd/go-nat"
	logging "github.com/ipfs/go-log"
	goprocess "github.com/jbenet/goprocess"
	periodic "github.com/jbenet/goprocess/periodic"
	notifier "github.com/whyrusleeping/go-notifier"
)

var (
	// ErrNoMapping signals no mapping exists for an address
	ErrNoMapping = errors.New("mapping not established")
)

var log = logging.Logger("nat")

// MappingDuration is a default port mapping duration.
// Port mappings are renewed every (MappingDuration / 3)
const MappingDuration = time.Second * 60

// CacheTime is the time a mapping will cache an external address for
const CacheTime = time.Second * 15

// DiscoverNAT looks for a NAT device in the network and
// returns an object that can manage port mappings.
func DiscoverNAT() *NAT {
	nat, err := nat.DiscoverGateway()
	if err != nil {
		log.Debug("DiscoverGateway error:", err)
		return nil
	}
	addr, err := nat.GetDeviceAddress()
	if err != nil {
		log.Debug("DiscoverGateway address error:", err)
	} else {
		log.Debug("DiscoverGateway address:", addr)
	}
	return newNAT(nat)
}

// NAT is an object that manages address port mappings in
// NATs (Network Address Translators). It is a long-running
// service that will periodically renew port mappings,
// and keep an up-to-date list of all the external addresses.
type NAT struct {
	nat  nat.NAT
	proc goprocess.Process // manages nat mappings lifecycle

	mappingmu sync.RWMutex // guards mappings
	mappings  map[*mapping]struct{}

	Notifier
}

func newNAT(realNAT nat.NAT) *NAT {
	return &NAT{
		nat:      realNAT,
		proc:     goprocess.WithParent(goprocess.Background()),
		mappings: make(map[*mapping]struct{}),
	}
}

// Close shuts down all port mappings. NAT can no longer be used.
func (nat *NAT) Close() error {
	return nat.proc.Close()
}

// Process returns the nat's life-cycle manager, for making it listen
// to close signals.
func (nat *NAT) Process() goprocess.Process {
	return nat.proc
}

// Notifier is an object that assists NAT in notifying listeners.
// It is implemented using thirdparty/notifier
type Notifier struct {
	n notifier.Notifier
}

func (n *Notifier) notifyAll(notify func(n Notifiee)) {
	n.n.NotifyAll(func(n notifier.Notifiee) {
		notify(n.(Notifiee))
	})
}

// Notify signs up notifiee to listen to NAT events.
func (n *Notifier) Notify(notifiee Notifiee) {
	n.n.Notify(n)
}

// StopNotify stops signaling events to notifiee.
func (n *Notifier) StopNotify(notifiee Notifiee) {
	n.n.StopNotify(notifiee)
}

// Notifiee is an interface objects must implement to listen to NAT events.
type Notifiee interface {

	// Called every time a successful mapping happens
	// Warning: the port mapping may have changed. If that is the
	// case, both MappingSuccess and MappingChanged are called.
	MappingSuccess(nat *NAT, m Mapping)

	// Called when mapping a port succeeds, but the mapping is
	// with a different port than an earlier success.
	MappingChanged(nat *NAT, m Mapping, oldport int, newport int)

	// Called when a port mapping fails. NAT will continue attempting after
	// the next period. To stop trying, use: mapping.Close(). After this failure,
	// mapping.ExternalPort() will be zero, and nat.ExternalAddrs() will not
	// return the address for this mapping. With luck, the next attempt will
	// succeed, without the client needing to do anything.
	MappingFailed(nat *NAT, m Mapping, oldport int, err error)
}

// Mapping represents a port mapping in a NAT.
type Mapping interface {
	// NAT returns the NAT object this Mapping belongs to.
	NAT() *NAT

	// Protocol returns the protocol of this port mapping. This is either
	// "tcp" or "udp" as no other protocols are likely to be NAT-supported.
	Protocol() string

	// InternalPort returns the internal device port. Mapping will continue to
	// try to map InternalPort() to an external facing port.
	InternalPort() int

	// ExternalPort returns the external facing port. If the mapping is not
	// established, port will be 0
	ExternalPort() int

	// InternalAddr returns the internal address.
	InternalAddr() ma.Multiaddr

	// ExternalAddr returns the external facing address. If the mapping is not
	// established, addr will be nil, and and ErrNoMapping will be returned.
	ExternalAddr() (addr ma.Multiaddr, err error)

	// Close closes the port mapping
	Close() error
}

// keeps republishing
type mapping struct {
	sync.Mutex // guards all fields

	nat     *NAT
	proto   string
	intport int
	extport int
	intaddr ma.Multiaddr
	proc    goprocess.Process

	cached    ma.Multiaddr
	cacheTime time.Time
}

func (m *mapping) NAT() *NAT {
	m.Lock()
	defer m.Unlock()
	return m.nat
}

func (m *mapping) Protocol() string {
	m.Lock()
	defer m.Unlock()
	return m.proto
}

func (m *mapping) InternalPort() int {
	m.Lock()
	defer m.Unlock()
	return m.intport
}

func (m *mapping) ExternalPort() int {
	m.Lock()
	defer m.Unlock()
	return m.extport
}

func (m *mapping) setExternalPort(p int) {
	m.Lock()
	defer m.Unlock()
	m.extport = p
}

func (m *mapping) InternalAddr() ma.Multiaddr {
	m.Lock()
	defer m.Unlock()
	return m.intaddr
}

func (m *mapping) ExternalAddr() (ma.Multiaddr, error) {
	if time.Now().Sub(m.cacheTime) < CacheTime {
		return m.cached, nil
	}

	if m.ExternalPort() == 0 { // dont even try right now.
		return nil, ErrNoMapping
	}

	ip, err := m.nat.nat.GetExternalAddress()
	if err != nil {
		return nil, err
	}

	ipmaddr, err := manet.FromIP(ip)
	if err != nil {
		return nil, fmt.Errorf("error parsing ip")
	}

	// call m.ExternalPort again, as mapping may have changed under our feet. (tocttou)
	extport := m.ExternalPort()
	if extport == 0 {
		return nil, ErrNoMapping
	}

	tcp, err := ma.NewMultiaddr(fmt.Sprintf("/%s/%d", m.Protocol(), extport))
	if err != nil {
		return nil, err
	}

	maddr2 := ipmaddr.Encapsulate(tcp)

	m.cached = maddr2
	m.cacheTime = time.Now()
	return maddr2, nil
}

func (m *mapping) Close() error {
	return m.proc.Close()
}

// Mappings returns a slice of all NAT mappings
func (nat *NAT) Mappings() []Mapping {
	nat.mappingmu.Lock()
	maps2 := make([]Mapping, 0, len(nat.mappings))
	for m := range nat.mappings {
		maps2 = append(maps2, m)
	}
	nat.mappingmu.Unlock()
	return maps2
}

func (nat *NAT) addMapping(m *mapping) {
	// make mapping automatically close when nat is closed.
	nat.proc.AddChild(m.proc)

	nat.mappingmu.Lock()
	nat.mappings[m] = struct{}{}
	nat.mappingmu.Unlock()
}

func (nat *NAT) rmMapping(m *mapping) {
	nat.mappingmu.Lock()
	delete(nat.mappings, m)
	nat.mappingmu.Unlock()
}

// NewMapping attemps to construct a mapping on protocol and internal port
// It will also periodically renew the mapping until the returned Mapping
// -- or its parent NAT -- is Closed.
//
// May not succeed, and mappings may change over time;
// NAT devices may not respect our port requests, and even lie.
// Clients should not store the mapped results, but rather always
// poll our object for the latest mappings.
func (nat *NAT) NewMapping(maddr ma.Multiaddr) (Mapping, error) {
	if nat == nil {
		return nil, fmt.Errorf("no nat available")
	}

	network, addr, err := manet.DialArgs(maddr)
	if err != nil {
		return nil, fmt.Errorf("DialArgs failed on addr:", maddr.String())
	}

	switch network {
	case "tcp", "tcp4", "tcp6":
		network = "tcp"
	case "udp", "udp4", "udp6":
		network = "udp"
	default:
		return nil, fmt.Errorf("transport not supported by NAT: %s", network)
	}

	intports := strings.Split(addr, ":")[1]
	intport, err := strconv.Atoi(intports)
	if err != nil {
		return nil, err
	}

	m := &mapping{
		nat:     nat,
		proto:   network,
		intport: intport,
		intaddr: maddr,
	}
	m.proc = goprocess.WithTeardown(func() error {
		nat.rmMapping(m)
		return nil
	})
	nat.addMapping(m)

	m.proc.AddChild(periodic.Every(MappingDuration/3, func(worker goprocess.Process) {
		nat.establishMapping(m)
	}))

	// do it once synchronously, so first mapping is done right away, and before exiting,
	// allowing users -- in the optimistic case -- to use results right after.
	nat.establishMapping(m)
	return m, nil
}

func (nat *NAT) establishMapping(m *mapping) {
	oldport := m.ExternalPort()
	log.Debugf("Attempting port map: %s/%d", m.Protocol(), m.InternalPort())
	newport, err := nat.nat.AddPortMapping(m.Protocol(), m.InternalPort(), "http", MappingDuration)

	failure := func() {
		m.setExternalPort(0) // clear mapping
		// TODO: log.Event
		log.Debugf("failed to establish port mapping: %s", err)
		nat.Notifier.notifyAll(func(n Notifiee) {
			n.MappingFailed(nat, m, oldport, err)
		})

		// we do not close if the mapping failed,
		// because it may work again next time.
	}

	if err != nil || newport == 0 {
		failure()
		return
	}

	m.setExternalPort(newport)
	ext, err := m.ExternalAddr()
	if err != nil {
		log.Debugf("NAT Mapping addr error: %s %s", m.InternalAddr(), err)
		failure()
		return
	}

	log.Debugf("NAT Mapping: %s --> %s", m.InternalAddr(), ext)
	if oldport != 0 && newport != oldport {
		log.Debugf("failed to renew same port mapping: ch %d -> %d", oldport, newport)
		nat.Notifier.notifyAll(func(n Notifiee) {
			n.MappingChanged(nat, m, oldport, newport)
		})
	}

	nat.Notifier.notifyAll(func(n Notifiee) {
		n.MappingSuccess(nat, m)
	})
}

// PortMapAddrs attempts to open (and continue to keep open)
// port mappings for given addrs. This function blocks until
// all  addresses have been tried. This allows clients to
// retrieve results immediately after:
//
//   nat.PortMapAddrs(addrs)
//   mapped := nat.ExternalAddrs()
//
// Some may not succeed, and mappings may change over time;
// NAT devices may not respect our port requests, and even lie.
// Clients should not store the mapped results, but rather always
// poll our object for the latest mappings.
func (nat *NAT) PortMapAddrs(addrs []ma.Multiaddr) {
	// spin off addr mappings independently.
	var wg sync.WaitGroup
	for _, addr := range addrs {
		// do all of them concurrently
		wg.Add(1)
		go func() {
			defer wg.Done()
			nat.NewMapping(addr)
		}()
	}
	wg.Wait()
}

// MappedAddrs returns address mappings NAT believes have been
// successfully established. Unsuccessful mappings are nil. This is:
//
// 		map[internalAddr]externalAddr
//
// This set of mappings _may not_ be correct, as NAT devices are finicky.
// Consider this with _best effort_ semantics.
func (nat *NAT) MappedAddrs() map[ma.Multiaddr]ma.Multiaddr {

	mappings := nat.Mappings()
	addrmap := make(map[ma.Multiaddr]ma.Multiaddr, len(mappings))

	for _, m := range mappings {
		i := m.InternalAddr()
		e, err := m.ExternalAddr()
		if err != nil {
			addrmap[i] = nil
		} else {
			addrmap[i] = e
		}
	}
	return addrmap
}

// ExternalAddrs returns a list of addresses that NAT believes have
// been successfully established. Unsuccessful mappings are omitted,
// so nat.ExternalAddrs() may return less addresses than nat.InternalAddrs().
// To see which addresses are mapped, use nat.MappedAddrs().
//
// This set of mappings _may not_ be correct, as NAT devices are finicky.
// Consider this with _best effort_ semantics.
func (nat *NAT) ExternalAddrs() []ma.Multiaddr {
	mappings := nat.Mappings()
	addrs := make([]ma.Multiaddr, 0, len(mappings))
	for _, m := range mappings {
		a, err := m.ExternalAddr()
		if err != nil {
			continue // this mapping not currently successful.
		}
		addrs = append(addrs, a)
	}
	return addrs
}
<commit_msg>path rewrites<commit_after>package nat

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr"
	manet "gx/ipfs/QmYtzQmUwPFGxjCXctJ8e6GXS8sYfoXy2pdeMbS5SFWqRi/go-multiaddr-net"

	nat "gx/ipfs/QmNLvkCDV6ZjUJsEwGNporYBuZdhWT6q7TBVYQwwRv12HT/go-nat"
	goprocess "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess"
	periodic "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess/periodic"
	logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log"
	notifier "gx/ipfs/QmbcS9XrwZkF1rZj8bBwwzoYhVuA2PCnPhFUL1pyWGgt2A/go-notifier"
)

var (
	// ErrNoMapping signals no mapping exists for an address
	ErrNoMapping = errors.New("mapping not established")
)

var log = logging.Logger("nat")

// MappingDuration is a default port mapping duration.
// Port mappings are renewed every (MappingDuration / 3)
const MappingDuration = time.Second * 60

// CacheTime is the time a mapping will cache an external address for
const CacheTime = time.Second * 15

// DiscoverNAT looks for a NAT device in the network and
// returns an object that can manage port mappings.
func DiscoverNAT() *NAT {
	nat, err := nat.DiscoverGateway()
	if err != nil {
		log.Debug("DiscoverGateway error:", err)
		return nil
	}
	addr, err := nat.GetDeviceAddress()
	if err != nil {
		log.Debug("DiscoverGateway address error:", err)
	} else {
		log.Debug("DiscoverGateway address:", addr)
	}
	return newNAT(nat)
}

// NAT is an object that manages address port mappings in
// NATs (Network Address Translators). It is a long-running
// service that will periodically renew port mappings,
// and keep an up-to-date list of all the external addresses.
type NAT struct {
	nat  nat.NAT
	proc goprocess.Process // manages nat mappings lifecycle

	mappingmu sync.RWMutex // guards mappings
	mappings  map[*mapping]struct{}

	Notifier
}

func newNAT(realNAT nat.NAT) *NAT {
	return &NAT{
		nat:      realNAT,
		proc:     goprocess.WithParent(goprocess.Background()),
		mappings: make(map[*mapping]struct{}),
	}
}

// Close shuts down all port mappings. NAT can no longer be used.
func (nat *NAT) Close() error {
	return nat.proc.Close()
}

// Process returns the nat's life-cycle manager, for making it listen
// to close signals.
func (nat *NAT) Process() goprocess.Process {
	return nat.proc
}

// Notifier is an object that assists NAT in notifying listeners.
// It is implemented using thirdparty/notifier
type Notifier struct {
	n notifier.Notifier
}

func (n *Notifier) notifyAll(notify func(n Notifiee)) {
	n.n.NotifyAll(func(n notifier.Notifiee) {
		notify(n.(Notifiee))
	})
}

// Notify signs up notifiee to listen to NAT events.
func (n *Notifier) Notify(notifiee Notifiee) {
	n.n.Notify(n)
}

// StopNotify stops signaling events to notifiee.
func (n *Notifier) StopNotify(notifiee Notifiee) {
	n.n.StopNotify(notifiee)
}

// Notifiee is an interface objects must implement to listen to NAT events.
type Notifiee interface {

	// Called every time a successful mapping happens
	// Warning: the port mapping may have changed. If that is the
	// case, both MappingSuccess and MappingChanged are called.
	MappingSuccess(nat *NAT, m Mapping)

	// Called when mapping a port succeeds, but the mapping is
	// with a different port than an earlier success.
	MappingChanged(nat *NAT, m Mapping, oldport int, newport int)

	// Called when a port mapping fails. NAT will continue attempting after
	// the next period. To stop trying, use: mapping.Close(). After this failure,
	// mapping.ExternalPort() will be zero, and nat.ExternalAddrs() will not
	// return the address for this mapping. With luck, the next attempt will
	// succeed, without the client needing to do anything.
	MappingFailed(nat *NAT, m Mapping, oldport int, err error)
}

// Mapping represents a port mapping in a NAT.
type Mapping interface {
	// NAT returns the NAT object this Mapping belongs to.
	NAT() *NAT

	// Protocol returns the protocol of this port mapping. This is either
	// "tcp" or "udp" as no other protocols are likely to be NAT-supported.
	Protocol() string

	// InternalPort returns the internal device port. Mapping will continue to
	// try to map InternalPort() to an external facing port.
	InternalPort() int

	// ExternalPort returns the external facing port. If the mapping is not
	// established, port will be 0
	ExternalPort() int

	// InternalAddr returns the internal address.
	InternalAddr() ma.Multiaddr

	// ExternalAddr returns the external facing address. If the mapping is not
	// established, addr will be nil, and and ErrNoMapping will be returned.
	ExternalAddr() (addr ma.Multiaddr, err error)

	// Close closes the port mapping
	Close() error
}

// keeps republishing
type mapping struct {
	sync.Mutex // guards all fields

	nat     *NAT
	proto   string
	intport int
	extport int
	intaddr ma.Multiaddr
	proc    goprocess.Process

	cached    ma.Multiaddr
	cacheTime time.Time
}

func (m *mapping) NAT() *NAT {
	m.Lock()
	defer m.Unlock()
	return m.nat
}

func (m *mapping) Protocol() string {
	m.Lock()
	defer m.Unlock()
	return m.proto
}

func (m *mapping) InternalPort() int {
	m.Lock()
	defer m.Unlock()
	return m.intport
}

func (m *mapping) ExternalPort() int {
	m.Lock()
	defer m.Unlock()
	return m.extport
}

func (m *mapping) setExternalPort(p int) {
	m.Lock()
	defer m.Unlock()
	m.extport = p
}

func (m *mapping) InternalAddr() ma.Multiaddr {
	m.Lock()
	defer m.Unlock()
	return m.intaddr
}

func (m *mapping) ExternalAddr() (ma.Multiaddr, error) {
	if time.Now().Sub(m.cacheTime) < CacheTime {
		return m.cached, nil
	}

	if m.ExternalPort() == 0 { // dont even try right now.
		return nil, ErrNoMapping
	}

	ip, err := m.nat.nat.GetExternalAddress()
	if err != nil {
		return nil, err
	}

	ipmaddr, err := manet.FromIP(ip)
	if err != nil {
		return nil, fmt.Errorf("error parsing ip")
	}

	// call m.ExternalPort again, as mapping may have changed under our feet. (tocttou)
	extport := m.ExternalPort()
	if extport == 0 {
		return nil, ErrNoMapping
	}

	tcp, err := ma.NewMultiaddr(fmt.Sprintf("/%s/%d", m.Protocol(), extport))
	if err != nil {
		return nil, err
	}

	maddr2 := ipmaddr.Encapsulate(tcp)

	m.cached = maddr2
	m.cacheTime = time.Now()
	return maddr2, nil
}

func (m *mapping) Close() error {
	return m.proc.Close()
}

// Mappings returns a slice of all NAT mappings
func (nat *NAT) Mappings() []Mapping {
	nat.mappingmu.Lock()
	maps2 := make([]Mapping, 0, len(nat.mappings))
	for m := range nat.mappings {
		maps2 = append(maps2, m)
	}
	nat.mappingmu.Unlock()
	return maps2
}

func (nat *NAT) addMapping(m *mapping) {
	// make mapping automatically close when nat is closed.
	nat.proc.AddChild(m.proc)

	nat.mappingmu.Lock()
	nat.mappings[m] = struct{}{}
	nat.mappingmu.Unlock()
}

func (nat *NAT) rmMapping(m *mapping) {
	nat.mappingmu.Lock()
	delete(nat.mappings, m)
	nat.mappingmu.Unlock()
}

// NewMapping attemps to construct a mapping on protocol and internal port
// It will also periodically renew the mapping until the returned Mapping
// -- or its parent NAT -- is Closed.
//
// May not succeed, and mappings may change over time;
// NAT devices may not respect our port requests, and even lie.
// Clients should not store the mapped results, but rather always
// poll our object for the latest mappings.
func (nat *NAT) NewMapping(maddr ma.Multiaddr) (Mapping, error) {
	if nat == nil {
		return nil, fmt.Errorf("no nat available")
	}

	network, addr, err := manet.DialArgs(maddr)
	if err != nil {
		return nil, fmt.Errorf("DialArgs failed on addr:", maddr.String())
	}

	switch network {
	case "tcp", "tcp4", "tcp6":
		network = "tcp"
	case "udp", "udp4", "udp6":
		network = "udp"
	default:
		return nil, fmt.Errorf("transport not supported by NAT: %s", network)
	}

	intports := strings.Split(addr, ":")[1]
	intport, err := strconv.Atoi(intports)
	if err != nil {
		return nil, err
	}

	m := &mapping{
		nat:     nat,
		proto:   network,
		intport: intport,
		intaddr: maddr,
	}
	m.proc = goprocess.WithTeardown(func() error {
		nat.rmMapping(m)
		return nil
	})
	nat.addMapping(m)

	m.proc.AddChild(periodic.Every(MappingDuration/3, func(worker goprocess.Process) {
		nat.establishMapping(m)
	}))

	// do it once synchronously, so first mapping is done right away, and before exiting,
	// allowing users -- in the optimistic case -- to use results right after.
	nat.establishMapping(m)
	return m, nil
}

func (nat *NAT) establishMapping(m *mapping) {
	oldport := m.ExternalPort()
	log.Debugf("Attempting port map: %s/%d", m.Protocol(), m.InternalPort())
	newport, err := nat.nat.AddPortMapping(m.Protocol(), m.InternalPort(), "http", MappingDuration)

	failure := func() {
		m.setExternalPort(0) // clear mapping
		// TODO: log.Event
		log.Debugf("failed to establish port mapping: %s", err)
		nat.Notifier.notifyAll(func(n Notifiee) {
			n.MappingFailed(nat, m, oldport, err)
		})

		// we do not close if the mapping failed,
		// because it may work again next time.
	}

	if err != nil || newport == 0 {
		failure()
		return
	}

	m.setExternalPort(newport)
	ext, err := m.ExternalAddr()
	if err != nil {
		log.Debugf("NAT Mapping addr error: %s %s", m.InternalAddr(), err)
		failure()
		return
	}

	log.Debugf("NAT Mapping: %s --> %s", m.InternalAddr(), ext)
	if oldport != 0 && newport != oldport {
		log.Debugf("failed to renew same port mapping: ch %d -> %d", oldport, newport)
		nat.Notifier.notifyAll(func(n Notifiee) {
			n.MappingChanged(nat, m, oldport, newport)
		})
	}

	nat.Notifier.notifyAll(func(n Notifiee) {
		n.MappingSuccess(nat, m)
	})
}

// PortMapAddrs attempts to open (and continue to keep open)
// port mappings for given addrs. This function blocks until
// all  addresses have been tried. This allows clients to
// retrieve results immediately after:
//
//   nat.PortMapAddrs(addrs)
//   mapped := nat.ExternalAddrs()
//
// Some may not succeed, and mappings may change over time;
// NAT devices may not respect our port requests, and even lie.
// Clients should not store the mapped results, but rather always
// poll our object for the latest mappings.
func (nat *NAT) PortMapAddrs(addrs []ma.Multiaddr) {
	// spin off addr mappings independently.
	var wg sync.WaitGroup
	for _, addr := range addrs {
		// do all of them concurrently
		wg.Add(1)
		go func() {
			defer wg.Done()
			nat.NewMapping(addr)
		}()
	}
	wg.Wait()
}

// MappedAddrs returns address mappings NAT believes have been
// successfully established. Unsuccessful mappings are nil. This is:
//
// 		map[internalAddr]externalAddr
//
// This set of mappings _may not_ be correct, as NAT devices are finicky.
// Consider this with _best effort_ semantics.
func (nat *NAT) MappedAddrs() map[ma.Multiaddr]ma.Multiaddr {

	mappings := nat.Mappings()
	addrmap := make(map[ma.Multiaddr]ma.Multiaddr, len(mappings))

	for _, m := range mappings {
		i := m.InternalAddr()
		e, err := m.ExternalAddr()
		if err != nil {
			addrmap[i] = nil
		} else {
			addrmap[i] = e
		}
	}
	return addrmap
}

// ExternalAddrs returns a list of addresses that NAT believes have
// been successfully established. Unsuccessful mappings are omitted,
// so nat.ExternalAddrs() may return less addresses than nat.InternalAddrs().
// To see which addresses are mapped, use nat.MappedAddrs().
//
// This set of mappings _may not_ be correct, as NAT devices are finicky.
// Consider this with _best effort_ semantics.
func (nat *NAT) ExternalAddrs() []ma.Multiaddr {
	mappings := nat.Mappings()
	addrs := make([]ma.Multiaddr, 0, len(mappings))
	for _, m := range mappings {
		a, err := m.ExternalAddr()
		if err != nil {
			continue // this mapping not currently successful.
		}
		addrs = append(addrs, a)
	}
	return addrs
}
<|endoftext|>


<commit_before>package halgo

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

// Navigator is a mechanism for navigating HAL-compliant REST APIs. You
// start by creating a Navigator with a base URI, then Follow the links
// exposed by the API until you reach the place where you want to perform
// an action.
//
// For example, to request an API exposed at api.example.com and follow a
// link named products and GET the resulting page you'd do this:
//
//     res, err := Navigator("http://api.example.com").
//       Follow("products").
//       Get()
//
// To do the same thing but POST to the products page, you'd do this:
//
//     res, err := Navigator("http://api.example.com").
//       Follow("products").
//       Post("application/json", someContent)
//
// Multiple links followed in sequence.
//
//     res, err := Navigator("http://api.example.com").
//       Follow("products").
//       Follow("next")
//       Get()
//
// Links can also be expanded with Followf if they are URI templates.
//
//     res, err := Navigator("http://api.example.com").
//       Follow("products").
//       Followf("page", halgo.P{"number": 10})
//       Get()
//
// Navigation of relations is lazy. Requests will only be triggered when
// you execute a method which returns a result. For example, this doesn't
// perform any HTTP requests.
//
//     Navigator("http://api.example.com").
//       Follow("products")
//
// It's only when you add a call to Get, Post, PostForm, Patch, or
// Unmarshal to the end will any requests be triggered.
//
// By default a Navigator will use http.DefaultClient as its mechanism for
// making HTTP requests. If you want to supply your own HttpClient, you
// can assign to nav.HttpClient after creation.
//
//     nav := Navigator("http://api.example.com")
//     nav.HttpClient = MyHttpClient{}
//
// Any Client you supply must implement halgo.HttpClient, which
// http.Client does implicitly. By creating decorators for the HttpClient,
// logging and caching clients are trivial. See LoggingHttpClient for an
// example.
func Navigator(uri string) navigator {
	return navigator{
		rootUri:    uri,
		path:       []relation{},
		HttpClient: http.DefaultClient,
	}
}

// relation is an instruction of a relation to follow and any params to
// expand with when executed.
type relation struct {
	rel    string
	params P
}

// navigator is the API navigator
type navigator struct {
	// HttpClient is used to execute requests. By default it's
	// http.DefaultClient. By decorating a HttpClient instance you can
	// easily write loggers or caching mechanisms.
	HttpClient HttpClient

	// path is the follow queue.
	path []relation

	// rootUri is where the navigation will begin from.
	rootUri string
}

// Follow adds a relation to the follow queue of the navigator.
func (n navigator) Follow(rel string) navigator {
	return n.Followf(rel, nil)
}

// Followf adds a relation to the follow queue of the navigator, with a
// set of parameters to expand on execution.
func (n navigator) Followf(rel string, params P) navigator {
	relations := make([]relation, 0, len(n.path)+1)
	copy(n.path, relations)
	relations = append(relations, relation{rel: rel, params: params})

	return navigator{
		HttpClient: n.HttpClient,
		path:       relations,
		rootUri:    n.rootUri,
	}
}

// url returns the URL of the tip of the follow queue. Will follow the
// usual pattern of requests.
func (n navigator) url() (string, error) {
	url := n.rootUri

	for _, link := range n.path {
		links, err := n.getLinks(url)
		if err != nil {
			return "", err
		}

		if _, ok := links.Items[link.rel]; !ok {
			return "", LinkNotFoundError{link.rel}
		}

		url, err = links.HrefParams(link.rel, link.params)
		if err != nil {
			return "", err
		}

		if url == "" {
			return "", InvalidUrlError{url}
		}

		url, err = makeAbsoluteIfNecessary(url, n.rootUri)
		if err != nil {
			return "", err
		}
	}

	return url, nil
}

// makeAbsoluteIfNecessary takes the current url and the root url, and
// will make the current URL absolute by using the root's Host, Scheme,
// and credentials if current isn't already absolute.
func makeAbsoluteIfNecessary(current, root string) (string, error) {
	currentUri, err := url.Parse(current)
	if err != nil {
		return "", err
	}

	if currentUri.IsAbs() {
		return current, nil
	}

	rootUri, err := url.Parse(root)
	if err != nil {
		return "", err
	}

	currentUri.Scheme = rootUri.Scheme
	currentUri.Host = rootUri.Host
	currentUri.User = rootUri.User

	return currentUri.String(), nil
}

// Get performs a GET request on the tip of the follow queue.
//
// When a navigator is evaluated it will first request the root, then
// request each relation on the queue until it reaches the tip. Once the
// tip is reached it will defer to the calling method. In the case of GET
// the last request will just be returned. For Post it will issue a post
// to the URL of the last relation. Any error along the way will terminate
// the walk and return immediately.
func (n navigator) Get() (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	return n.HttpClient.Do(req)
}

// PostForm performs a POST request on the tip of the follow queue with
// the given form data.
//
// See GET for a note on how the navigator executes requests.
func (n navigator) PostForm(data url.Values) (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("PATCH", url, strings.NewReader(data.Encode()))
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	return n.HttpClient.Do(req)
}

// Patch parforms a PATCH request on the tip of the follow queue with the
// given bodyType and body content.
//
// See GET for a note on how the navigator executes requests.
func (n navigator) Patch(bodyType string, body io.Reader) (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("PATCH", url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", bodyType)

	return n.HttpClient.Do(req)
}

// Post performs a POST request on the tip of the follow queue with the
// given bodyType and body content.
//
// See GET for a note on how the navigator executes requests.
func (n navigator) Post(bodyType string, body io.Reader) (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("POST", url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", bodyType)

	return n.HttpClient.Do(req)
}

// Unmarshal is a shorthand for Get followed by json.Unmarshal. Handles
// closing the response body and unmarshalling the body.
func (n navigator) Unmarshal(v interface{}) error {
	res, err := n.Get()
	if err != nil {
		return err
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}

	return json.Unmarshal(body, &v)
}

func newHalRequest(method, url string, body io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Accept", "application/hal+json, application/json")

	return req, nil
}

// getLinks does a GET on a particular URL and try to deserialise it into
// a HAL links collection.
func (n navigator) getLinks(uri string) (Links, error) {
	req, err := newHalRequest("GET", uri, nil)
	if err != nil {
		return Links{}, err
	}

	res, err := n.HttpClient.Do(req)
	if err != nil {
		return Links{}, err
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return Links{}, err
	}

	var m Links

	if err := json.Unmarshal(body, &m); err != nil {
		return Links{}, err
	}

	return m, nil
}
<commit_msg>Added a Delete method for completeness<commit_after>package halgo

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

// Navigator is a mechanism for navigating HAL-compliant REST APIs. You
// start by creating a Navigator with a base URI, then Follow the links
// exposed by the API until you reach the place where you want to perform
// an action.
//
// For example, to request an API exposed at api.example.com and follow a
// link named products and GET the resulting page you'd do this:
//
//     res, err := Navigator("http://api.example.com").
//       Follow("products").
//       Get()
//
// To do the same thing but POST to the products page, you'd do this:
//
//     res, err := Navigator("http://api.example.com").
//       Follow("products").
//       Post("application/json", someContent)
//
// Multiple links followed in sequence.
//
//     res, err := Navigator("http://api.example.com").
//       Follow("products").
//       Follow("next")
//       Get()
//
// Links can also be expanded with Followf if they are URI templates.
//
//     res, err := Navigator("http://api.example.com").
//       Follow("products").
//       Followf("page", halgo.P{"number": 10})
//       Get()
//
// Navigation of relations is lazy. Requests will only be triggered when
// you execute a method which returns a result. For example, this doesn't
// perform any HTTP requests.
//
//     Navigator("http://api.example.com").
//       Follow("products")
//
// It's only when you add a call to Get, Post, PostForm, Patch, or
// Unmarshal to the end will any requests be triggered.
//
// By default a Navigator will use http.DefaultClient as its mechanism for
// making HTTP requests. If you want to supply your own HttpClient, you
// can assign to nav.HttpClient after creation.
//
//     nav := Navigator("http://api.example.com")
//     nav.HttpClient = MyHttpClient{}
//
// Any Client you supply must implement halgo.HttpClient, which
// http.Client does implicitly. By creating decorators for the HttpClient,
// logging and caching clients are trivial. See LoggingHttpClient for an
// example.
func Navigator(uri string) navigator {
	return navigator{
		rootUri:    uri,
		path:       []relation{},
		HttpClient: http.DefaultClient,
	}
}

// relation is an instruction of a relation to follow and any params to
// expand with when executed.
type relation struct {
	rel    string
	params P
}

// navigator is the API navigator
type navigator struct {
	// HttpClient is used to execute requests. By default it's
	// http.DefaultClient. By decorating a HttpClient instance you can
	// easily write loggers or caching mechanisms.
	HttpClient HttpClient

	// path is the follow queue.
	path []relation

	// rootUri is where the navigation will begin from.
	rootUri string
}

// Follow adds a relation to the follow queue of the navigator.
func (n navigator) Follow(rel string) navigator {
	return n.Followf(rel, nil)
}

// Followf adds a relation to the follow queue of the navigator, with a
// set of parameters to expand on execution.
func (n navigator) Followf(rel string, params P) navigator {
	relations := make([]relation, 0, len(n.path)+1)
	copy(n.path, relations)
	relations = append(relations, relation{rel: rel, params: params})

	return navigator{
		HttpClient: n.HttpClient,
		path:       relations,
		rootUri:    n.rootUri,
	}
}

// url returns the URL of the tip of the follow queue. Will follow the
// usual pattern of requests.
func (n navigator) url() (string, error) {
	url := n.rootUri

	for _, link := range n.path {
		links, err := n.getLinks(url)
		if err != nil {
			return "", err
		}

		if _, ok := links.Items[link.rel]; !ok {
			return "", LinkNotFoundError{link.rel}
		}

		url, err = links.HrefParams(link.rel, link.params)
		if err != nil {
			return "", err
		}

		if url == "" {
			return "", InvalidUrlError{url}
		}

		url, err = makeAbsoluteIfNecessary(url, n.rootUri)
		if err != nil {
			return "", err
		}
	}

	return url, nil
}

// makeAbsoluteIfNecessary takes the current url and the root url, and
// will make the current URL absolute by using the root's Host, Scheme,
// and credentials if current isn't already absolute.
func makeAbsoluteIfNecessary(current, root string) (string, error) {
	currentUri, err := url.Parse(current)
	if err != nil {
		return "", err
	}

	if currentUri.IsAbs() {
		return current, nil
	}

	rootUri, err := url.Parse(root)
	if err != nil {
		return "", err
	}

	currentUri.Scheme = rootUri.Scheme
	currentUri.Host = rootUri.Host
	currentUri.User = rootUri.User

	return currentUri.String(), nil
}

// Get performs a GET request on the tip of the follow queue.
//
// When a navigator is evaluated it will first request the root, then
// request each relation on the queue until it reaches the tip. Once the
// tip is reached it will defer to the calling method. In the case of GET
// the last request will just be returned. For Post it will issue a post
// to the URL of the last relation. Any error along the way will terminate
// the walk and return immediately.
func (n navigator) Get() (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	return n.HttpClient.Do(req)
}

// PostForm performs a POST request on the tip of the follow queue with
// the given form data.
//
// See GET for a note on how the navigator executes requests.
func (n navigator) PostForm(data url.Values) (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("PATCH", url, strings.NewReader(data.Encode()))
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	return n.HttpClient.Do(req)
}

// Patch parforms a PATCH request on the tip of the follow queue with the
// given bodyType and body content.
//
// See GET for a note on how the navigator executes requests.
func (n navigator) Patch(bodyType string, body io.Reader) (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("PATCH", url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", bodyType)

	return n.HttpClient.Do(req)
}

// Post performs a POST request on the tip of the follow queue with the
// given bodyType and body content.
//
// See GET for a note on how the navigator executes requests.
func (n navigator) Post(bodyType string, body io.Reader) (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("POST", url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", bodyType)

	return n.HttpClient.Do(req)
}

// Delete performs a DELETE request on the tip of the follow queue.
//
// See GET for a note on how the navigator executes requests.
func (n navigator) Delete() (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("DELETE", url, nil)
	if err != nil {
		return nil, err
	}

	return n.HttpClient.Do(req)
}

// Unmarshal is a shorthand for Get followed by json.Unmarshal. Handles
// closing the response body and unmarshalling the body.
func (n navigator) Unmarshal(v interface{}) error {
	res, err := n.Get()
	if err != nil {
		return err
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}

	return json.Unmarshal(body, &v)
}

func newHalRequest(method, url string, body io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Accept", "application/hal+json, application/json")

	return req, nil
}

// getLinks does a GET on a particular URL and try to deserialise it into
// a HAL links collection.
func (n navigator) getLinks(uri string) (Links, error) {
	req, err := newHalRequest("GET", uri, nil)
	if err != nil {
		return Links{}, err
	}

	res, err := n.HttpClient.Do(req)
	if err != nil {
		return Links{}, err
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return Links{}, err
	}

	var m Links

	if err := json.Unmarshal(body, &m); err != nil {
		return Links{}, err
	}

	return m, nil
}
<|endoftext|>


<commit_before>package cajun

import "testing"

type parserTest struct {
	name   string
	input  string
	output string
}

var parserTests = []parserTest{
	{"empty", "", ""},
	{"spaces", "   ", "   "},
	{"heading1", "= Level 1 =", "
 Level 1
"},
	{"should not be heading1. no space", "=Level 1 =", "
=Level 1 =
"},
	{"heading2", "== Level 2 ==", "
 Level 2
"},
	{"heading3", "=== Level 3 ===", "
 Level 3
"},
	{"heading4", "==== Level 4 ====", "
 Level 4
"},
	{"heading5", "===== Level 5 =====", "
 Level 5
"},
	{"heading6", "====== Level 6 ======", "
 Level 6
"},
	{"heading6", "====== Level 6 ========", "
 Level 6
"},
	{"heading: should close h1 as h3", "=== Level =", "
 Level
"},
	{"hr", "----", "
"},
	{"hr preceeded by space", "  ----", "
"},
	{"hr preceeded by space, break, then text", "  ----  \n more", "

 more
"},
	{"hr followed by space", "----  ", "
  "},
	{"hr too many dashes", "-----", "
-----
"},
	{"text", `now is the time`, "
now is the time
"},
	{"text with bold", "hello-**blah**-world", "
hello-blah-world
"},
	{"text with italics", "hello-//blah//-world", "
hello-blah-world
"},
	{"text with bad order", "hello-**//blah**//-world", "
hello-blah-world
"},
	{"text with bad order, twice", "hello-**//blah**//-world**//blah**// this is a **test**", "
hello-blah-worldblah this is a test
"},
	{"text with bad order, twice", "hello-**//blah**//**//blah**//", "
hello-blahblah
"},
	{"test closing bold accross line breaks", "close this ** testing a \n    \n bold... more stuff here", "
close this  testing a

 bold... more stuff here
"},

	{"line break", "line \\\\break", "
line
break
"},
	{"unordered list", "* list item\n** child item", "

 list item

 child item

"},
	{"unordered list -", "* list item\n** child item\n* list item", "

 list item

 child item

 list item

"},
	{"unordered list -", "* item1\n** item1.1\n** item1.2\n* item2", "

 item1

 item1.1

 item1.2

 item2

"},
	{"unordered list -", "* item1\n** item1.1\n** item1.2\n* item2\n* item3", "

 item1

 item1.1

 item1.2

 item2

 item3

"},
}

func TestParser(t *testing.T) {
	p := parser{}
	for _, test := range parserTests {
		output, _ := p.Transform(test.input)
		if test.output != output {
			t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, output, test.output)
		}

	}
}
<commit_msg>tests<commit_after>package cajun

import "testing"

type parserTest struct {
	name   string
	input  string
	output string
}

var parserTests = []parserTest{
	{"empty", "", ""},
	{"spaces", "   ", "   "},
	{"heading1", "= Level 1 =", "
 Level 1
"},
	{"should not be heading1. no space", "=Level 1 =", "
=Level 1 =
"},
	{"heading2", "== Level 2 ==", "
 Level 2
"},
	{"heading3", "=== Level 3 ===", "
 Level 3
"},
	{"heading4", "==== Level 4 ====", "
 Level 4
"},
	{"heading5", "===== Level 5 =====", "
 Level 5
"},
	{"heading6", "====== Level 6 ======", "
 Level 6
"},
	{"heading6", "====== Level 6 ========", "
 Level 6
"},
	{"heading: should close h1 as h3", "=== Level =", "
 Level
"},
	{"hr", "----", "
"},
	{"hr preceeded by space", "  ----", "
"},
	{"hr preceeded by space, break, then text", "  ----  \n more", "

 more
"},
	{"hr followed by space", "----  ", "
  "},
	{"hr too many dashes", "-----", "
-----
"},
	{"text", `now is the time`, "
now is the time
"},
	{"text with bold", "hello-**blah**-world", "
hello-blah-world
"},
	{"text with italics", "hello-//blah//-world", "
hello-blah-world
"},
	{"text with bad order", "hello-**//blah**//-world", "
hello-blah-world
"},
	{"text with bad order, twice", "hello-**//blah**//-world**//blah**// this is a **test**", "
hello-blah-worldblah this is a test
"},
	{"text with bad order, twice", "hello-**//blah**//**//blah**//", "
hello-blahblah
"},
	{"test closing bold accross line breaks", "close this ** testing a \n    \n bold... more stuff here", "
close this  testing a

 bold... more stuff here
"},

	{"line break", "line \\\\break", "
line
break
"},
	{"unordered list simple", "* list item\n** child item", "

 list item

 child item

"},
	{"unordered list - one child, in first parent", "* list item\n** child item\n* list item", "

 list item

 child item

 list item

"},
	{"unordered list - two child in first parent", "* item1\n** item1.1\n** item1.2\n* item2", "

 item1

 item1.1

 item1.2

 item2

"},
	{"unordered list - 3 first level, 2 child", "* item1\n** item1.1\n** item1.2\n* item2\n* item3", "

 item1

 item1.1

 item1.2

 item2

 item3

"},
	{"unordered list, ", "* item1\n** item1.1\n** item1.2", "

 item1

 item1.1

 item1.2

"},
	{"unordered list, ", "* item1\n** item1.1\n** item1.2\n** item1.3", "

 item1

 item1.1

 item1.2

 item1.3

"},
	//	{"unordered list - ", "* item1\n** item1.1\n** item1.2\n* item2\n** item2.1\n** item2.2\n*** item2.2.1", "

 item1

 item1.1

 item1.2

 item2

 item2.1

 item2.2

 item2.2.1

"},
}

func TestParser(t *testing.T) {
	p := parser{}
	for _, test := range parserTests {
		output, _ := p.Transform(test.input)
		if test.output != output {
			t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, output, test.output)
		}

	}
}
<|endoftext|>


<commit_before>package gox12

import ()

//type X12PathFinder interface {
//	FindNext(x12Path string, segment Segment) (newpath string, found bool, err error)
//}

type PathFinder func(string, Segment) (string, bool, error)

type EmptyPath struct {
	Path string
}

//func (e *EmptyPath) Run2 PathFinder {
//    return "", true, nil
//}

//func MakeFinder() func() {
//    f := func(x12Path string, segment Segment) {
//    }
//    return f
//}

//func makeFinderFunction() (func(string, Segment) string, bool, error) {
//    return
//}

// this is the method signature
// need to close lookup maps
func findPath(rawpath string, seg Segment) (newpath string, ok bool, err error) {
	return "", true, nil
}
<commit_msg>update matcher func closures<commit_after>package gox12

import ()

//type X12PathFinder interface {
//	FindNext(x12Path string, segment Segment) (newpath string, found bool, err error)
//}

type PathFinder func(string, Segment) (string, bool, error)

type EmptyPath struct {
	Path string
}

// map[start_x12path] []struct {match_func func(seg) bool, newpath string}

//testLookups := map[string]

//func (e *EmptyPath) Run2 PathFinder {
//    return "", true, nil
//}

//func MakeFinder() func() {
//    f := func(x12Path string, segment Segment) {
//    }
//    return f
//}

//func makeFinderFunction() (func(string, Segment) string, bool, error) {
//    return
//}

// this is the method signature
// need to close lookup maps
func findPath(rawpath string, seg Segment) (newpath string, ok bool, err error) {
	return "", true, nil
}

// is the segment "matched"
type segMatcher func(seg Segment) bool

func segmentMatchBySegmentId(segmentId string) segMatcher {
	return func(seg Segment) bool {
		return seg.SegmentId == segmentId
	}
}

func segmentMatchIdByPath(segmentId string, x12path string, id_value string) segMatcher {
	return func(seg Segment) bool {
		v, found, _ := seg.GetValue(x12path)
		return seg.SegmentId == segmentId && found && v == id_value
	}
}

func segmentMatchIdListByPath(segmentId string, x12path string, id_list []string) segMatcher {
	return func(seg Segment) bool {
		v, found, _ := seg.GetValue(x12path)
		x := stringInSlice(v, id_list)
		return seg.SegmentId == segmentId && found && x
	}
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>/*
 * Copyright 2018 The Kythe Authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package graph

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"log"
	"regexp"

	"kythe.io/kythe/go/services/graph"
	"kythe.io/kythe/go/services/xrefs"
	"kythe.io/kythe/go/serving/graph/columnar"
	"kythe.io/kythe/go/storage/keyvalue"
	"kythe.io/kythe/go/storage/table"
	"kythe.io/kythe/go/util/keys"
	"kythe.io/kythe/go/util/kytheuri"
	"kythe.io/kythe/go/util/schema"
	"kythe.io/kythe/go/util/schema/facts"

	"bitbucket.org/creachadair/stringset"
	"google.golang.org/protobuf/proto"

	cpb "kythe.io/kythe/proto/common_go_proto"
	gpb "kythe.io/kythe/proto/graph_go_proto"
	gspb "kythe.io/kythe/proto/graph_serving_go_proto"
	scpb "kythe.io/kythe/proto/schema_go_proto"
)

// ColumnarTableKeyMarker is stored within a Kythe columnar table to
// differentiate it from the legacy combined table format.
const ColumnarTableKeyMarker = "kythe:columnar"

// NewService returns an graph.Service backed by the given table.  The format of
// the table with be automatically detected.
func NewService(ctx context.Context, t keyvalue.DB) graph.Service {
	_, err := t.Get(ctx, []byte(ColumnarTableKeyMarker), nil)
	if err == nil {
		log.Println("WARNING: detected a experimental columnar graph table")
		return NewColumnarTable(t)
	}
	return NewCombinedTable(&table.KVProto{t})
}

// NewColumnarTable returns a table for the given columnar graph lookup table.
func NewColumnarTable(t keyvalue.DB) *ColumnarTable { return &ColumnarTable{t} }

// ColumnarTable implements an graph.Service backed by a columnar serving table.
type ColumnarTable struct{ keyvalue.DB }

// Nodes implements part of the graph.Service interface.
func (c *ColumnarTable) Nodes(ctx context.Context, req *gpb.NodesRequest) (*gpb.NodesReply, error) {
	reply := &gpb.NodesReply{Nodes: make(map[string]*cpb.NodeInfo, len(req.Ticket))}

	filters := req.Filter
	if len(filters) == 0 {
		filters = append(filters, "**")
	}
	patterns := xrefs.ConvertFilters(filters)

	for _, ticket := range req.Ticket {
		srcURI, err := kytheuri.Parse(ticket)
		if err != nil {
			return nil, err
		}

		src := srcURI.VName()
		key, err := keys.Append(columnar.EdgesKeyPrefix, src)
		if err != nil {
			return nil, err
		}

		val, err := c.DB.Get(ctx, key, &keyvalue.Options{LargeRead: true})
		if err == io.EOF {
			continue
		} else if err != nil {
			return nil, err
		}

		var idx gspb.Edges_Index
		if err := proto.Unmarshal(val, &idx); err != nil {
			return nil, fmt.Errorf("error decoding index: %v", err)
		}

		if info := filterNode(patterns, idx.Node); len(info.Facts) > 0 {
			reply.Nodes[ticket] = info
		}
	}

	if len(reply.Nodes) == 0 {
		reply.Nodes = nil
	}

	return reply, nil
}

// Edges implements part of the graph.Service interface.
func (c *ColumnarTable) Edges(ctx context.Context, req *gpb.EdgesRequest) (*gpb.EdgesReply, error) {
	// TODO(schroederc): implement edge paging
	reply := &gpb.EdgesReply{
		EdgeSets: make(map[string]*gpb.EdgeSet, len(req.Ticket)),
		Nodes:    make(map[string]*cpb.NodeInfo),

		// TODO(schroederc): TotalEdgesByKind: make(map[string]int64),
	}
	patterns := xrefs.ConvertFilters(req.Filter)
	allowedKinds := stringset.New(req.Kind...)

	for _, ticket := range req.Ticket {
		srcURI, err := kytheuri.Parse(ticket)
		if err != nil {
			return nil, err
		}

		src := srcURI.VName()
		prefix, err := keys.Append(columnar.EdgesKeyPrefix, src)
		if err != nil {
			return nil, err
		}

		it, err := c.DB.ScanPrefix(ctx, prefix, &keyvalue.Options{LargeRead: true})
		if err != nil {
			return nil, err
		}

		k, val, err := it.Next()
		if err == io.EOF || !bytes.Equal(k, prefix) {
			continue
		} else if err != nil {
			return nil, err
		}

		// Decode Edges Index
		var idx gspb.Edges_Index
		if err := proto.Unmarshal(val, &idx); err != nil {
			return nil, fmt.Errorf("error decoding index: %v", err)
		}
		if len(patterns) > 0 {
			if info := filterNode(patterns, idx.Node); len(info.Facts) > 0 {
				reply.Nodes[ticket] = info
			}
		}

		edges := &gpb.EdgeSet{Groups: make(map[string]*gpb.EdgeSet_Group)}
		reply.EdgeSets[ticket] = edges
		targets := stringset.New()

		// Main loop to scan over each columnar kv entry.
		for {
			k, val, err := it.Next()
			if err == io.EOF {
				break
			} else if err != nil {
				return nil, err
			}
			key := string(k[len(prefix):])

			// TODO(schroederc): only parse needed entries
			e, err := columnar.DecodeEdgesEntry(src, key, val)
			if err != nil {
				return nil, err
			}

			switch e := e.Entry.(type) {
			case *gspb.Edges_Edge_:
				edge := e.Edge

				kind := edge.GetGenericKind()
				if kind == "" {
					kind = schema.EdgeKindString(edge.GetKytheKind())
				}
				if edge.Reverse {
					kind = "%" + kind
				}

				if len(allowedKinds) != 0 && !allowedKinds.Contains(kind) {
					continue
				}

				target := kytheuri.ToString(edge.Target)
				targets.Add(target)

				g := edges.Groups[kind]
				if g == nil {
					g = &gpb.EdgeSet_Group{}
					edges.Groups[kind] = g
				}
				g.Edge = append(g.Edge, &gpb.EdgeSet_Group_Edge{
					TargetTicket: target,
					Ordinal:      edge.Ordinal,
				})
			case *gspb.Edges_Target_:
				if len(patterns) == 0 || len(targets) == 0 {
					break
				}

				target := e.Target
				ticket := kytheuri.ToString(target.Node.Source)
				if targets.Contains(ticket) {
					if info := filterNode(patterns, target.Node); len(info.Facts) > 0 {
						reply.Nodes[ticket] = info
					}
				}
			default:
				return nil, fmt.Errorf("unknown Edges entry: %T", e)
			}
		}

		if len(edges.Groups) == 0 {
			delete(reply.EdgeSets, ticket)
		}
	}

	if len(reply.EdgeSets) == 0 {
		reply.EdgeSets = nil
	}
	if len(reply.Nodes) == 0 {
		reply.Nodes = nil
	}

	return reply, nil
}

func filterNode(patterns []*regexp.Regexp, n *scpb.Node) *cpb.NodeInfo {
	c := &cpb.NodeInfo{Facts: make(map[string][]byte, len(n.Fact))}
	for _, f := range n.Fact {
		name := schema.GetFactName(f)
		if xrefs.MatchesAny(name, patterns) {
			c.Facts[name] = f.Value
		}
	}
	if kind := schema.GetNodeKind(n); kind != "" && xrefs.MatchesAny(facts.NodeKind, patterns) {
		c.Facts[facts.NodeKind] = []byte(kind)
	}
	if subkind := schema.GetSubkind(n); subkind != "" && xrefs.MatchesAny(facts.Subkind, patterns) {
		c.Facts[facts.Subkind] = []byte(subkind)
	}
	return c
}
<commit_msg>Close the graph columnar db iterator. (#4859)<commit_after>/*
 * Copyright 2018 The Kythe Authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package graph

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"log"
	"regexp"

	"kythe.io/kythe/go/services/graph"
	"kythe.io/kythe/go/services/xrefs"
	"kythe.io/kythe/go/serving/graph/columnar"
	"kythe.io/kythe/go/storage/keyvalue"
	"kythe.io/kythe/go/storage/table"
	"kythe.io/kythe/go/util/keys"
	"kythe.io/kythe/go/util/kytheuri"
	"kythe.io/kythe/go/util/schema"
	"kythe.io/kythe/go/util/schema/facts"

	"bitbucket.org/creachadair/stringset"
	"google.golang.org/protobuf/proto"

	cpb "kythe.io/kythe/proto/common_go_proto"
	gpb "kythe.io/kythe/proto/graph_go_proto"
	gspb "kythe.io/kythe/proto/graph_serving_go_proto"
	scpb "kythe.io/kythe/proto/schema_go_proto"
)

// ColumnarTableKeyMarker is stored within a Kythe columnar table to
// differentiate it from the legacy combined table format.
const ColumnarTableKeyMarker = "kythe:columnar"

// NewService returns an graph.Service backed by the given table.  The format of
// the table with be automatically detected.
func NewService(ctx context.Context, t keyvalue.DB) graph.Service {
	_, err := t.Get(ctx, []byte(ColumnarTableKeyMarker), nil)
	if err == nil {
		log.Println("WARNING: detected a experimental columnar graph table")
		return NewColumnarTable(t)
	}
	return NewCombinedTable(&table.KVProto{t})
}

// NewColumnarTable returns a table for the given columnar graph lookup table.
func NewColumnarTable(t keyvalue.DB) *ColumnarTable { return &ColumnarTable{t} }

// ColumnarTable implements an graph.Service backed by a columnar serving table.
type ColumnarTable struct{ keyvalue.DB }

// Nodes implements part of the graph.Service interface.
func (c *ColumnarTable) Nodes(ctx context.Context, req *gpb.NodesRequest) (*gpb.NodesReply, error) {
	reply := &gpb.NodesReply{Nodes: make(map[string]*cpb.NodeInfo, len(req.Ticket))}

	filters := req.Filter
	if len(filters) == 0 {
		filters = append(filters, "**")
	}
	patterns := xrefs.ConvertFilters(filters)

	for _, ticket := range req.Ticket {
		srcURI, err := kytheuri.Parse(ticket)
		if err != nil {
			return nil, err
		}

		src := srcURI.VName()
		key, err := keys.Append(columnar.EdgesKeyPrefix, src)
		if err != nil {
			return nil, err
		}

		val, err := c.DB.Get(ctx, key, &keyvalue.Options{LargeRead: true})
		if err == io.EOF {
			continue
		} else if err != nil {
			return nil, err
		}

		var idx gspb.Edges_Index
		if err := proto.Unmarshal(val, &idx); err != nil {
			return nil, fmt.Errorf("error decoding index: %v", err)
		}

		if info := filterNode(patterns, idx.Node); len(info.Facts) > 0 {
			reply.Nodes[ticket] = info
		}
	}

	if len(reply.Nodes) == 0 {
		reply.Nodes = nil
	}

	return reply, nil
}

// processTicket loads values associated with the search ticket and adds them to the reply.
func (c *ColumnarTable) processTicket(ctx context.Context, ticket string, patterns []*regexp.Regexp, allowedKinds stringset.Set, reply *gpb.EdgesReply) error {
	srcURI, err := kytheuri.Parse(ticket)
	if err != nil {
		return err
	}

	src := srcURI.VName()
	prefix, err := keys.Append(columnar.EdgesKeyPrefix, src)
	if err != nil {
		return err
	}

	it, err := c.DB.ScanPrefix(ctx, prefix, &keyvalue.Options{LargeRead: true})
	defer it.Close()
	if err != nil {
		return err
	}

	k, val, err := it.Next()
	if err == io.EOF || !bytes.Equal(k, prefix) {
		return nil
	} else if err != nil {
		return err
	}

	// Decode Edges Index
	var idx gspb.Edges_Index
	if err := proto.Unmarshal(val, &idx); err != nil {
		return fmt.Errorf("error decoding index: %v", err)
	}
	if len(patterns) > 0 {
		if info := filterNode(patterns, idx.Node); len(info.Facts) > 0 {
			reply.Nodes[ticket] = info
		}
	}

	edges := &gpb.EdgeSet{Groups: make(map[string]*gpb.EdgeSet_Group)}
	reply.EdgeSets[ticket] = edges
	targets := stringset.New()

	// Main loop to scan over each columnar kv entry.
	for {
		k, val, err := it.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		key := string(k[len(prefix):])

		// TODO(schroederc): only parse needed entries
		e, err := columnar.DecodeEdgesEntry(src, key, val)
		if err != nil {
			return err
		}

		switch e := e.Entry.(type) {
		case *gspb.Edges_Edge_:
			edge := e.Edge

			kind := edge.GetGenericKind()
			if kind == "" {
				kind = schema.EdgeKindString(edge.GetKytheKind())
			}
			if edge.Reverse {
				kind = "%" + kind
			}

			if len(allowedKinds) != 0 && !allowedKinds.Contains(kind) {
				continue
			}

			target := kytheuri.ToString(edge.Target)
			targets.Add(target)

			g := edges.Groups[kind]
			if g == nil {
				g = &gpb.EdgeSet_Group{}
				edges.Groups[kind] = g
			}
			g.Edge = append(g.Edge, &gpb.EdgeSet_Group_Edge{
				TargetTicket: target,
				Ordinal:      edge.Ordinal,
			})
		case *gspb.Edges_Target_:
			if len(patterns) == 0 || len(targets) == 0 {
				break
			}

			target := e.Target
			ticket := kytheuri.ToString(target.Node.Source)
			if targets.Contains(ticket) {
				if info := filterNode(patterns, target.Node); len(info.Facts) > 0 {
					reply.Nodes[ticket] = info
				}
			}
		default:
			return fmt.Errorf("unknown Edges entry: %T", e)
		}
	}

	if len(edges.Groups) == 0 {
		delete(reply.EdgeSets, ticket)
	}
	return nil
}

// Edges implements part of the graph.Service interface.
func (c *ColumnarTable) Edges(ctx context.Context, req *gpb.EdgesRequest) (*gpb.EdgesReply, error) {
	// TODO(schroederc): implement edge paging
	reply := &gpb.EdgesReply{
		EdgeSets: make(map[string]*gpb.EdgeSet, len(req.Ticket)),
		Nodes:    make(map[string]*cpb.NodeInfo),

		// TODO(schroederc): TotalEdgesByKind: make(map[string]int64),
	}
	patterns := xrefs.ConvertFilters(req.Filter)
	allowedKinds := stringset.New(req.Kind...)

	for _, ticket := range req.Ticket {
		err := c.processTicket(ctx, ticket, patterns, allowedKinds, reply)
		if err != nil {
			return nil, err
		}
	}

	if len(reply.EdgeSets) == 0 {
		reply.EdgeSets = nil
	}
	if len(reply.Nodes) == 0 {
		reply.Nodes = nil
	}

	return reply, nil
}

func filterNode(patterns []*regexp.Regexp, n *scpb.Node) *cpb.NodeInfo {
	c := &cpb.NodeInfo{Facts: make(map[string][]byte, len(n.Fact))}
	for _, f := range n.Fact {
		name := schema.GetFactName(f)
		if xrefs.MatchesAny(name, patterns) {
			c.Facts[name] = f.Value
		}
	}
	if kind := schema.GetNodeKind(n); kind != "" && xrefs.MatchesAny(facts.NodeKind, patterns) {
		c.Facts[facts.NodeKind] = []byte(kind)
	}
	if subkind := schema.GetSubkind(n); subkind != "" && xrefs.MatchesAny(facts.Subkind, patterns) {
		c.Facts[facts.Subkind] = []byte(subkind)
	}
	return c
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/igneous-systems/logit"
	"gopkg.in/alecthomas/kingpin.v1"

	"github.com/igneous-systems/pickett"
	"github.com/igneous-systems/pickett/io"
)

var (
	PickettVersion = "0.0.1"

	app = kingpin.New("Pickett", "Make for the docker world.")

	// Global flags
	debug      = app.Flag("debug", "Enable debug mode.").Short('d').Bool()
	configFile = app.Flag("configFile", "Config file.").Short('f').Default("Pickett.json").String()

	// Actions
	run     = app.Command("run", "Runs a specific node in a topology, including all depedencies.")
	runTopo = run.Arg("topo", "Topo node.").Required().String()

	status        = app.Command("status", "Shows the status of all the known buildable tags and/or runnable nodes.")
	statusTargets = status.Arg("targets", "Tags / Nodes").Strings()

	build     = app.Command("build", "Build all tags or specified tags.")
	buildTags = build.Arg("tags", "Tags").Strings()

	stop      = app.Command("stop", "Stop all or a specific node.")
	stopNodes = stop.Arg("topology.nodes", "Topology Nodes").Strings()

	drop      = app.Command("drop", "Stop and delete all or specific node.")
	dropNodes = drop.Arg("topology.nodes", "Topology Nodes").Strings()

	wipe     = app.Command("wipe", "Delete all or specified tag (force rebuild next time).")
	wipeTags = wipe.Arg("tags", "Tags").Strings()

	ps      = app.Command("ps", "Give 'docker ps' like output of running topologies.")
	psNodes = ps.Arg("topology.nodes", "Topology Nodes").Strings()

	inject     = app.Command("inject", "Run the given command in the given topology node")
	injectNode = inject.Arg("topology.node", "Topology Node").Required().String()
	injectCmd  = inject.Arg("Cmd", "Node").Required().Strings()
)

func contains(s []string, target string) bool {
	for _, candidate := range s {
		if candidate == target {
			return true
		}
	}
	return false
}

func makeIOObjects(path string) (io.Helper, io.DockerCli, io.EtcdClient, io.VirtualBox, error) {
	helper, err := io.NewHelper(path)
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("can't read %s: %v", path, err)
	}
	cli, err := io.NewDockerCli()
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("failed to connect to docker server, maybe its not running? %v", err)
	}
	etcd, err := io.NewEtcdClient()
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("failed to connect to etcd, maybe its not running? %v", err)
	}
	vbox, err := io.NewVirtualBox()
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("failed to run vboxmanage: %v", err)
	}
	return helper, cli, etcd, vbox, nil
}

// trueMain is the entry point of the program with the targets filled in
// and a working helper.
func trueMain(targets []string, helper io.Helper, cli io.DockerCli, etcd io.EtcdClient, vbox io.VirtualBox) error {
	reader := helper.ConfigReader()
	config, err := pickett.NewConfig(reader, helper, cli, etcd, vbox)
	if err != nil {
		return fmt.Errorf("can't understand config file %s: %v", helper.ConfigFile(), err)
	}
	buildables, runnables := config.EntryPoints()
	run := false
	runTarget := ""

	// if you don't tell us what to build, we build everything with no outgoing
	// edges, the "root" of a backchain
	if len(targets) == 0 {
		targets = buildables
	} else {
		//if you do tell us, we need know if it's runnable
		for _, t := range targets {
			if contains(buildables, t) {
				continue
			}
			if contains(runnables, t) {
				if run {
					return fmt.Errorf("can only run one target (%s and %s both runnable)", runTarget, t)
				}
				run = true
				runTarget = t
				continue
			}
			return fmt.Errorf("don't know anything about target %s", t)
		}
	}
	for _, target := range targets {
		if target == runTarget {
			continue
		}
		err := config.Build(target)
		if err != nil {
			return fmt.Errorf("an error occurred while building target '%v': %v", target, err)
		}
	}
	if runTarget != "" {
		err = config.Execute(runTarget)
		if err != nil {
			return fmt.Errorf("an error occurred while running target '%v': %v", runTarget, err)
		}
	}
	return nil
}

var flog = logit.NewNestedLoggerFromCaller(logit.Global)

func main() {
	os.Exit(wrappedMain())
}

// Wrapped to make os.Exit work well with logit
func wrappedMain() int {

	kingpin.Version(PickettVersion)

	action := kingpin.MustParse(app.Parse(os.Args[1:]))

	var logFilterLvl logit.Level
	if *debug {
		logFilterLvl = logit.DEBUG
	} else {
		logFilterLvl = logit.INFO
	}
	logit.Global.ModifyFilterLvl("stdout", logFilterLvl, nil, nil)
	defer logit.Flush(-1)

	wd, err := os.Getwd()
	if err != nil {
		panic("can't get working directory!")
	}

	if os.Getenv("DOCKER_HOST") == "" {
		fmt.Fprintf(os.Stderr, "DOCKER_HOST not set; suggest DOCKER_HOST=tcp://:2375 (for local launcher)\n")
		return 1
	}

	_, err = os.Open(filepath.Join(wd, *configFile))
	if err != nil {
		fmt.Fprintf(os.Stderr, "./%s not found (cwd: %s)\n", *configFile, wd)
		return 1
	}

	helper, docker, etcd, vbox, err := makeIOObjects(filepath.Join(wd, *configFile))
	if err != nil {
		flog.Errorf("%v", err)
		return 1
	}
	reader := helper.ConfigReader()
	config, err := pickett.NewConfig(reader, helper, docker, etcd, vbox)
	if err != nil {
		flog.Errorf("Can't understand config file %s: %v", err.Error(), helper.ConfigFile())
		return 1
	}

	switch action {
	case "run":
		err = pickett.CmdRun(*runTopo, config)
	case "build":
		err = pickett.CmdBuild(*buildTags, config)
	case "status":
		err = pickett.CmdStatus(*statusTargets, config)
	case "stop":
		err = pickett.CmdStop(*stopNodes, config)
	case "drop":
		err = pickett.CmdDrop(*dropNodes, config)
	case "wipe":
		err = pickett.CmdWipe(*wipeTags, config)
	case "ps":
		err = pickett.CmdPs(*psNodes, config)
	case "inject":
		err = pickett.CmdInject(*injectNode, *injectCmd, config)
	default:
		app.Usage(os.Stderr)
		return 1
	}

	if err != nil {
		flog.Errorf("%s: %v", action, err)
		return 1
	}
	return 0
}
<commit_msg>remove trueMain (it's not being called)<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/igneous-systems/logit"
	"gopkg.in/alecthomas/kingpin.v1"

	"github.com/igneous-systems/pickett"
	"github.com/igneous-systems/pickett/io"
)

var (
	PickettVersion = "0.0.1"

	app = kingpin.New("Pickett", "Make for the docker world.")

	// Global flags
	debug      = app.Flag("debug", "Enable debug mode.").Short('d').Bool()
	configFile = app.Flag("configFile", "Config file.").Short('f').Default("Pickett.json").String()

	// Actions
	run     = app.Command("run", "Runs a specific node in a topology, including all depedencies.")
	runTopo = run.Arg("topo", "Topo node.").Required().String()

	status        = app.Command("status", "Shows the status of all the known buildable tags and/or runnable nodes.")
	statusTargets = status.Arg("targets", "Tags / Nodes").Strings()

	build     = app.Command("build", "Build all tags or specified tags.")
	buildTags = build.Arg("tags", "Tags").Strings()

	stop      = app.Command("stop", "Stop all or a specific node.")
	stopNodes = stop.Arg("topology.nodes", "Topology Nodes").Strings()

	drop      = app.Command("drop", "Stop and delete all or specific node.")
	dropNodes = drop.Arg("topology.nodes", "Topology Nodes").Strings()

	wipe     = app.Command("wipe", "Delete all or specified tag (force rebuild next time).")
	wipeTags = wipe.Arg("tags", "Tags").Strings()

	ps      = app.Command("ps", "Give 'docker ps' like output of running topologies.")
	psNodes = ps.Arg("topology.nodes", "Topology Nodes").Strings()

	inject     = app.Command("inject", "Run the given command in the given topology node")
	injectNode = inject.Arg("topology.node", "Topology Node").Required().String()
	injectCmd  = inject.Arg("Cmd", "Node").Required().Strings()
)

func contains(s []string, target string) bool {
	for _, candidate := range s {
		if candidate == target {
			return true
		}
	}
	return false
}

func makeIOObjects(path string) (io.Helper, io.DockerCli, io.EtcdClient, io.VirtualBox, error) {
	helper, err := io.NewHelper(path)
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("can't read %s: %v", path, err)
	}
	cli, err := io.NewDockerCli()
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("failed to connect to docker server, maybe its not running? %v", err)
	}
	etcd, err := io.NewEtcdClient()
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("failed to connect to etcd, maybe its not running? %v", err)
	}
	vbox, err := io.NewVirtualBox()
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("failed to run vboxmanage: %v", err)
	}
	return helper, cli, etcd, vbox, nil
}

var flog = logit.NewNestedLoggerFromCaller(logit.Global)

func main() {
	os.Exit(wrappedMain())
}

// Wrapped to make os.Exit work well with logit
func wrappedMain() int {

	kingpin.Version(PickettVersion)

	action := kingpin.MustParse(app.Parse(os.Args[1:]))

	var logFilterLvl logit.Level
	if *debug {
		logFilterLvl = logit.DEBUG
	} else {
		logFilterLvl = logit.INFO
	}
	logit.Global.ModifyFilterLvl("stdout", logFilterLvl, nil, nil)
	defer logit.Flush(-1)

	wd, err := os.Getwd()
	if err != nil {
		panic("can't get working directory!")
	}

	if os.Getenv("DOCKER_HOST") == "" {
		fmt.Fprintf(os.Stderr, "DOCKER_HOST not set; suggest DOCKER_HOST=tcp://:2375 (for local launcher)\n")
		return 1
	}

	_, err = os.Open(filepath.Join(wd, *configFile))
	if err != nil {
		fmt.Fprintf(os.Stderr, "./%s not found (cwd: %s)\n", *configFile, wd)
		return 1
	}

	helper, docker, etcd, vbox, err := makeIOObjects(filepath.Join(wd, *configFile))
	if err != nil {
		flog.Errorf("%v", err)
		return 1
	}
	reader := helper.ConfigReader()
	config, err := pickett.NewConfig(reader, helper, docker, etcd, vbox)
	if err != nil {
		flog.Errorf("Can't understand config file %s: %v", err.Error(), helper.ConfigFile())
		return 1
	}

	switch action {
	case "run":
		err = pickett.CmdRun(*runTopo, config)
	case "build":
		err = pickett.CmdBuild(*buildTags, config)
	case "status":
		err = pickett.CmdStatus(*statusTargets, config)
	case "stop":
		err = pickett.CmdStop(*stopNodes, config)
	case "drop":
		err = pickett.CmdDrop(*dropNodes, config)
	case "wipe":
		err = pickett.CmdWipe(*wipeTags, config)
	case "ps":
		err = pickett.CmdPs(*psNodes, config)
	case "inject":
		err = pickett.CmdInject(*injectNode, *injectCmd, config)
	default:
		app.Usage(os.Stderr)
		return 1
	}

	if err != nil {
		flog.Errorf("%s: %v", action, err)
		return 1
	}
	return 0
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"log"
	"net"
	"strconv"
	"strings"
)

var port int

func main() {
	flag.IntVar(&port, "port", 8000, "the port on which to listen")
	flag.Parse()

	log.Printf("starting on %d\n", port)
	server, err := net.Listen("tcp", ":"+strconv.Itoa(port))
	if server == nil {
		log.Fatal(err)
	}
	conns := clientConns(server)
	for {
		go handleConn(<-conns)
	}
}

func clientConns(listener net.Listener) chan net.Conn {
	ch := make(chan net.Conn)
	i := 0
	go func() {
		for {
			client, err := listener.Accept()
			if client == nil {
				log.Fatal(err)
				continue
			}
			i++
			log.Printf("%d: %v <-> %v\n", i, client.LocalAddr(), client.RemoteAddr())
			ch <- client
		}
	}()
	return ch
}

func handleConn(client net.Conn) {
	b := bufio.NewReader(client)
	for {
		line, err := b.ReadString('\n')
		if err != nil { // EOF, or worse
			break
		}
		log.Println(strings.TrimSpace(line))
	}
}
<commit_msg>use pointer<commit_after>package main

import (
	"bufio"
	"flag"
	"log"
	"net"
	"strconv"
	"strings"
)

func main() {
	var port = flag.Int("port", 8000, "the port on which to listen")
	flag.Parse()

	log.Printf("starting on %d\n", *port)
	server, err := net.Listen("tcp", ":"+strconv.Itoa(*port))
	if server == nil {
		log.Fatal(err)
	}
	conns := clientConns(server)
	for {
		go handleConn(<-conns)
	}
}

func clientConns(listener net.Listener) chan net.Conn {
	ch := make(chan net.Conn)
	i := 0
	go func() {
		for {
			client, err := listener.Accept()
			if client == nil {
				log.Fatal(err)
				continue
			}
			i++
			log.Printf("%d: %v <-> %v\n", i, client.LocalAddr(), client.RemoteAddr())
			ch <- client
		}
	}()
	return ch
}

func handleConn(client net.Conn) {
	b := bufio.NewReader(client)
	for {
		line, err := b.ReadString('\n')
		if err != nil { // EOF, or worse
			break
		}
		log.Println(strings.TrimSpace(line))
	}
}
<|endoftext|>


<commit_before>// $G $F.go && $L $F.$A && ./$A.out

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

type T struct { i int; f float; s string; next *T }

type R struct { num int }

func itor(a int) *R {
	r := new(R);
	r.num = a;
	return r;
}

func eq(a []*R) {
	for i := 0; i < len(a); i++ {
		if a[i].num != i { panic("bad") }
	}
}

type P struct { a, b int };
func NewP(a, b int) *P {
	return &P{a, b}
}

func main() {
	var t T;
	t = T{0, 7.2, "hi", &t};

	var tp *T;
	tp = &T{0, 7.2, "hi", &t};

	a1 := []int{1,2,3};
	if len(a1) != 3 { panic("a1") }
	a2 := [10]int{1,2,3};
	if len(a2) != 10 || cap(a2) != 10 { panic("a2") }
	//a3 := [10]int{1,2,3,};  // BUG: trailing commas not allowed
	//if len(a3) != 10 || a2[3] != 0 { panic("a3") }

	var oai []int;
	oai = &[]int{1,2,3};
	if len(oai) != 3 { panic("oai") }

	at := []*T{&t, &t, &t};
	if len(at) != 3 { panic("at") }

	c := make(chan int);
	ac := []chan int{c, c, c};
	if len(ac) != 3 { panic("ac") }

	aat := [][len(at)]*T{at, at};
	if len(aat) != 2 || len(aat[1]) != 3 { panic("at") }

	s := string([]byte{'h', 'e', 'l', 'l', 'o'});
	if s != "hello" { panic("s") }

	m := map[string]float{"one":1.0, "two":2.0, "pi":22./7.};
	if len(m) != 3 { panic("m") }

	eq(&[]*R{itor(0), itor(1), itor(2), itor(3), itor(4), itor(5)});

	p1 := NewP(1, 2);
	p2 := NewP(1, 2);
	if p1 == p2 { panic("NewP") }
}
<commit_msg>18 tests are behaving incorrectly no more surprises - all caught up<commit_after>// $G $F.go && $L $F.$A && ./$A.out

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

type T struct { i int; f float; s string; next *T }

type R struct { num int }

func itor(a int) *R {
	r := new(R);
	r.num = a;
	return r;
}

func eq(a []*R) {
	for i := 0; i < len(a); i++ {
		if a[i].num != i { panic("bad") }
	}
}

type P struct { a, b int };
func NewP(a, b int) *P {
	return &P{a, b}
}

func main() {
	var t T;
	t = T{0, 7.2, "hi", &t};

	var tp *T;
	tp = &T{0, 7.2, "hi", &t};

	a1 := []int{1,2,3};
	if len(a1) != 3 { panic("a1") }
	a2 := [10]int{1,2,3};
	if len(a2) != 10 || cap(a2) != 10 { panic("a2") }
	//a3 := [10]int{1,2,3,};  // BUG: trailing commas not allowed
	//if len(a3) != 10 || a2[3] != 0 { panic("a3") }

	var oai []int;
	oai = []int{1,2,3};
	if len(oai) != 3 { panic("oai") }

	at := [...]*T{&t, &t, &t};
	if len(at) != 3 { panic("at") }

	c := make(chan int);
	ac := []chan int{c, c, c};
	if len(ac) != 3 { panic("ac") }

	aat := [][len(at)]*T{at, at};
	if len(aat) != 2 || len(aat[1]) != 3 { panic("aat") }

	s := string([]byte{'h', 'e', 'l', 'l', 'o'});
	if s != "hello" { panic("s") }

	m := map[string]float{"one":1.0, "two":2.0, "pi":22./7.};
	if len(m) != 3 { panic("m") }

	eq([]*R{itor(0), itor(1), itor(2), itor(3), itor(4), itor(5)});

	p1 := NewP(1, 2);
	p2 := NewP(1, 2);
	if p1 == p2 { panic("NewP") }
}
<|endoftext|>


<commit_before>package apps

import (
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/cozy/cozy-stack/pkg/vfs"
	gitFS "gopkg.in/src-d/go-billy.v2"
	git "gopkg.in/src-d/go-git.v4"
	gitPlumbing "gopkg.in/src-d/go-git.v4/plumbing"
	gitObject "gopkg.in/src-d/go-git.v4/plumbing/object"
	gitStorage "gopkg.in/src-d/go-git.v4/storage/filesystem"
)

// ghURLRegex is used to identify github
var ghURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)

const ghRawManifestURL = "https://raw.githubusercontent.com/%s/%s/%s/%s"

// glURLRegex is used to identify gitlab
var glURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)

const glRawManifestURL = "https://%s/%s/%s/raw/%s/%s"

type gitFetcher struct {
	fs          vfs.VFS
	manFilename string
}

func newGitFetcher(fs vfs.VFS, manFilename string) *gitFetcher {
	return &gitFetcher{fs: fs, manFilename: manFilename}
}

var manifestClient = &http.Client{
	Timeout: 60 * time.Second,
}

func isGithub(src *url.URL) bool {
	return src.Host == "github.com"
}

func isGitlab(src *url.URL) bool {
	return src.Host == "framagit.org" || strings.Contains(src.Host, "gitlab")
}

func (g *gitFetcher) FetchManifest(src *url.URL) (io.ReadCloser, error) {
	var err error

	var u string
	if isGithub(src) {
		u, err = resolveGithubURL(src, g.manFilename)
	} else if isGitlab(src) {
		u, err = resolveGitlabURL(src, g.manFilename)
	} else {
		u, err = resolveManifestURL(src, g.manFilename)
	}
	if err != nil {
		return nil, err
	}

	res, err := manifestClient.Get(u)
	if err != nil || res.StatusCode != 200 {
		return nil, ErrManifestNotReachable
	}

	return res.Body, nil
}

func (g *gitFetcher) Fetch(src *url.URL, baseDir *vfs.DirDoc) error {
	log.Debugf("[git] Fetch %s", src.String())
	fs := g.fs

	gitDirName := path.Join(baseDir.Fullpath, ".git")
	gitDir, err := fs.DirByPath(gitDirName)
	if err == nil {
		return g.pull(baseDir, gitDir, src)
	}
	if !os.IsNotExist(err) {
		return err
	}
	gitDir, err = vfs.Mkdir(fs, gitDirName, nil)
	if err != nil {
		return err
	}
	return g.clone(baseDir, gitDir, src)
}

func getGitBranch(src *url.URL) string {
	if src.Fragment != "" {
		return "refs/heads/" + src.Fragment
	}
	return "HEAD"
}

func getWebBranch(src *url.URL) string {
	if src.Fragment != "" {
		return src.Fragment
	}
	return "HEAD"
}

// clone creates a new bare git repository and install all the files of the
// last commit in the application tree.
func (g *gitFetcher) clone(baseDir, gitDir *vfs.DirDoc, src *url.URL) error {
	fs := g.fs

	storage, err := gitStorage.NewStorage(newGFS(fs, gitDir))
	if err != nil {
		return err
	}

	// XXX Gitlab doesn't support the git protocol
	if isGitlab(src) {
		src.Scheme = "https"
	}

	branch := getGitBranch(src)
	log.Debugf("[git] Clone %s %s", src.String(), branch)

	rep, err := git.Clone(storage, nil, &git.CloneOptions{
		URL:           src.String(),
		Depth:         1,
		SingleBranch:  true,
		ReferenceName: gitPlumbing.ReferenceName(branch),
	})
	if err != nil {
		return err
	}

	return g.copyFiles(baseDir, rep)
}

// pull will fetch the latest objects from the default remote and if updates
// are available, it will update the application tree files.
func (g *gitFetcher) pull(baseDir, gitDir *vfs.DirDoc, src *url.URL) error {
	fs := g.fs

	storage, err := gitStorage.NewStorage(newGFS(fs, gitDir))
	if err != nil {
		return err
	}

	rep, err := git.Open(storage, nil)
	if err != nil {
		return err
	}

	branch := getGitBranch(src)
	log.Debugf("[git] Pull %s %s", src.String(), branch)

	err = rep.Pull(&git.PullOptions{
		SingleBranch:  true,
		ReferenceName: gitPlumbing.ReferenceName(branch),
	})
	if err == git.NoErrAlreadyUpToDate {
		return nil
	}
	if err != nil {
		return err
	}

	iter := fs.DirIterator(baseDir, nil)
	for {
		d, f, err := iter.Next()
		if err == vfs.ErrIteratorDone {
			break
		}
		if err != nil {
			return err
		}
		if d != nil && d.DocName == ".git" {
			continue
		}
		if d != nil {
			err = fs.DestroyDirAndContent(d)
		} else {
			err = fs.DestroyFile(f)
		}
		if err != nil {
			return err
		}
	}

	return g.copyFiles(baseDir, rep)
}

func (g *gitFetcher) copyFiles(baseDir *vfs.DirDoc, rep *git.Repository) error {
	fs := g.fs

	ref, err := rep.Head()
	if err != nil {
		return err
	}

	commit, err := rep.CommitObject(ref.Hash())
	if err != nil {
		return err
	}

	files, err := commit.Files()
	if err != nil {
		return err
	}

	return files.ForEach(func(f *gitObject.File) error {
		abs := path.Join(baseDir.Fullpath, f.Name)
		dir := path.Dir(abs)

		_, err := vfs.MkdirAll(fs, dir, nil)
		if err != nil {
			return err
		}

		file, err := vfs.Create(fs, abs)
		if err != nil {
			return err
		}

		defer func() {
			if cerr := file.Close(); cerr != nil && err == nil {
				err = cerr
			}
		}()

		r, err := f.Reader()
		if err != nil {
			return err
		}

		defer r.Close()
		_, err = io.Copy(file, r)

		return err
	})
}

func resolveGithubURL(src *url.URL, filename string) (string, error) {
	match := ghURLRegex.FindStringSubmatch(src.Path)
	if len(match) != 3 {
		return "", &url.Error{
			Op:  "parsepath",
			URL: src.String(),
			Err: errors.New("Could not parse url git path"),
		}
	}

	user, project := match[1], match[2]
	branch := getWebBranch(src)

	u := fmt.Sprintf(ghRawManifestURL, user, project, branch, filename)
	return u, nil
}

func resolveGitlabURL(src *url.URL, filename string) (string, error) {
	match := glURLRegex.FindStringSubmatch(src.Path)
	if len(match) != 3 {
		return "", &url.Error{
			Op:  "parsepath",
			URL: src.String(),
			Err: errors.New("Could not parse url git path"),
		}
	}

	user, project := match[1], match[2]
	branch := getWebBranch(src)

	u := fmt.Sprintf(glRawManifestURL, src.Host, user, project, branch, filename)
	return u, nil
}

func resolveManifestURL(src *url.URL, filename string) (string, error) {
	// TODO check that it works with a branch
	srccopy, _ := url.Parse(src.String())
	srccopy.Scheme = "http"
	if srccopy.Path == "" || srccopy.Path[len(srccopy.Path)-1] != '/' {
		srccopy.Path += "/"
	}
	srccopy.Path = srccopy.Path + filename
	return srccopy.String(), nil
}

type gfs struct {
	fs   vfs.VFS
	base string
	dir  *vfs.DirDoc
}

type gfile struct {
	f      vfs.File
	name   string
	closed bool
}

func newGFile(f vfs.File, name string) *gfile {
	return &gfile{
		f:      f,
		name:   name,
		closed: false,
	}
}

func (f *gfile) Filename() string {
	return f.name
}

func (f *gfile) IsClosed() bool {
	return f.closed
}

func (f *gfile) Read(p []byte) (int, error) {
	return f.f.Read(p)
}

func (f *gfile) Write(p []byte) (int, error) {
	return f.f.Write(p)
}

func (f *gfile) Seek(offset int64, whence int) (int64, error) {
	return f.f.Seek(offset, whence)
}

func (f *gfile) Close() error {
	f.closed = true
	return f.f.Close()
}

func newGFS(fs vfs.VFS, base *vfs.DirDoc) *gfs {
	return &gfs{
		fs:   fs,
		base: path.Clean(base.Fullpath),
		dir:  base,
	}
}

func (fs *gfs) OpenFile(name string, flag int, perm os.FileMode) (gitFS.File, error) {
	var err error

	fullpath := path.Join(fs.base, name)
	dirbase := path.Dir(fullpath)

	if flag&os.O_CREATE != 0 {
		if _, err = vfs.MkdirAll(fs.fs, dirbase, nil); err != nil {
			return nil, err
		}
	}

	file, err := vfs.OpenFile(fs.fs, fullpath, flag, perm)
	if err != nil {
		return nil, err
	}

	return newGFile(file, name), nil
}

func (fs *gfs) Create(name string) (gitFS.File, error) {
	return fs.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_TRUNC, 0666)
}

func (fs *gfs) Open(name string) (gitFS.File, error) {
	fullpath := fs.Join(fs.base, name)
	f, err := vfs.OpenFile(fs.fs, fullpath, os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	return newGFile(f, fullpath[len(fs.base)+1:]), nil
}

func (fs *gfs) Remove(name string) error {
	return vfs.Remove(fs.fs, fs.Join(fs.base, name))
}

func (fs *gfs) Stat(name string) (gitFS.FileInfo, error) {
	return vfs.Stat(fs.fs, fs.Join(fs.base, name))
}

func (fs *gfs) ReadDir(name string) ([]gitFS.FileInfo, error) {
	var s []gitFS.FileInfo
	dir, err := fs.fs.DirByPath(fs.Join(fs.base, name))
	if err != nil {
		return nil, err
	}
	iter := fs.fs.DirIterator(dir, nil)
	for {
		d, f, err := iter.Next()
		if err == vfs.ErrIteratorDone {
			break
		}
		if err != nil {
			return nil, err
		}
		if d != nil {
			s = append(s, d)
		} else {
			s = append(s, f)
		}
	}
	return s, nil
}

func (fs *gfs) MkdirAll(path string, perm os.FileMode) error {
	_, err := vfs.MkdirAll(fs.fs, fs.Join(fs.base, path), nil)
	return err
}

func (fs *gfs) TempFile(dirname, prefix string) (gitFS.File, error) {
	// TODO: not really robust tempfile...
	name := fs.Join("/", dirname, prefix+"_"+strconv.Itoa(int(time.Now().UnixNano())))
	file, err := fs.Create(name)
	if err != nil {
		return nil, err
	}
	if err := file.Close(); err != nil {
		return nil, err
	}
	return fs.OpenFile(name, os.O_WRONLY|os.O_TRUNC, 0666)
}

func (fs *gfs) Rename(from, to string) error {
	return vfs.Rename(fs.fs, fs.Join(fs.base, from), fs.Join(fs.base, to))
}

func (fs *gfs) Join(elem ...string) string {
	return path.Join(elem...)
}

func (fs *gfs) Dir(name string) gitFS.Filesystem {
	name = fs.Join(fs.base, name)
	dir, err := fs.fs.DirByPath(name)
	if err != nil {
		// FIXME https://issues.apache.org/jira/browse/COUCHDB-3336
		// With a cluster of couchdb, we can have a race condition where we
		// query an index before it has been updated for a directory that has
		// just been created.
		time.Sleep(1 * time.Second)
		dir, err = fs.fs.DirByPath(name)
		if err != nil {
			panic(err)
		}
	}
	return newGFS(fs.fs, dir)
}

func (fs *gfs) Base() string {
	return fs.base
}

var (
	_ Fetcher          = &gitFetcher{}
	_ gitFS.Filesystem = &gfs{}
	_ gitFS.File       = &gfile{}
)
<commit_msg>Git gitlab branch<commit_after>package apps

import (
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/cozy/cozy-stack/pkg/vfs"
	gitFS "gopkg.in/src-d/go-billy.v2"
	git "gopkg.in/src-d/go-git.v4"
	gitPlumbing "gopkg.in/src-d/go-git.v4/plumbing"
	gitObject "gopkg.in/src-d/go-git.v4/plumbing/object"
	gitStorage "gopkg.in/src-d/go-git.v4/storage/filesystem"
)

// ghURLRegex is used to identify github
var ghURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)

const ghRawManifestURL = "https://raw.githubusercontent.com/%s/%s/%s/%s"

// glURLRegex is used to identify gitlab
var glURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)

const glRawManifestURL = "https://%s/%s/%s/raw/%s/%s"

type gitFetcher struct {
	fs          vfs.VFS
	manFilename string
}

func newGitFetcher(fs vfs.VFS, manFilename string) *gitFetcher {
	return &gitFetcher{fs: fs, manFilename: manFilename}
}

var manifestClient = &http.Client{
	Timeout: 60 * time.Second,
}

func isGithub(src *url.URL) bool {
	return src.Host == "github.com"
}

func isGitlab(src *url.URL) bool {
	return src.Host == "framagit.org" || strings.Contains(src.Host, "gitlab")
}

func (g *gitFetcher) FetchManifest(src *url.URL) (io.ReadCloser, error) {
	var err error

	var u string
	if isGithub(src) {
		u, err = resolveGithubURL(src, g.manFilename)
	} else if isGitlab(src) {
		u, err = resolveGitlabURL(src, g.manFilename)
	} else {
		u, err = resolveManifestURL(src, g.manFilename)
	}
	if err != nil {
		return nil, err
	}

	res, err := manifestClient.Get(u)
	if err != nil || res.StatusCode != 200 {
		return nil, ErrManifestNotReachable
	}

	return res.Body, nil
}

func (g *gitFetcher) Fetch(src *url.URL, baseDir *vfs.DirDoc) error {
	log.Debugf("[git] Fetch %s", src.String())
	fs := g.fs

	gitDirName := path.Join(baseDir.Fullpath, ".git")
	gitDir, err := fs.DirByPath(gitDirName)
	if err == nil {
		return g.pull(baseDir, gitDir, src)
	}
	if !os.IsNotExist(err) {
		return err
	}
	gitDir, err = vfs.Mkdir(fs, gitDirName, nil)
	if err != nil {
		return err
	}
	return g.clone(baseDir, gitDir, src)
}

func getGitBranch(src *url.URL) string {
	if src.Fragment != "" {
		return "refs/heads/" + src.Fragment
	}
	return "HEAD"
}

func getWebBranch(src *url.URL) string {
	if src.Fragment != "" {
		return src.Fragment
	}
	return "HEAD"
}

// clone creates a new bare git repository and install all the files of the
// last commit in the application tree.
func (g *gitFetcher) clone(baseDir, gitDir *vfs.DirDoc, src *url.URL) error {
	fs := g.fs

	storage, err := gitStorage.NewStorage(newGFS(fs, gitDir))
	if err != nil {
		return err
	}

	branch := getGitBranch(src)
	log.Debugf("[git] Clone %s %s", src.String(), branch)

	// XXX Gitlab doesn't support the git protocol
	if isGitlab(src) {
		src.Scheme = "https"
		src.Fragment = ""
	}

	rep, err := git.Clone(storage, nil, &git.CloneOptions{
		URL:           src.String(),
		Depth:         1,
		SingleBranch:  true,
		ReferenceName: gitPlumbing.ReferenceName(branch),
	})
	if err != nil {
		return err
	}

	return g.copyFiles(baseDir, rep)
}

// pull will fetch the latest objects from the default remote and if updates
// are available, it will update the application tree files.
func (g *gitFetcher) pull(baseDir, gitDir *vfs.DirDoc, src *url.URL) error {
	fs := g.fs

	storage, err := gitStorage.NewStorage(newGFS(fs, gitDir))
	if err != nil {
		return err
	}

	rep, err := git.Open(storage, nil)
	if err != nil {
		return err
	}

	branch := getGitBranch(src)
	log.Debugf("[git] Pull %s %s", src.String(), branch)

	err = rep.Pull(&git.PullOptions{
		SingleBranch:  true,
		ReferenceName: gitPlumbing.ReferenceName(branch),
	})
	if err == git.NoErrAlreadyUpToDate {
		return nil
	}
	if err != nil {
		return err
	}

	iter := fs.DirIterator(baseDir, nil)
	for {
		d, f, err := iter.Next()
		if err == vfs.ErrIteratorDone {
			break
		}
		if err != nil {
			return err
		}
		if d != nil && d.DocName == ".git" {
			continue
		}
		if d != nil {
			err = fs.DestroyDirAndContent(d)
		} else {
			err = fs.DestroyFile(f)
		}
		if err != nil {
			return err
		}
	}

	return g.copyFiles(baseDir, rep)
}

func (g *gitFetcher) copyFiles(baseDir *vfs.DirDoc, rep *git.Repository) error {
	fs := g.fs

	ref, err := rep.Head()
	if err != nil {
		return err
	}

	commit, err := rep.CommitObject(ref.Hash())
	if err != nil {
		return err
	}

	files, err := commit.Files()
	if err != nil {
		return err
	}

	return files.ForEach(func(f *gitObject.File) error {
		abs := path.Join(baseDir.Fullpath, f.Name)
		dir := path.Dir(abs)

		_, err := vfs.MkdirAll(fs, dir, nil)
		if err != nil {
			return err
		}

		file, err := vfs.Create(fs, abs)
		if err != nil {
			return err
		}

		defer func() {
			if cerr := file.Close(); cerr != nil && err == nil {
				err = cerr
			}
		}()

		r, err := f.Reader()
		if err != nil {
			return err
		}

		defer r.Close()
		_, err = io.Copy(file, r)

		return err
	})
}

func resolveGithubURL(src *url.URL, filename string) (string, error) {
	match := ghURLRegex.FindStringSubmatch(src.Path)
	if len(match) != 3 {
		return "", &url.Error{
			Op:  "parsepath",
			URL: src.String(),
			Err: errors.New("Could not parse url git path"),
		}
	}

	user, project := match[1], match[2]
	branch := getWebBranch(src)

	u := fmt.Sprintf(ghRawManifestURL, user, project, branch, filename)
	return u, nil
}

func resolveGitlabURL(src *url.URL, filename string) (string, error) {
	match := glURLRegex.FindStringSubmatch(src.Path)
	if len(match) != 3 {
		return "", &url.Error{
			Op:  "parsepath",
			URL: src.String(),
			Err: errors.New("Could not parse url git path"),
		}
	}

	user, project := match[1], match[2]
	branch := getWebBranch(src)

	u := fmt.Sprintf(glRawManifestURL, src.Host, user, project, branch, filename)
	return u, nil
}

func resolveManifestURL(src *url.URL, filename string) (string, error) {
	// TODO check that it works with a branch
	srccopy, _ := url.Parse(src.String())
	srccopy.Scheme = "http"
	if srccopy.Path == "" || srccopy.Path[len(srccopy.Path)-1] != '/' {
		srccopy.Path += "/"
	}
	srccopy.Path = srccopy.Path + filename
	return srccopy.String(), nil
}

type gfs struct {
	fs   vfs.VFS
	base string
	dir  *vfs.DirDoc
}

type gfile struct {
	f      vfs.File
	name   string
	closed bool
}

func newGFile(f vfs.File, name string) *gfile {
	return &gfile{
		f:      f,
		name:   name,
		closed: false,
	}
}

func (f *gfile) Filename() string {
	return f.name
}

func (f *gfile) IsClosed() bool {
	return f.closed
}

func (f *gfile) Read(p []byte) (int, error) {
	return f.f.Read(p)
}

func (f *gfile) Write(p []byte) (int, error) {
	return f.f.Write(p)
}

func (f *gfile) Seek(offset int64, whence int) (int64, error) {
	return f.f.Seek(offset, whence)
}

func (f *gfile) Close() error {
	f.closed = true
	return f.f.Close()
}

func newGFS(fs vfs.VFS, base *vfs.DirDoc) *gfs {
	return &gfs{
		fs:   fs,
		base: path.Clean(base.Fullpath),
		dir:  base,
	}
}

func (fs *gfs) OpenFile(name string, flag int, perm os.FileMode) (gitFS.File, error) {
	var err error

	fullpath := path.Join(fs.base, name)
	dirbase := path.Dir(fullpath)

	if flag&os.O_CREATE != 0 {
		if _, err = vfs.MkdirAll(fs.fs, dirbase, nil); err != nil {
			return nil, err
		}
	}

	file, err := vfs.OpenFile(fs.fs, fullpath, flag, perm)
	if err != nil {
		return nil, err
	}

	return newGFile(file, name), nil
}

func (fs *gfs) Create(name string) (gitFS.File, error) {
	return fs.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_TRUNC, 0666)
}

func (fs *gfs) Open(name string) (gitFS.File, error) {
	fullpath := fs.Join(fs.base, name)
	f, err := vfs.OpenFile(fs.fs, fullpath, os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	return newGFile(f, fullpath[len(fs.base)+1:]), nil
}

func (fs *gfs) Remove(name string) error {
	return vfs.Remove(fs.fs, fs.Join(fs.base, name))
}

func (fs *gfs) Stat(name string) (gitFS.FileInfo, error) {
	return vfs.Stat(fs.fs, fs.Join(fs.base, name))
}

func (fs *gfs) ReadDir(name string) ([]gitFS.FileInfo, error) {
	var s []gitFS.FileInfo
	dir, err := fs.fs.DirByPath(fs.Join(fs.base, name))
	if err != nil {
		return nil, err
	}
	iter := fs.fs.DirIterator(dir, nil)
	for {
		d, f, err := iter.Next()
		if err == vfs.ErrIteratorDone {
			break
		}
		if err != nil {
			return nil, err
		}
		if d != nil {
			s = append(s, d)
		} else {
			s = append(s, f)
		}
	}
	return s, nil
}

func (fs *gfs) MkdirAll(path string, perm os.FileMode) error {
	_, err := vfs.MkdirAll(fs.fs, fs.Join(fs.base, path), nil)
	return err
}

func (fs *gfs) TempFile(dirname, prefix string) (gitFS.File, error) {
	// TODO: not really robust tempfile...
	name := fs.Join("/", dirname, prefix+"_"+strconv.Itoa(int(time.Now().UnixNano())))
	file, err := fs.Create(name)
	if err != nil {
		return nil, err
	}
	if err := file.Close(); err != nil {
		return nil, err
	}
	return fs.OpenFile(name, os.O_WRONLY|os.O_TRUNC, 0666)
}

func (fs *gfs) Rename(from, to string) error {
	return vfs.Rename(fs.fs, fs.Join(fs.base, from), fs.Join(fs.base, to))
}

func (fs *gfs) Join(elem ...string) string {
	return path.Join(elem...)
}

func (fs *gfs) Dir(name string) gitFS.Filesystem {
	name = fs.Join(fs.base, name)
	dir, err := fs.fs.DirByPath(name)
	if err != nil {
		// FIXME https://issues.apache.org/jira/browse/COUCHDB-3336
		// With a cluster of couchdb, we can have a race condition where we
		// query an index before it has been updated for a directory that has
		// just been created.
		time.Sleep(1 * time.Second)
		dir, err = fs.fs.DirByPath(name)
		if err != nil {
			panic(err)
		}
	}
	return newGFS(fs.fs, dir)
}

func (fs *gfs) Base() string {
	return fs.base
}

var (
	_ Fetcher          = &gitFetcher{}
	_ gitFS.Filesystem = &gfs{}
	_ gitFS.File       = &gfile{}
)
<|endoftext|>


<commit_before>// Copyright 2019 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package ulog exposes logging via a Go interface.
//
// ulog has three implementations of the Logger interface: a Go standard
// library "log" package Logger, a kernel syslog (dmesg) Logger, and a test
// Logger that logs via a test's testing.TB.Logf.
// To use the test logger import "ulog/ulogtest".
package ulog

import (
	"log"
	"os"
)

// Logger is a log receptacle.
//
// It puts your information somewhere for safekeeping.
type Logger interface {
	Printf(format string, v ...interface{})
	Print(v ...interface{})
}

// Log is a Logger that prints to stderr, like the default log package.
var Log Logger = log.New(os.Stderr, "", log.LstdFlags)

type emptyLogger struct{}

func (emptyLogger) Printf(format string, v ...interface{}) {}
func (emptyLogger) Print(v ...interface{})                 {}

// Null is a logger that prints nothing.
var Null Logger = emptyLogger{}

// LogIfError logs "msg: err" if err is not nil.
func LogIfError(l Logger, err error, msg string) {
	if err != nil {
		l.Printf("%s: %v", msg, err)
	}
}
<commit_msg>LogIfError was never a good idea<commit_after>// Copyright 2019 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package ulog exposes logging via a Go interface.
//
// ulog has three implementations of the Logger interface: a Go standard
// library "log" package Logger, a kernel syslog (dmesg) Logger, and a test
// Logger that logs via a test's testing.TB.Logf.
// To use the test logger import "ulog/ulogtest".
package ulog

import (
	"log"
	"os"
)

// Logger is a log receptacle.
//
// It puts your information somewhere for safekeeping.
type Logger interface {
	Printf(format string, v ...interface{})
	Print(v ...interface{})
}

// Log is a Logger that prints to stderr, like the default log package.
var Log Logger = log.New(os.Stderr, "", log.LstdFlags)

type emptyLogger struct{}

func (emptyLogger) Printf(format string, v ...interface{}) {}
func (emptyLogger) Print(v ...interface{})                 {}

// Null is a logger that prints nothing.
var Null Logger = emptyLogger{}
<|endoftext|>