<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tls

import (
	"bufio"
	"encoding/hex"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"strconv"
	"strings"
	"sync"
)

// TLS reference tests run a connection against a reference implementation
// (OpenSSL) of TLS and record the bytes of the resulting connection. The Go
// code, during a test, is configured with deterministic randomness and so the
// reference test can be reproduced exactly in the future.
//
// In order to save everyone who wishes to run the tests from needing the
// reference implementation installed, the reference connections are saved in
// files in the testdata directory. Thus running the tests involves nothing
// external, but creating and updating them requires the reference
// implementation.
//
// Tests can be updated by running them with the -update flag. This will cause
// the test files. Generally one should combine the -update flag with -test.run
// to updated a specific test. Since the reference implementation will always
// generate fresh random numbers, large parts of the reference connection will
// always change.

var update = flag.Bool("update", false, "update golden files on disk")

// recordingConn is a net.Conn that records the traffic that passes through it.
// WriteTo can be used to produce output that can be later be loaded with
// ParseTestData.
type recordingConn struct {
	net.Conn
	sync.Mutex
	flows   [][]byte
	reading bool
}

func (r *recordingConn) Read(b []byte) (n int, err error) {
	if n, err = r.Conn.Read(b); n == 0 {
		return
	}
	b = b[:n]

	r.Lock()
	defer r.Unlock()

	if l := len(r.flows); l == 0 || !r.reading {
		buf := make([]byte, len(b))
		copy(buf, b)
		r.flows = append(r.flows, buf)
	} else {
		r.flows[l-1] = append(r.flows[l-1], b[:n]...)
	}
	r.reading = true
	return
}

func (r *recordingConn) Write(b []byte) (n int, err error) {
	if n, err = r.Conn.Write(b); n == 0 {
		return
	}
	b = b[:n]

	r.Lock()
	defer r.Unlock()

	if l := len(r.flows); l == 0 || r.reading {
		buf := make([]byte, len(b))
		copy(buf, b)
		r.flows = append(r.flows, buf)
	} else {
		r.flows[l-1] = append(r.flows[l-1], b[:n]...)
	}
	r.reading = false
	return
}

// WriteTo writes Go source code to w that contains the recorded traffic.
func (r *recordingConn) WriteTo(w io.Writer) {
	// TLS always starts with a client to server flow.
	clientToServer := true

	for i, flow := range r.flows {
		source, dest := "client", "server"
		if !clientToServer {
			source, dest = dest, source
		}
		fmt.Fprintf(w, ">>> Flow %d (%s to %s)\n", i+1, source, dest)
		dumper := hex.Dumper(w)
		dumper.Write(flow)
		dumper.Close()
		clientToServer = !clientToServer
	}
}

func parseTestData(r io.Reader) (flows [][]byte, err error) {
	var currentFlow []byte

	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		// If the line starts with ">>> " then it marks the beginning
		// of a new flow.
		if strings.HasPrefix(line, ">>> ") {
			if len(currentFlow) > 0 || len(flows) > 0 {
				flows = append(flows, currentFlow)
				currentFlow = nil
			}
			continue
		}

		// Otherwise the line is a line of hex dump that looks like:
		// 00000170  fc f5 06 bf (...)  |.....X{&?......!|
		// (Some bytes have been omitted from the middle section.)

		if i := strings.IndexByte(line, ' '); i >= 0 {
			line = line[i:]
		} else {
			return nil, errors.New("invalid test data")
		}

		if i := strings.IndexByte(line, '|'); i >= 0 {
			line = line[:i]
		} else {
			return nil, errors.New("invalid test data")
		}

		hexBytes := strings.Fields(line)
		for _, hexByte := range hexBytes {
			val, err := strconv.ParseUint(hexByte, 16, 8)
			if err != nil {
				return nil, errors.New("invalid hex byte in test data: " + err.Error())
			}
			currentFlow = append(currentFlow, byte(val))
		}
	}

	if len(currentFlow) > 0 {
		flows = append(flows, currentFlow)
	}

	return flows, nil
}

// tempFile creates a temp file containing contents and returns its path.
func tempFile(contents string) string {
	file, err := ioutil.TempFile("", "go-tls-test")
	if err != nil {
		panic("failed to create temp file: " + err.Error())
	}
	path := file.Name()
	file.WriteString(contents)
	file.Close()
	return path
}
<commit_msg>crypto/tls: fix WriteTo method signature<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tls

import (
	"bufio"
	"encoding/hex"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"strconv"
	"strings"
	"sync"
)

// TLS reference tests run a connection against a reference implementation
// (OpenSSL) of TLS and record the bytes of the resulting connection. The Go
// code, during a test, is configured with deterministic randomness and so the
// reference test can be reproduced exactly in the future.
//
// In order to save everyone who wishes to run the tests from needing the
// reference implementation installed, the reference connections are saved in
// files in the testdata directory. Thus running the tests involves nothing
// external, but creating and updating them requires the reference
// implementation.
//
// Tests can be updated by running them with the -update flag. This will cause
// the test files. Generally one should combine the -update flag with -test.run
// to updated a specific test. Since the reference implementation will always
// generate fresh random numbers, large parts of the reference connection will
// always change.

var update = flag.Bool("update", false, "update golden files on disk")

// recordingConn is a net.Conn that records the traffic that passes through it.
// WriteTo can be used to produce output that can be later be loaded with
// ParseTestData.
type recordingConn struct {
	net.Conn
	sync.Mutex
	flows   [][]byte
	reading bool
}

func (r *recordingConn) Read(b []byte) (n int, err error) {
	if n, err = r.Conn.Read(b); n == 0 {
		return
	}
	b = b[:n]

	r.Lock()
	defer r.Unlock()

	if l := len(r.flows); l == 0 || !r.reading {
		buf := make([]byte, len(b))
		copy(buf, b)
		r.flows = append(r.flows, buf)
	} else {
		r.flows[l-1] = append(r.flows[l-1], b[:n]...)
	}
	r.reading = true
	return
}

func (r *recordingConn) Write(b []byte) (n int, err error) {
	if n, err = r.Conn.Write(b); n == 0 {
		return
	}
	b = b[:n]

	r.Lock()
	defer r.Unlock()

	if l := len(r.flows); l == 0 || r.reading {
		buf := make([]byte, len(b))
		copy(buf, b)
		r.flows = append(r.flows, buf)
	} else {
		r.flows[l-1] = append(r.flows[l-1], b[:n]...)
	}
	r.reading = false
	return
}

// WriteTo writes Go source code to w that contains the recorded traffic.
func (r *recordingConn) WriteTo(w io.Writer) (int64, error) {
	// TLS always starts with a client to server flow.
	clientToServer := true
	var written int64
	for i, flow := range r.flows {
		source, dest := "client", "server"
		if !clientToServer {
			source, dest = dest, source
		}
		n, err := fmt.Fprintf(w, ">>> Flow %d (%s to %s)\n", i+1, source, dest)
		written += int64(n)
		if err != nil {
			return written, err
		}
		dumper := hex.Dumper(w)
		n, err = dumper.Write(flow)
		written += int64(n)
		if err != nil {
			return written, err
		}
		err = dumper.Close()
		if err != nil {
			return written, err
		}
		clientToServer = !clientToServer
	}
	return written, nil
}

func parseTestData(r io.Reader) (flows [][]byte, err error) {
	var currentFlow []byte

	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		// If the line starts with ">>> " then it marks the beginning
		// of a new flow.
		if strings.HasPrefix(line, ">>> ") {
			if len(currentFlow) > 0 || len(flows) > 0 {
				flows = append(flows, currentFlow)
				currentFlow = nil
			}
			continue
		}

		// Otherwise the line is a line of hex dump that looks like:
		// 00000170  fc f5 06 bf (...)  |.....X{&?......!|
		// (Some bytes have been omitted from the middle section.)

		if i := strings.IndexByte(line, ' '); i >= 0 {
			line = line[i:]
		} else {
			return nil, errors.New("invalid test data")
		}

		if i := strings.IndexByte(line, '|'); i >= 0 {
			line = line[:i]
		} else {
			return nil, errors.New("invalid test data")
		}

		hexBytes := strings.Fields(line)
		for _, hexByte := range hexBytes {
			val, err := strconv.ParseUint(hexByte, 16, 8)
			if err != nil {
				return nil, errors.New("invalid hex byte in test data: " + err.Error())
			}
			currentFlow = append(currentFlow, byte(val))
		}
	}

	if len(currentFlow) > 0 {
		flows = append(flows, currentFlow)
	}

	return flows, nil
}

// tempFile creates a temp file containing contents and returns its path.
func tempFile(contents string) string {
	file, err := ioutil.TempFile("", "go-tls-test")
	if err != nil {
		panic("failed to create temp file: " + err.Error())
	}
	path := file.Name()
	file.WriteString(contents)
	file.Close()
	return path
}
<|endoftext|>


<commit_before>// Copyright 2014 Rana Ian. All rights reserved.
// Use of this source code is governed by The MIT License
// found in the accompanying LICENSE file.

package ora_test

import (
	"sync"
	"testing"

	"gopkg.in/rana/ora.v3"
)

func TestServer_OpenCloseSession(t *testing.T) {
	env, err := ora.OpenEnv(nil)
	defer env.Close()
	testErr(err, t)
	srv, err := env.OpenSrv(testSrvCfg)
	defer srv.Close()
	testErr(err, t)

	ses, err := srv.OpenSes(testSesCfg)
	testErr(err, t)
	if ses == nil {
		t.Fatal("session is nil")
	} else {
		err = ses.Close()
		if err != nil {
			t.Fatal(err)
		}
	}
}

func TestServer_Ping(t *testing.T) {
	env, err := ora.OpenEnv(nil)
	defer env.Close()
	testErr(err, t)
	srv, err := env.OpenSrv(testSrvCfg)
	defer srv.Close()
	testErr(err, t)
	ses, err := srv.OpenSes(testSesCfg)
	defer ses.Close()
	testErr(err, t)

	err = ses.Ping()
	testErr(err, t)
}

func TestServer_Version(t *testing.T) {
	env, err := ora.OpenEnv(nil)
	defer env.Close()
	testErr(err, t)
	srv, err := env.OpenSrv(testSrvCfg)
	defer srv.Close()
	testErr(err, t)
	ses, err := srv.OpenSes(testSesCfg)
	defer ses.Close()
	testErr(err, t)

	version, err := srv.Version()
	testErr(err, t)
	if version == "" {
		t.Fatal("Version is empty.")
	}
}

func TestPool(t *testing.T) {
	env, err := ora.OpenEnv(nil)
	testErr(err, t)
	defer env.Close()
	const idleSize = 2
	pool := env.NewPool(testSrvCfg, testSesCfg, idleSize)
	defer pool.Close()

	getProcCount := func() int {
		ses, err := pool.Get()
		testErr(err, t)
		defer pool.Put(ses)
		rset, err := ses.PrepAndQry("SELECT COUNT(0) FROM v$process")
		if err != nil {
			t.Log(err)
			return -1
		}
		rset.Next()
		c := int(rset.Row[0].(float64))
		for rset.Next() {
		}
		return c
	}

	var wg sync.WaitGroup
	c1 := getProcCount()
	for i := 0; i < 2*idleSize+1; i++ {
		wg.Add(1)
		go func(c bool) {
			defer wg.Done()
			ses, err := pool.Get()
			testErr(err, t)
			if c {
				ses.Close()
			} else {
				pool.Put(ses)
			}
		}(i%2 == 0)
	}
	wg.Wait()

	c2 := getProcCount()
	t.Logf("c1=%d c2=%d", c1, c2)
	if c2-c1 > 2 {
		t.Errorf("process count went to %d from %d!", c2, c1)
	}
}
<commit_msg>type switch for TestPool<commit_after>// Copyright 2014 Rana Ian. All rights reserved.
// Use of this source code is governed by The MIT License
// found in the accompanying LICENSE file.

package ora_test

import (
	"fmt"
	"strconv"
	"sync"
	"testing"

	"gopkg.in/rana/ora.v3"
)

func TestServer_OpenCloseSession(t *testing.T) {
	env, err := ora.OpenEnv(nil)
	defer env.Close()
	testErr(err, t)
	srv, err := env.OpenSrv(testSrvCfg)
	defer srv.Close()
	testErr(err, t)

	ses, err := srv.OpenSes(testSesCfg)
	testErr(err, t)
	if ses == nil {
		t.Fatal("session is nil")
	} else {
		err = ses.Close()
		if err != nil {
			t.Fatal(err)
		}
	}
}

func TestServer_Ping(t *testing.T) {
	env, err := ora.OpenEnv(nil)
	defer env.Close()
	testErr(err, t)
	srv, err := env.OpenSrv(testSrvCfg)
	defer srv.Close()
	testErr(err, t)
	ses, err := srv.OpenSes(testSesCfg)
	defer ses.Close()
	testErr(err, t)

	err = ses.Ping()
	testErr(err, t)
}

func TestServer_Version(t *testing.T) {
	env, err := ora.OpenEnv(nil)
	defer env.Close()
	testErr(err, t)
	srv, err := env.OpenSrv(testSrvCfg)
	defer srv.Close()
	testErr(err, t)
	ses, err := srv.OpenSes(testSesCfg)
	defer ses.Close()
	testErr(err, t)

	version, err := srv.Version()
	testErr(err, t)
	if version == "" {
		t.Fatal("Version is empty.")
	}
}

func TestPool(t *testing.T) {
	env, err := ora.OpenEnv(nil)
	testErr(err, t)
	defer env.Close()
	const idleSize = 2
	pool := env.NewPool(testSrvCfg, testSesCfg, idleSize)
	defer pool.Close()

	getProcCount := func() int {
		ses, err := pool.Get()
		testErr(err, t)
		defer pool.Put(ses)
		rset, err := ses.PrepAndQry("SELECT COUNT(0) FROM v$process")
		if err != nil {
			t.Log(err)
			return -1
		}
		rset.Next()
		var c int
		switch x := rset.Row[0].(type) {
		case float64:
			c = int(x)
		case ora.OCINum:
			c, _ = strconv.Atoi(x.String())
		default:
			c, _ = strconv.Atoi(fmt.Sprintf("%v", x))
		}
		for rset.Next() {
		}
		return c
	}

	var wg sync.WaitGroup
	c1 := getProcCount()
	for i := 0; i < 2*idleSize+1; i++ {
		wg.Add(1)
		go func(c bool) {
			defer wg.Done()
			ses, err := pool.Get()
			testErr(err, t)
			if c {
				ses.Close()
			} else {
				pool.Put(ses)
			}
		}(i%2 == 0)
	}
	wg.Wait()

	c2 := getProcCount()
	t.Logf("c1=%d c2=%d", c1, c2)
	if c2-c1 > 2 {
		t.Errorf("process count went to %d from %d!", c2, c1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	ggio "github.com/gogo/protobuf/io"
	p2p_net "github.com/libp2p/go-libp2p-net"
	p2p_peer "github.com/libp2p/go-libp2p-peer"
	p2p_pstore "github.com/libp2p/go-libp2p-peerstore"
	mc "github.com/mediachain/concat/mc"
	pb "github.com/mediachain/concat/proto"
	multiaddr "github.com/multiformats/go-multiaddr"
	"log"
	"time"
)

// goOffline stops the network
func (node *Node) goOffline() error {
	node.mx.Lock()
	defer node.mx.Unlock()

	switch node.status {
	case StatusPublic:
		fallthrough
	case StatusOnline:
		node.netCancel()

		err := node.dht.Close()
		if err != nil {
			log.Printf("Error closing DHT: %s", err.Error())
		}
		node.dht = nil

		err = node.host.Close()
		if err != nil {
			log.Printf("Error closing host: %s", err.Error())
		}
		node.host = nil

		node.status = StatusOffline
		node.natCfg.Clear()
		log.Println("Node is offline")
		return err

	default:
		return nil
	}
}

// goOnline starts the network; if the node is already public, it stays public
func (node *Node) goOnline() error {
	node.mx.Lock()
	defer node.mx.Unlock()

	switch node.status {
	case StatusOffline:
		err := node._goOnline()
		if err != nil {
			return err
		}

		node.status = StatusOnline
		log.Println("Node is online")
		return nil

	default:
		return nil
	}
}

func (node *Node) _goOnline() error {
	var opts []interface{}
	if node.natCfg.Opt == mc.NATConfigAuto {
		opts = []interface{}{mc.NATPortMap}
	}

	ctx, cancel := context.WithCancel(context.Background())
	host, err := mc.NewHost(ctx, node.PeerIdentity, node.laddr, opts...)
	if err != nil {
		cancel()
		return err
	}

	host.SetStreamHandler("/mediachain/node/id", node.idHandler)
	host.SetStreamHandler("/mediachain/node/ping", node.pingHandler)
	host.SetStreamHandler("/mediachain/node/query", node.queryHandler)
	host.SetStreamHandler("/mediachain/node/data", node.dataHandler)
	host.SetStreamHandler("/mediachain/node/push", node.pushHandler)

	dht := NewDHT(ctx, host)

	err = dht.Bootstrap()
	if err != nil {
		// that's non-fatal, it will just fail to lookup
		log.Printf("Error boostrapping DHT: %s", err.Error())
	}

	node.host = host
	node.netCtx = ctx
	node.netCancel = cancel
	node.dht = dht

	return nil
}

// goPublic starts the network if it's not already up and registers with the
// directory; fails with NoDirectory if that hasn't been configured.
func (node *Node) goPublic() error {
	if node.dir == nil {
		return NoDirectory
	}

	node.mx.Lock()
	defer node.mx.Unlock()

	switch node.status {
	case StatusOffline:
		err := node._goOnline()
		if err != nil {
			return err
		}

		if node.natCfg.Opt == mc.NATConfigAuto {
			// wait a bit for NAT port mapping to take effect
			time.Sleep(2 * time.Second)
		}
		fallthrough

	case StatusOnline:
		go node.registerPeer(node.netCtx)
		node.status = StatusPublic

		log.Println("Node is public")
		return nil

	default:
		return nil
	}
}

func (node *Node) registerPeer(ctx context.Context) {
	for {
		err := node.registerPeerImpl(ctx)
		if err == nil {
			return
		}

		// sleep and retry
		select {
		case <-ctx.Done():
			return

		case <-time.After(5 * time.Minute):
			log.Println("Retrying to register with directory")
			continue
		}
	}
}

func (node *Node) registerPeerImpl(ctx context.Context) error {
	err := node.host.Connect(node.netCtx, *node.dir)
	if err != nil {
		log.Printf("Failed to connect to directory: %s", err.Error())
		return err
	}

	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/register")
	if err != nil {
		log.Printf("Failed to open directory stream: %s", err.Error())
		return err
	}
	defer s.Close()

	var pinfo = p2p_pstore.PeerInfo{ID: node.ID}
	var pbpi pb.PeerInfo

	w := ggio.NewDelimitedWriter(s)
	for {
		addrs := node.publicAddrs()

		if len(addrs) > 0 {
			pinfo.Addrs = addrs
			mc.PBFromPeerInfo(&pbpi, pinfo)
			msg := pb.RegisterPeer{&pbpi}

			err = w.WriteMsg(&msg)
			if err != nil {
				log.Printf("Failed to register with directory: %s", err.Error())
				return err
			}
		} else {
			log.Printf("Skipped directory registration; no public address")
		}

		select {
		case <-ctx.Done():
			return nil

		case <-time.After(5 * time.Minute):
			continue
		}
	}
}

// The notion of public address is relative to the network location of the directory
// We want to support directories running on localhost (testing) or in a private network,
// and on the same time not leak internal addresses in public directory announcements.
// So, depending on the directory ip address:
//  If the directory is on the localhost, return the localhost address reported
//   by the host
//  If the directory is in a private range, filter Addrs reported by the
//   host, dropping unroutable addresses
//  If the directory is in a public range, then
//   If the NAT config is manual, return the configured address
//   If the NAT is auto or none, filter the addresses returned by the host,
//      and return only public addresses
func (node *Node) publicAddrs() []multiaddr.Multiaddr {
	if node.status == StatusOffline || node.dir == nil {
		return nil
	}

	dir := node.dir.Addrs[0]
	switch {
	case mc.IsLocalhostAddr(dir):
		return mc.FilterAddrs(node.host.Addrs(), mc.IsLocalhostAddr)

	case mc.IsPrivateAddr(dir):
		return mc.FilterAddrs(node.host.Addrs(), mc.IsRoutableAddr)

	default:
		switch node.natCfg.Opt {
		case mc.NATConfigManual:
			addr := node.natAddr()
			if addr == nil {
				return nil
			}
			return []multiaddr.Multiaddr{addr}

		default:
			return mc.FilterAddrs(node.host.Addrs(), mc.IsPublicAddr)
		}
	}
}

// netAddrs retrieves all routable addresses for the node, regardless of directory
// this includes the auto-detected or manually configured NAT address
func (node *Node) netAddrs() []multiaddr.Multiaddr {
	if node.status == StatusOffline {
		return nil
	}

	addrs := mc.FilterAddrs(node.host.Addrs(), mc.IsRoutableAddr)

	nataddr := node.natAddr()
	if nataddr != nil {
		addrs = append(addrs, nataddr)
	}

	return addrs
}

func (node *Node) natAddr() multiaddr.Multiaddr {
	if node.natCfg.Opt != mc.NATConfigManual {
		return nil
	}

	addr, err := node.natCfg.PublicAddr(node.laddr)
	if err != nil {
		log.Printf("Error determining pubic address: %s", err.Error())
	}
	return addr
}

func (node *Node) netPeerAddrs(pid p2p_peer.ID) []multiaddr.Multiaddr {
	if node.status == StatusOffline {
		return nil
	}

	pinfo := node.host.Peerstore().PeerInfo(pid)
	return pinfo.Addrs
}

func (node *Node) netConns() []p2p_pstore.PeerInfo {
	if node.status == StatusOffline {
		return nil
	}

	conns := node.host.Network().Conns()
	peers := make([]p2p_pstore.PeerInfo, len(conns))
	for x, conn := range conns {
		peers[x].ID = conn.RemotePeer()
		peers[x].Addrs = []multiaddr.Multiaddr{conn.RemoteMultiaddr()}
	}

	return peers
}

// Connectivity
func (node *Node) doConnect(ctx context.Context, pid p2p_peer.ID) error {
	if node.status == StatusOffline {
		return NodeOffline
	}

	if node.host.Network().Connectedness(pid) == p2p_net.Connected {
		return nil
	}

	addrs := node.host.Peerstore().Addrs(pid)
	if len(addrs) > 0 {
		return node.host.Connect(ctx, p2p_pstore.PeerInfo{pid, addrs})
	}

	pinfo, err := node.doLookup(ctx, pid)
	if err != nil {
		return err
	}

	return node.host.Connect(node.netCtx, pinfo)
}

func (node *Node) doLookup(ctx context.Context, pid p2p_peer.ID) (pinfo p2p_pstore.PeerInfo, err error) {
	pinfo, err = node.doLookupImpl(ctx, pid)
	if err == nil {
		node.host.Peerstore().AddAddrs(pid, pinfo.Addrs, p2p_pstore.ProviderAddrTTL)
	}

	return
}

func (node *Node) doLookupImpl(ctx context.Context, pid p2p_peer.ID) (pinfo p2p_pstore.PeerInfo, err error) {
	if node.status == StatusOffline {
		return pinfo, NodeOffline
	}

	if node.dir == nil {
		goto lookup_dht
	}

	pinfo, err = node.doDirLookup(ctx, pid)
	if err == nil {
		return
	}

	if err != UnknownPeer {
		log.Printf("Directory lookup error: %s", err.Error())
	}

lookup_dht:
	return node.dht.Lookup(ctx, pid)
}

func (node *Node) doDirLookup(ctx context.Context, pid p2p_peer.ID) (empty p2p_pstore.PeerInfo, err error) {
	if node.status == StatusOffline {
		return empty, NodeOffline
	}

	if node.dir == nil {
		return empty, NoDirectory
	}

	node.host.Connect(node.netCtx, *node.dir)
	if err != nil {
		return empty, err
	}

	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/lookup")
	if err != nil {
		return empty, err
	}
	defer s.Close()

	req := pb.LookupPeerRequest{pid.Pretty()}
	w := ggio.NewDelimitedWriter(s)
	err = w.WriteMsg(&req)
	if err != nil {
		return empty, err
	}

	var resp pb.LookupPeerResponse
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	err = r.ReadMsg(&resp)
	if err != nil {
		return empty, err
	}

	if resp.Peer == nil {
		return empty, UnknownPeer
	}

	pinfo, err := mc.PBToPeerInfo(resp.Peer)
	if err != nil {
		return empty, err
	}

	return pinfo, nil
}

func (node *Node) doDirList(ctx context.Context) ([]string, error) {
	if node.status == StatusOffline {
		return nil, NodeOffline
	}

	if node.dir == nil {
		return nil, NoDirectory
	}

	err := node.host.Connect(node.netCtx, *node.dir)
	if err != nil {
		return nil, err
	}

	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/list")
	if err != nil {
		return nil, err
	}
	defer s.Close()

	w := ggio.NewDelimitedWriter(s)
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)

	var req pb.ListPeersRequest
	var res pb.ListPeersResponse

	err = w.WriteMsg(&req)
	if err != nil {
		return nil, err
	}

	err = r.ReadMsg(&res)
	if err != nil {
		return nil, err
	}

	return res.Peers, nil
}
<commit_msg>mcnode/net: update pb.RegisterPeer message constructor<commit_after>package main

import (
	"context"
	ggio "github.com/gogo/protobuf/io"
	p2p_net "github.com/libp2p/go-libp2p-net"
	p2p_peer "github.com/libp2p/go-libp2p-peer"
	p2p_pstore "github.com/libp2p/go-libp2p-peerstore"
	mc "github.com/mediachain/concat/mc"
	pb "github.com/mediachain/concat/proto"
	multiaddr "github.com/multiformats/go-multiaddr"
	"log"
	"time"
)

// goOffline stops the network
func (node *Node) goOffline() error {
	node.mx.Lock()
	defer node.mx.Unlock()

	switch node.status {
	case StatusPublic:
		fallthrough
	case StatusOnline:
		node.netCancel()

		err := node.dht.Close()
		if err != nil {
			log.Printf("Error closing DHT: %s", err.Error())
		}
		node.dht = nil

		err = node.host.Close()
		if err != nil {
			log.Printf("Error closing host: %s", err.Error())
		}
		node.host = nil

		node.status = StatusOffline
		node.natCfg.Clear()
		log.Println("Node is offline")
		return err

	default:
		return nil
	}
}

// goOnline starts the network; if the node is already public, it stays public
func (node *Node) goOnline() error {
	node.mx.Lock()
	defer node.mx.Unlock()

	switch node.status {
	case StatusOffline:
		err := node._goOnline()
		if err != nil {
			return err
		}

		node.status = StatusOnline
		log.Println("Node is online")
		return nil

	default:
		return nil
	}
}

func (node *Node) _goOnline() error {
	var opts []interface{}
	if node.natCfg.Opt == mc.NATConfigAuto {
		opts = []interface{}{mc.NATPortMap}
	}

	ctx, cancel := context.WithCancel(context.Background())
	host, err := mc.NewHost(ctx, node.PeerIdentity, node.laddr, opts...)
	if err != nil {
		cancel()
		return err
	}

	host.SetStreamHandler("/mediachain/node/id", node.idHandler)
	host.SetStreamHandler("/mediachain/node/ping", node.pingHandler)
	host.SetStreamHandler("/mediachain/node/query", node.queryHandler)
	host.SetStreamHandler("/mediachain/node/data", node.dataHandler)
	host.SetStreamHandler("/mediachain/node/push", node.pushHandler)

	dht := NewDHT(ctx, host)

	err = dht.Bootstrap()
	if err != nil {
		// that's non-fatal, it will just fail to lookup
		log.Printf("Error boostrapping DHT: %s", err.Error())
	}

	node.host = host
	node.netCtx = ctx
	node.netCancel = cancel
	node.dht = dht

	return nil
}

// goPublic starts the network if it's not already up and registers with the
// directory; fails with NoDirectory if that hasn't been configured.
func (node *Node) goPublic() error {
	if node.dir == nil {
		return NoDirectory
	}

	node.mx.Lock()
	defer node.mx.Unlock()

	switch node.status {
	case StatusOffline:
		err := node._goOnline()
		if err != nil {
			return err
		}

		if node.natCfg.Opt == mc.NATConfigAuto {
			// wait a bit for NAT port mapping to take effect
			time.Sleep(2 * time.Second)
		}
		fallthrough

	case StatusOnline:
		go node.registerPeer(node.netCtx)
		node.status = StatusPublic

		log.Println("Node is public")
		return nil

	default:
		return nil
	}
}

func (node *Node) registerPeer(ctx context.Context) {
	for {
		err := node.registerPeerImpl(ctx)
		if err == nil {
			return
		}

		// sleep and retry
		select {
		case <-ctx.Done():
			return

		case <-time.After(5 * time.Minute):
			log.Println("Retrying to register with directory")
			continue
		}
	}
}

func (node *Node) registerPeerImpl(ctx context.Context) error {
	err := node.host.Connect(node.netCtx, *node.dir)
	if err != nil {
		log.Printf("Failed to connect to directory: %s", err.Error())
		return err
	}

	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/register")
	if err != nil {
		log.Printf("Failed to open directory stream: %s", err.Error())
		return err
	}
	defer s.Close()

	var pinfo = p2p_pstore.PeerInfo{ID: node.ID}
	var pbpi pb.PeerInfo

	w := ggio.NewDelimitedWriter(s)
	for {
		addrs := node.publicAddrs()

		if len(addrs) > 0 {
			pinfo.Addrs = addrs
			mc.PBFromPeerInfo(&pbpi, pinfo)
			msg := pb.RegisterPeer{Info: &pbpi}

			err = w.WriteMsg(&msg)
			if err != nil {
				log.Printf("Failed to register with directory: %s", err.Error())
				return err
			}
		} else {
			log.Printf("Skipped directory registration; no public address")
		}

		select {
		case <-ctx.Done():
			return nil

		case <-time.After(5 * time.Minute):
			continue
		}
	}
}

// The notion of public address is relative to the network location of the directory
// We want to support directories running on localhost (testing) or in a private network,
// and on the same time not leak internal addresses in public directory announcements.
// So, depending on the directory ip address:
//  If the directory is on the localhost, return the localhost address reported
//   by the host
//  If the directory is in a private range, filter Addrs reported by the
//   host, dropping unroutable addresses
//  If the directory is in a public range, then
//   If the NAT config is manual, return the configured address
//   If the NAT is auto or none, filter the addresses returned by the host,
//      and return only public addresses
func (node *Node) publicAddrs() []multiaddr.Multiaddr {
	if node.status == StatusOffline || node.dir == nil {
		return nil
	}

	dir := node.dir.Addrs[0]
	switch {
	case mc.IsLocalhostAddr(dir):
		return mc.FilterAddrs(node.host.Addrs(), mc.IsLocalhostAddr)

	case mc.IsPrivateAddr(dir):
		return mc.FilterAddrs(node.host.Addrs(), mc.IsRoutableAddr)

	default:
		switch node.natCfg.Opt {
		case mc.NATConfigManual:
			addr := node.natAddr()
			if addr == nil {
				return nil
			}
			return []multiaddr.Multiaddr{addr}

		default:
			return mc.FilterAddrs(node.host.Addrs(), mc.IsPublicAddr)
		}
	}
}

// netAddrs retrieves all routable addresses for the node, regardless of directory
// this includes the auto-detected or manually configured NAT address
func (node *Node) netAddrs() []multiaddr.Multiaddr {
	if node.status == StatusOffline {
		return nil
	}

	addrs := mc.FilterAddrs(node.host.Addrs(), mc.IsRoutableAddr)

	nataddr := node.natAddr()
	if nataddr != nil {
		addrs = append(addrs, nataddr)
	}

	return addrs
}

func (node *Node) natAddr() multiaddr.Multiaddr {
	if node.natCfg.Opt != mc.NATConfigManual {
		return nil
	}

	addr, err := node.natCfg.PublicAddr(node.laddr)
	if err != nil {
		log.Printf("Error determining pubic address: %s", err.Error())
	}
	return addr
}

func (node *Node) netPeerAddrs(pid p2p_peer.ID) []multiaddr.Multiaddr {
	if node.status == StatusOffline {
		return nil
	}

	pinfo := node.host.Peerstore().PeerInfo(pid)
	return pinfo.Addrs
}

func (node *Node) netConns() []p2p_pstore.PeerInfo {
	if node.status == StatusOffline {
		return nil
	}

	conns := node.host.Network().Conns()
	peers := make([]p2p_pstore.PeerInfo, len(conns))
	for x, conn := range conns {
		peers[x].ID = conn.RemotePeer()
		peers[x].Addrs = []multiaddr.Multiaddr{conn.RemoteMultiaddr()}
	}

	return peers
}

// Connectivity
func (node *Node) doConnect(ctx context.Context, pid p2p_peer.ID) error {
	if node.status == StatusOffline {
		return NodeOffline
	}

	if node.host.Network().Connectedness(pid) == p2p_net.Connected {
		return nil
	}

	addrs := node.host.Peerstore().Addrs(pid)
	if len(addrs) > 0 {
		return node.host.Connect(ctx, p2p_pstore.PeerInfo{pid, addrs})
	}

	pinfo, err := node.doLookup(ctx, pid)
	if err != nil {
		return err
	}

	return node.host.Connect(node.netCtx, pinfo)
}

func (node *Node) doLookup(ctx context.Context, pid p2p_peer.ID) (pinfo p2p_pstore.PeerInfo, err error) {
	pinfo, err = node.doLookupImpl(ctx, pid)
	if err == nil {
		node.host.Peerstore().AddAddrs(pid, pinfo.Addrs, p2p_pstore.ProviderAddrTTL)
	}

	return
}

func (node *Node) doLookupImpl(ctx context.Context, pid p2p_peer.ID) (pinfo p2p_pstore.PeerInfo, err error) {
	if node.status == StatusOffline {
		return pinfo, NodeOffline
	}

	if node.dir == nil {
		goto lookup_dht
	}

	pinfo, err = node.doDirLookup(ctx, pid)
	if err == nil {
		return
	}

	if err != UnknownPeer {
		log.Printf("Directory lookup error: %s", err.Error())
	}

lookup_dht:
	return node.dht.Lookup(ctx, pid)
}

func (node *Node) doDirLookup(ctx context.Context, pid p2p_peer.ID) (empty p2p_pstore.PeerInfo, err error) {
	if node.status == StatusOffline {
		return empty, NodeOffline
	}

	if node.dir == nil {
		return empty, NoDirectory
	}

	node.host.Connect(node.netCtx, *node.dir)
	if err != nil {
		return empty, err
	}

	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/lookup")
	if err != nil {
		return empty, err
	}
	defer s.Close()

	req := pb.LookupPeerRequest{pid.Pretty()}
	w := ggio.NewDelimitedWriter(s)
	err = w.WriteMsg(&req)
	if err != nil {
		return empty, err
	}

	var resp pb.LookupPeerResponse
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	err = r.ReadMsg(&resp)
	if err != nil {
		return empty, err
	}

	if resp.Peer == nil {
		return empty, UnknownPeer
	}

	pinfo, err := mc.PBToPeerInfo(resp.Peer)
	if err != nil {
		return empty, err
	}

	return pinfo, nil
}

func (node *Node) doDirList(ctx context.Context) ([]string, error) {
	if node.status == StatusOffline {
		return nil, NodeOffline
	}

	if node.dir == nil {
		return nil, NoDirectory
	}

	err := node.host.Connect(node.netCtx, *node.dir)
	if err != nil {
		return nil, err
	}

	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/list")
	if err != nil {
		return nil, err
	}
	defer s.Close()

	w := ggio.NewDelimitedWriter(s)
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)

	var req pb.ListPeersRequest
	var res pb.ListPeersResponse

	err = w.WriteMsg(&req)
	if err != nil {
		return nil, err
	}

	err = r.ReadMsg(&res)
	if err != nil {
		return nil, err
	}

	return res.Peers, nil
}
<|endoftext|>


<commit_before>package ld

// EmbedNode represents embed meta info
type EmbedNode struct {
	parent   interface{}
	property string
}

// FramingContext stores framing state
type FramingContext struct {
	embed       bool
	explicit    bool
	omitDefault bool
	embeds      map[string]*EmbedNode
}

// NewFramingContext creates and returns as new framing context.
func NewFramingContext(opts *JsonLdOptions) *FramingContext {
	context := &FramingContext{
		embed:       true,
		explicit:    false,
		omitDefault: false,
	}

	if opts != nil {
		context.embed = opts.Embed
		context.explicit = opts.Explicit
		context.omitDefault = opts.OmitDefault
	}

	return context
}

// Frame performs JSON-LD framing as defined in:
//
// http://json-ld.org/spec/latest/json-ld-framing/
//
// Frames the given input using the frame according to the steps in the Framing Algorithm.
// The input is used to build the framed output and is returned if there are no errors.
//
// Returns the framed output.
func (api *JsonLdApi) Frame(input interface{}, frame []interface{}, opts *JsonLdOptions) ([]interface{}, error) {
	idGen := NewBlankNodeIDGenerator()

	// create framing state
	state := NewFramingContext(opts)

	nodes := make(map[string]interface{})
	api.GenerateNodeMap(input, nodes, "@default", nil, "", nil, idGen)
	nodeMap := nodes["@default"].(map[string]interface{})

	framed := make([]interface{}, 0)

	// NOTE: frame validation is done by the function not allowing anything
	// other than list to be passed
	var frameParam map[string]interface{}
	if frame != nil && len(frame) > 0 {
		frameParam = frame[0].(map[string]interface{})
	} else {
		frameParam = make(map[string]interface{})
	}
	framedObj, _ := api.frame(state, nodeMap, nodeMap, frameParam, framed, "")
	// because we know framed is an array, we can safely cast framedObj back to an array
	return framedObj.([]interface{}), nil
}

// frame subjects according to the given frame.
// state: the current framing state
// nodes:
// nodeMap: node map
// frame: the frame
// parent: the parent subject or top-level array
// property: the parent property, initialized to nil
func (api *JsonLdApi) frame(state *FramingContext, nodes map[string]interface{}, nodeMap map[string]interface{},
	frame map[string]interface{}, parent interface{}, property string) (interface{}, error) {

	// filter out subjects that match the frame
	matches, _ := FilterNodes(nodes, frame)

	// get flags for current frame
	embedOn := GetFrameFlag(frame, "@embed", state.embed)
	explicitOn := GetFrameFlag(frame, "@explicit", state.explicit)

	// add matches to output
	for _, id := range GetOrderedKeys(matches) {
		if property == "" {
			state.embeds = make(map[string]*EmbedNode)
		}

		// start output
		output := make(map[string]interface{})
		output["@id"] = id

		// prepare embed meta info
		embeddedNode := &EmbedNode{}
		embeddedNode.parent = parent
		embeddedNode.property = property

		// if embed is on and there is an existing embed
		if existing, hasID := state.embeds[id]; embedOn && hasID {
			embedOn = false

			if parentList, isList := existing.parent.([]interface{}); isList {
				for _, p := range parentList {
					if CompareValues(output, p) {
						embedOn = true
						break
					}
				}
			} else {
				// existing embed's parent is an object
				parentMap := existing.parent.(map[string]interface{})
				if propertyVal, hasProperty := parentMap[existing.property]; hasProperty {
					for _, v := range propertyVal.([]interface{}) {
						if vMap, isMap := v.(map[string]interface{}); isMap && vMap["@id"] == id {
							embedOn = true
							break
						}
					}
				}
			}

			// existing embed has already been added, so allow an overwrite
			if embedOn {
				removeEmbed(state, id)
			}
		}

		// not embedding, add output without any other properties
		if !embedOn {
			parent = addFrameOutput(parent, property, output)
		} else {
			// add embed meta info
			state.embeds[id] = embeddedNode

			// iterate over subject properties
			element := matches[id].(map[string]interface{})
			for _, prop := range GetOrderedKeys(element) {

				// copy keywords to output
				if IsKeyword(prop) {
					output[prop] = CloneDocument(element[prop])
					continue
				}

				// if property isn't in the frame
				if _, containsProp := frame[prop]; !containsProp {
					// if explicit is off, embed values
					if !explicitOn {
						api.embedValues(state, nodeMap, element, prop, output)
					}
					continue
				}

				// add objects
				value := element[prop].([]interface{})

				for _, item := range value {
					// recurse into list
					itemMap, isMap := item.(map[string]interface{})
					listValue, hasList := itemMap["@list"]
					if isMap && hasList {
						// add empty list
						list := make(map[string]interface{})
						list["@list"] = make([]interface{}, 0)
						addFrameOutput(output, prop, list)

						// add list objects
						for _, listitem := range listValue.([]interface{}) {
							// recurse into subject reference
							if IsNodeReference(listitem) {
								tmp := make(map[string]interface{})
								itemid := listitem.(map[string]interface{})["id"].(string)
								// TODO: nodes may need to be node_map,
								// which is global
								tmp[itemid] = nodeMap[itemid]
								api.frame(state, tmp, nodeMap, frame[prop].([]interface{})[0].(map[string]interface{}),
									list, "@list")
							} else {
								// include other values automatcially (TODO:
								// may need Clone(n)
								addFrameOutput(list, "@list", listitem)
							}
						}
					} else if IsNodeReference(item) {
						// recurse into subject reference
						tmp := make(map[string]interface{})
						itemid := itemMap["@id"].(string)
						// TODO: nodes may need to be node_map, which is
						// global
						tmp[itemid] = nodeMap[itemid]
						api.frame(state, tmp, nodeMap, frame[prop].([]interface{})[0].(map[string]interface{}), output,
							prop)
					} else {
						// include other values automatically (TODO: may
						// need Clone(o)
						addFrameOutput(output, prop, item)
					}
				}
			}

			// handle defaults
			for _, prop := range GetOrderedKeys(frame) {
				// skip keywords
				if IsKeyword(prop) {
					continue
				}

				pf := frame[prop].([]interface{})
				var propertyFrame map[string]interface{}
				if len(pf) > 0 {
					propertyFrame = pf[0].(map[string]interface{})
				}

				if propertyFrame == nil {
					propertyFrame = make(map[string]interface{})
				}

				omitDefaultOn := GetFrameFlag(propertyFrame, "@omitDefault", state.omitDefault)
				if _, hasProp := output[prop]; !omitDefaultOn && !hasProp {
					var def interface{} = "@null"
					if defaultVal, hasDefault := propertyFrame["@default"]; hasDefault {
						def = CloneDocument(defaultVal)
					}
					if _, isList := def.([]interface{}); !isList {
						def = []interface{}{def}
					}
					output[prop] = []interface{}{
						map[string]interface{}{
							"@preserve": def,
						},
					}
				}
			}

			// add output to parent
			parent = addFrameOutput(parent, property, output)
		}
	}
	return parent, nil
}

// GetFrameFlag gets the frame flag value for the given flag name.
// If boolean value is not found, returns theDefault
func GetFrameFlag(frame map[string]interface{}, name string, theDefault bool) bool {
	value := frame[name]
	switch v := value.(type) {
	case []interface{}:
		if len(v) > 0 {
			value = v[0]
		}
	case map[string]interface{}:
		if valueVal, present := v["@value"]; present {
			value = valueVal
		}
	case bool:
		return v
	}

	if valueBool, isBool := value.(bool); isBool {
		return valueBool
	}

	return theDefault
}

// removeEmbed removes an existing embed with the given id.
func removeEmbed(state *FramingContext, id string) {
	// get existing embed
	embeds := state.embeds
	embed := embeds[id]
	parent := embed.parent
	property := embed.property

	// create reference to replace embed
	node := make(map[string]interface{})
	node["@id"] = id

	// remove existing embed
	if IsNode(parent) {
		// replace subject with reference
		newVals := make([]interface{}, 0)
		parentMap := parent.(map[string]interface{})
		oldvals := parentMap[property].([]interface{})
		for _, v := range oldvals {
			vMap, isMap := v.(map[string]interface{})
			if isMap && vMap["@id"] == id {
				newVals = append(newVals, node)
			} else {
				newVals = append(newVals, v)
			}
		}
		parentMap[property] = newVals
	}
	// recursively remove dependent dangling embeds
	removeDependents(embeds, id)
}

// removeDependents recursively removes dependent dangling embeds.
func removeDependents(embeds map[string]*EmbedNode, id string) {
	// get embed keys as a separate array to enable deleting keys in map
	for idDep, e := range embeds {
		var p map[string]interface{}
		if e.parent != nil {
			var isMap bool
			p, isMap = e.parent.(map[string]interface{})
			if !isMap {
				continue
			}
		} else {
			p = make(map[string]interface{})
		}

		pid := p["@id"].(string)
		if id == pid {
			delete(embeds, idDep)
			removeDependents(embeds, idDep)
		}
	}
}

// FilterNodes returns a map of all of the nodes that match a parsed frame.
func FilterNodes(nodes map[string]interface{}, frame map[string]interface{}) (map[string]interface{}, error) {
	rval := make(map[string]interface{})
	for id, elementVal := range nodes {
		element, _ := elementVal.(map[string]interface{})
		if element != nil {
			if res, err := FilterNode(element, frame); res {
				if err != nil {
					return nil, err
				}
				rval[id] = element
			}
		}
	}
	return rval, nil
}

// FilterNode returns true if the given node matches the given frame.
func FilterNode(node map[string]interface{}, frame map[string]interface{}) (bool, error) {
	types, _ := frame["@type"]
	if types != nil {
		typesList, isList := types.([]interface{})
		if !isList {
			return false, NewJsonLdError(SyntaxError, "frame @type must be an array")
		}
		nodeTypesVal, nodeHasType := node["@type"]
		var nodeTypes []interface{}
		if !nodeHasType {
			nodeTypes = make([]interface{}, 0)
		} else if nodeTypes, isList = nodeTypesVal.([]interface{}); !isList {
			return false, NewJsonLdError(SyntaxError, "node @type must be an array")
		}
		if len(typesList) == 1 {
			vMap, isMap := typesList[0].(map[string]interface{})
			if isMap && len(vMap) == 0 {
				return len(nodeTypes) > 0, nil
			}
		}

		for _, i := range nodeTypes {
			for _, j := range typesList {
				if DeepCompare(i, j, false) {
					return true, nil
				}
			}
		}
		return false, nil
	}

	for _, key := range GetKeys(frame) {
		_, nodeContainsKey := node[key]
		if key == "@id" || !IsKeyword(key) && !nodeContainsKey {
			return false, nil
		}
	}
	return true, nil
}

// addFrameOutput adds framing output to the given parent.
// parent: the parent to add to.
// property: the parent property.
// output: the output to add.
func addFrameOutput(parent interface{}, property string, output interface{}) interface{} {
	if parentMap, isMap := parent.(map[string]interface{}); isMap {
		propVal, hasProperty := parentMap[property]
		if hasProperty {
			parentMap[property] = append(propVal.([]interface{}), output)
		} else {
			parentMap[property] = []interface{}{output}
		}
		return parentMap
	}

	return append(parent.([]interface{}), output)
}

// embedValues embeds values for the given subject [element] and property into the given output
// during the framing algorithm.
func (api *JsonLdApi) embedValues(state *FramingContext, nodeMap map[string]interface{},
	element map[string]interface{}, property string, output interface{}) {
	// embed subject properties in output
	objects := element[property].([]interface{})
	for _, o := range objects {
		// handle subject reference
		if IsNodeReference(o) {
			oMap := o.(map[string]interface{})
			sid := oMap["@id"].(string)

			// embed full subject if isn't already embedded
			if _, hasSID := state.embeds[sid]; !hasSID {
				// add embed
				state.embeds[sid] = &EmbedNode{
					parent:   output,
					property: property,
				}

				// recurse into subject
				o = make(map[string]interface{})
				s, hasSID := nodeMap[sid]
				sMap, isMap := s.(map[string]interface{})
				if !hasSID || !isMap {
					sMap = map[string]interface{}{
						"@id": sid,
					}
				}
				for prop, propValue := range sMap {
					// copy keywords
					if IsKeyword(prop) {
						o.(map[string]interface{})[prop] = CloneDocument(propValue)
						continue
					}
					api.embedValues(state, nodeMap, sMap, prop, o)
				}
			}
			addFrameOutput(output, property, o)
		} else {
			// copy non-subject value
			addFrameOutput(output, property, CloneDocument(o))
		}
	}
}
<commit_msg>Fix an issue with @list in framing algorithm<commit_after>package ld

// EmbedNode represents embed meta info
type EmbedNode struct {
	parent   interface{}
	property string
}

// FramingContext stores framing state
type FramingContext struct {
	embed       bool
	explicit    bool
	omitDefault bool
	embeds      map[string]*EmbedNode
}

// NewFramingContext creates and returns as new framing context.
func NewFramingContext(opts *JsonLdOptions) *FramingContext {
	context := &FramingContext{
		embed:       true,
		explicit:    false,
		omitDefault: false,
	}

	if opts != nil {
		context.embed = opts.Embed
		context.explicit = opts.Explicit
		context.omitDefault = opts.OmitDefault
	}

	return context
}

// Frame performs JSON-LD framing as defined in:
//
// http://json-ld.org/spec/latest/json-ld-framing/
//
// Frames the given input using the frame according to the steps in the Framing Algorithm.
// The input is used to build the framed output and is returned if there are no errors.
//
// Returns the framed output.
func (api *JsonLdApi) Frame(input interface{}, frame []interface{}, opts *JsonLdOptions) ([]interface{}, error) {
	idGen := NewBlankNodeIDGenerator()

	// create framing state
	state := NewFramingContext(opts)

	nodes := make(map[string]interface{})
	api.GenerateNodeMap(input, nodes, "@default", nil, "", nil, idGen)
	nodeMap := nodes["@default"].(map[string]interface{})

	framed := make([]interface{}, 0)

	// NOTE: frame validation is done by the function not allowing anything
	// other than list to be passed
	var frameParam map[string]interface{}
	if frame != nil && len(frame) > 0 {
		frameParam = frame[0].(map[string]interface{})
	} else {
		frameParam = make(map[string]interface{})
	}
	framedObj, _ := api.frame(state, nodeMap, nodeMap, frameParam, framed, "")
	// because we know framed is an array, we can safely cast framedObj back to an array
	return framedObj.([]interface{}), nil
}

// frame subjects according to the given frame.
// state: the current framing state
// nodes:
// nodeMap: node map
// frame: the frame
// parent: the parent subject or top-level array
// property: the parent property, initialized to nil
func (api *JsonLdApi) frame(state *FramingContext, nodes map[string]interface{}, nodeMap map[string]interface{},
	frame map[string]interface{}, parent interface{}, property string) (interface{}, error) {

	// filter out subjects that match the frame
	matches, err := FilterNodes(nodes, frame)
	if err != nil {
		return nil, err
	}

	// get flags for current frame
	embedOn := GetFrameFlag(frame, "@embed", state.embed)
	explicitOn := GetFrameFlag(frame, "@explicit", state.explicit)

	// add matches to output
	for _, id := range GetOrderedKeys(matches) {
		if property == "" {
			state.embeds = make(map[string]*EmbedNode)
		}

		// start output
		output := make(map[string]interface{})
		output["@id"] = id

		// prepare embed meta info
		embeddedNode := &EmbedNode{}
		embeddedNode.parent = parent
		embeddedNode.property = property

		// if embed is on and there is an existing embed
		if existing, hasID := state.embeds[id]; embedOn && hasID {
			embedOn = false

			if parentList, isList := existing.parent.([]interface{}); isList {
				for _, p := range parentList {
					if CompareValues(output, p) {
						embedOn = true
						break
					}
				}
			} else {
				// existing embed's parent is an object
				parentMap := existing.parent.(map[string]interface{})
				if propertyVal, hasProperty := parentMap[existing.property]; hasProperty {
					for _, v := range propertyVal.([]interface{}) {
						if vMap, isMap := v.(map[string]interface{}); isMap && vMap["@id"] == id {
							embedOn = true
							break
						}
					}
				}
			}

			// existing embed has already been added, so allow an overwrite
			if embedOn {
				removeEmbed(state, id)
			}
		}

		// not embedding, add output without any other properties
		if !embedOn {
			parent = addFrameOutput(parent, property, output)
		} else {
			// add embed meta info
			state.embeds[id] = embeddedNode

			// iterate over subject properties
			element := matches[id].(map[string]interface{})
			for _, prop := range GetOrderedKeys(element) {

				// copy keywords to output
				if IsKeyword(prop) {
					output[prop] = CloneDocument(element[prop])
					continue
				}

				// if property isn't in the frame
				if _, containsProp := frame[prop]; !containsProp {
					// if explicit is off, embed values
					if !explicitOn {
						api.embedValues(state, nodeMap, element, prop, output)
					}
					continue
				}

				// add objects
				value := element[prop].([]interface{})

				for _, item := range value {
					// recurse into list
					itemMap, isMap := item.(map[string]interface{})
					listValue, hasList := itemMap["@list"]
					if isMap && hasList {
						// add empty list
						list := make(map[string]interface{})
						list["@list"] = make([]interface{}, 0)
						addFrameOutput(output, prop, list)

						// add list objects
						for _, listitem := range listValue.([]interface{}) {
							// recurse into subject reference
							if IsNodeReference(listitem) {
								tmp := make(map[string]interface{})
								itemid := listitem.(map[string]interface{})["@id"].(string)
								// TODO: nodes may need to be node_map,
								// which is global
								tmp[itemid] = nodeMap[itemid]
								api.frame(state, tmp, nodeMap, frame[prop].([]interface{})[0].(map[string]interface{}),
									list, "@list")
							} else {
								// include other values automatcially (TODO:
								// may need Clone(n)
								addFrameOutput(list, "@list", listitem)
							}
						}
					} else if IsNodeReference(item) {
						// recurse into subject reference
						tmp := make(map[string]interface{})
						itemid := itemMap["@id"].(string)
						// TODO: nodes may need to be node_map, which is
						// global
						tmp[itemid] = nodeMap[itemid]
						api.frame(state, tmp, nodeMap, frame[prop].([]interface{})[0].(map[string]interface{}), output,
							prop)
					} else {
						// include other values automatically (TODO: may
						// need Clone(o)
						addFrameOutput(output, prop, item)
					}
				}
			}

			// handle defaults
			for _, prop := range GetOrderedKeys(frame) {
				// skip keywords
				if IsKeyword(prop) {
					continue
				}

				pf := frame[prop].([]interface{})
				var propertyFrame map[string]interface{}
				if len(pf) > 0 {
					propertyFrame = pf[0].(map[string]interface{})
				}

				if propertyFrame == nil {
					propertyFrame = make(map[string]interface{})
				}

				omitDefaultOn := GetFrameFlag(propertyFrame, "@omitDefault", state.omitDefault)
				if _, hasProp := output[prop]; !omitDefaultOn && !hasProp {
					var def interface{} = "@null"
					if defaultVal, hasDefault := propertyFrame["@default"]; hasDefault {
						def = CloneDocument(defaultVal)
					}
					if _, isList := def.([]interface{}); !isList {
						def = []interface{}{def}
					}
					output[prop] = []interface{}{
						map[string]interface{}{
							"@preserve": def,
						},
					}
				}
			}

			// add output to parent
			parent = addFrameOutput(parent, property, output)
		}
	}
	return parent, nil
}

// GetFrameFlag gets the frame flag value for the given flag name.
// If boolean value is not found, returns theDefault
func GetFrameFlag(frame map[string]interface{}, name string, theDefault bool) bool {
	value := frame[name]
	switch v := value.(type) {
	case []interface{}:
		if len(v) > 0 {
			value = v[0]
		}
	case map[string]interface{}:
		if valueVal, present := v["@value"]; present {
			value = valueVal
		}
	case bool:
		return v
	}

	if valueBool, isBool := value.(bool); isBool {
		return valueBool
	}

	return theDefault
}

// removeEmbed removes an existing embed with the given id.
func removeEmbed(state *FramingContext, id string) {
	// get existing embed
	embeds := state.embeds
	embed := embeds[id]
	parent := embed.parent
	property := embed.property

	// create reference to replace embed
	node := make(map[string]interface{})
	node["@id"] = id

	// remove existing embed
	if IsNode(parent) {
		// replace subject with reference
		newVals := make([]interface{}, 0)
		parentMap := parent.(map[string]interface{})
		oldvals := parentMap[property].([]interface{})
		for _, v := range oldvals {
			vMap, isMap := v.(map[string]interface{})
			if isMap && vMap["@id"] == id {
				newVals = append(newVals, node)
			} else {
				newVals = append(newVals, v)
			}
		}
		parentMap[property] = newVals
	}
	// recursively remove dependent dangling embeds
	removeDependents(embeds, id)
}

// removeDependents recursively removes dependent dangling embeds.
func removeDependents(embeds map[string]*EmbedNode, id string) {
	// get embed keys as a separate array to enable deleting keys in map
	for idDep, e := range embeds {
		var p map[string]interface{}
		if e.parent != nil {
			var isMap bool
			p, isMap = e.parent.(map[string]interface{})
			if !isMap {
				continue
			}
		} else {
			p = make(map[string]interface{})
		}

		pid := p["@id"].(string)
		if id == pid {
			delete(embeds, idDep)
			removeDependents(embeds, idDep)
		}
	}
}

// FilterNodes returns a map of all of the nodes that match a parsed frame.
func FilterNodes(nodes map[string]interface{}, frame map[string]interface{}) (map[string]interface{}, error) {
	rval := make(map[string]interface{})
	for id, elementVal := range nodes {
		element, _ := elementVal.(map[string]interface{})
		if element != nil {
			if res, err := FilterNode(element, frame); res {
				if err != nil {
					return nil, err
				}
				rval[id] = element
			}
		}
	}
	return rval, nil
}

// FilterNode returns true if the given node matches the given frame.
func FilterNode(node map[string]interface{}, frame map[string]interface{}) (bool, error) {
	types, _ := frame["@type"]
	if types != nil {
		typesList, isList := types.([]interface{})
		if !isList {
			return false, NewJsonLdError(SyntaxError, "frame @type must be an array")
		}
		nodeTypesVal, nodeHasType := node["@type"]
		var nodeTypes []interface{}
		if !nodeHasType {
			nodeTypes = make([]interface{}, 0)
		} else if nodeTypes, isList = nodeTypesVal.([]interface{}); !isList {
			return false, NewJsonLdError(SyntaxError, "node @type must be an array")
		}
		if len(typesList) == 1 {
			vMap, isMap := typesList[0].(map[string]interface{})
			if isMap && len(vMap) == 0 {
				return len(nodeTypes) > 0, nil
			}
		}

		for _, i := range nodeTypes {
			for _, j := range typesList {
				if DeepCompare(i, j, false) {
					return true, nil
				}
			}
		}
		return false, nil
	}

	for _, key := range GetKeys(frame) {
		_, nodeContainsKey := node[key]
		if key == "@id" || !IsKeyword(key) && !nodeContainsKey {
			return false, nil
		}
	}
	return true, nil
}

// addFrameOutput adds framing output to the given parent.
// parent: the parent to add to.
// property: the parent property.
// output: the output to add.
func addFrameOutput(parent interface{}, property string, output interface{}) interface{} {
	if parentMap, isMap := parent.(map[string]interface{}); isMap {
		propVal, hasProperty := parentMap[property]
		if hasProperty {
			parentMap[property] = append(propVal.([]interface{}), output)
		} else {
			parentMap[property] = []interface{}{output}
		}
		return parentMap
	}

	return append(parent.([]interface{}), output)
}

// embedValues embeds values for the given subject [element] and property into the given output
// during the framing algorithm.
func (api *JsonLdApi) embedValues(state *FramingContext, nodeMap map[string]interface{},
	element map[string]interface{}, property string, output interface{}) {
	// embed subject properties in output
	objects := element[property].([]interface{})
	for _, o := range objects {
		oMap, isMap := o.(map[string]interface{})
		_, hasList := oMap["@list"]
		if isMap && hasList {
			list := make(map[string]interface{})
			list["@list"] = make([]interface{}, 0)
			api.embedValues(state, nodeMap, oMap, "@list", list)
			addFrameOutput(output, property, list)
		} else if IsNodeReference(o) {
			// handle subject reference
			oMap := o.(map[string]interface{})
			sid := oMap["@id"].(string)

			// embed full subject if isn't already embedded
			if _, hasSID := state.embeds[sid]; !hasSID {
				// add embed
				state.embeds[sid] = &EmbedNode{
					parent:   output,
					property: property,
				}

				// recurse into subject
				o = make(map[string]interface{})
				s, hasSID := nodeMap[sid]
				sMap, isMap := s.(map[string]interface{})
				if !hasSID || !isMap {
					sMap = map[string]interface{}{
						"@id": sid,
					}
				}
				for prop, propValue := range sMap {
					// copy keywords
					if IsKeyword(prop) {
						o.(map[string]interface{})[prop] = CloneDocument(propValue)
						continue
					}
					api.embedValues(state, nodeMap, sMap, prop, o)
				}
			}
			addFrameOutput(output, property, o)
		} else {
			// copy non-subject value
			addFrameOutput(output, property, CloneDocument(o))
		}
	}
}
<|endoftext|>


<commit_before>package models

import (
	"fmt"
	tm "github.com/buger/goterm"
	"sync"
	"time"
)

type Stage struct {
	Name         string
	PulpRootNode *Node
	Leafs        []*Node
	Nodes        []*Node
}

// Matches the given fqdn?
func (s Stage) MatchName(name string) bool {
	if s.Name == name {
		return true
	} else {
		return false
	}
}

func (s *Stage) NodeTreeWalker(node *Node, f func(*Node)) {
	f(node)
	for _, n := range node.Children {
		s.NodeTreeWalker(n, f)
	}
}

func (s *Stage) Init() {
	pos := 1
	s.NodeTreeWalker(s.PulpRootNode, func(node *Node) {
		s.Nodes = append(s.Nodes, node)
		// set treePosition
		node.TreePosition = pos
		pos++
		// set the leafs
		if node.IsLeaf() {
			s.Leafs = append(s.Leafs, node)
		}
		for _, n := range node.Children {
			// set the depth
			n.Depth = node.Depth + 1
			// set the parent node
			n.Parent = node
		}
	})
}

func (s *Stage) GetNodeByFqdn(nodeFqdn string) (node *Node) {
	s.NodeTreeWalker(s.PulpRootNode, func(n *Node) {
		if n.Fqdn == nodeFqdn {
			node = n
		}
	})
	return node
}

func (s *Stage) SyncedNodeTreeWalker(f func(n *Node) error) {
	s.Init()
	// initialize the tree (waitgroups, prents, depth, etc)
	inWg := make(map[string]*sync.WaitGroup)
	s.NodeTreeWalker(s.PulpRootNode, func(n *Node) {
		var wg sync.WaitGroup
		inWg[n.Fqdn] = &wg
		inWg[n.Fqdn].Add(1)
	})

	// Set a waitgroup for synconization of compteted leafs
	var leafsWaitGroup sync.WaitGroup
	leafsCount := len(s.Leafs)
	leafsWaitGroup.Add(leafsCount)
	// Walk the tree with syncronization
	s.NodeTreeWalker(s.PulpRootNode, func(n *Node) {
		go func() {
			time.Sleep(time.Millisecond * 50)
			// Wait
			inWg[n.Fqdn].Wait()
			// execute the function
			err := f(n)
			if err != nil {
				// log.Error.Println(err)
				n.Errors = append(n.Errors, err)
			}
			// Set done on waitgroup
			if n.IsLeaf() {
				leafsWaitGroup.Done()
			}
			// set Done on each child unlock start
			for _, child := range n.Children {
				inWg[child.Fqdn].Done()
			}

		}()
	})
	// start the execucution on root node
	inWg[s.PulpRootNode.Fqdn].Done()
	// Wait on all leafs to complete
	leafsWaitGroup.Wait()
}

func (s *Stage) Sync(repository string) (err error) {
	// Use the synced walk
	s.SyncedNodeTreeWalker(func(n *Node) (err error) {
		// Create a progress channel
		progressChannel := make(chan SyncProgress)
		// Read the progressChannel for this node until it's closed
		go func() {
			state := "init"
			for sp := range progressChannel {
				switch sp.State {
				case "skipped":
					line := fmt.Sprintf("%v %v", n.GetTreeRaw(n.Fqdn), sp.State)
					tm.Printf(tm.Color(tm.Bold(line), tm.MAGENTA))
					tm.Flush()
				case "error":
					line := fmt.Sprintf("%v %v", n.GetTreeRaw(n.Fqdn), sp.State)
					tm.Printf(tm.Color(tm.Bold(line), tm.RED))
					tm.Flush()
				case "running":
					if state != sp.State {
						line := fmt.Sprintf("%v %v", n.GetTreeRaw(n.Fqdn), sp.State)
						tm.Printf(tm.Color(line, tm.BLUE))
						tm.Flush()
					}
					state = sp.State
				case "finished":
					line := fmt.Sprintf("%v %v", n.GetTreeRaw(n.Fqdn), sp.State)
					tm.Printf(tm.Color(tm.Bold(line), tm.GREEN))
					tm.Flush()
				}
			}
		}()
		// Execute the sync
		n.Sync(repository, progressChannel)
		if err != nil {
			return err
		}
		return
	})
	return
}

func (s *Stage) Show() {
	s.Init()
	s.NodeTreeWalker(s.PulpRootNode, func(n *Node) {
		n.Show()
	})
}

// get a filtered stage.
func (s *Stage) Filter(nodeFqdns []string, nodeTags []string) (filteredStage *Stage) {
	s.Init()
	filteredStage = s
	s.NodeTreeWalker(filteredStage.PulpRootNode, func(n *Node) {
		childsToKeep := []*Node{}
		for _, child := range n.Children {
			if child.FqdnsAreDescendant(nodeFqdns) ||
				child.MatchFqdns(nodeFqdns) ||
				child.TagsInDescendant(nodeTags) ||
				child.ContainsTags(nodeTags) {
				childsToKeep = append(childsToKeep, child)
			}
		}
		n.Children = childsToKeep
	})
	return filteredStage
}
<commit_msg>fix filtered leafs<commit_after>package models

import (
	"fmt"
	tm "github.com/buger/goterm"
	"sync"
	"time"
)

type Stage struct {
	Name         string
	PulpRootNode *Node
	Leafs        []*Node
	Nodes        []*Node
}

// Matches the given fqdn?
func (s Stage) MatchName(name string) bool {
	if s.Name == name {
		return true
	} else {
		return false
	}
}

func (s *Stage) NodeTreeWalker(node *Node, f func(*Node)) {
	f(node)
	for _, n := range node.Children {
		s.NodeTreeWalker(n, f)
	}
}

func (s *Stage) Init() {
	pos := 1
	s.NodeTreeWalker(s.PulpRootNode, func(node *Node) {
		s.Nodes = append(s.Nodes, node)
		// set treePosition
		node.TreePosition = pos
		pos++
		// set the leafs
		if node.IsLeaf() {
			s.Leafs = append(s.Leafs, node)
		}
		for _, n := range node.Children {
			// set the depth
			n.Depth = node.Depth + 1
			// set the parent node
			n.Parent = node
		}
	})
}

func (s *Stage) GetNodeByFqdn(nodeFqdn string) (node *Node) {
	s.NodeTreeWalker(s.PulpRootNode, func(n *Node) {
		if n.Fqdn == nodeFqdn {
			node = n
		}
	})
	return node
}

func (s *Stage) SyncedNodeTreeWalker(f func(n *Node) error) {
	s.Init()
	// initialize the tree (waitgroups, prents, depth, etc)
	inWg := make(map[string]*sync.WaitGroup)
	s.NodeTreeWalker(s.PulpRootNode, func(n *Node) {
		var wg sync.WaitGroup
		inWg[n.Fqdn] = &wg
		inWg[n.Fqdn].Add(1)
	})

	// Set a waitgroup for synconization of compteted leafs
	var leafsWaitGroup sync.WaitGroup
	leafsCount := len(s.Leafs)
	leafsWaitGroup.Add(leafsCount)
	// Walk the tree with syncronization
	s.NodeTreeWalker(s.PulpRootNode, func(n *Node) {
		go func() {
			time.Sleep(time.Millisecond * 50)
			// Wait
			inWg[n.Fqdn].Wait()
			// execute the function
			err := f(n)
			if err != nil {
				// log.Error.Println(err)
				n.Errors = append(n.Errors, err)
			}
			// Set done on waitgroup
			if n.IsLeaf() {
				leafsWaitGroup.Done()
			}
			// set Done on each child unlock start
			for _, child := range n.Children {
				inWg[child.Fqdn].Done()
			}

		}()
	})
	// start the execucution on root node
	inWg[s.PulpRootNode.Fqdn].Done()
	// Wait on all leafs to complete
	leafsWaitGroup.Wait()
}

func (s *Stage) Sync(repository string) (err error) {
	// Use the synced walk
	s.SyncedNodeTreeWalker(func(n *Node) (err error) {
		// Create a progress channel
		progressChannel := make(chan SyncProgress)
		// Read the progressChannel for this node until it's closed
		go func() {
			state := "init"
			for sp := range progressChannel {
				switch sp.State {
				case "skipped":
					line := fmt.Sprintf("%v %v", n.GetTreeRaw(n.Fqdn), sp.State)
					tm.Printf(tm.Color(tm.Bold(line), tm.MAGENTA))
					tm.Flush()
				case "error":
					line := fmt.Sprintf("%v %v", n.GetTreeRaw(n.Fqdn), sp.State)
					tm.Printf(tm.Color(tm.Bold(line), tm.RED))
					tm.Flush()
				case "running":
					if state != sp.State {
						line := fmt.Sprintf("%v %v", n.GetTreeRaw(n.Fqdn), sp.State)
						tm.Printf(tm.Color(line, tm.BLUE))
						tm.Flush()
					}
					state = sp.State
				case "finished":
					line := fmt.Sprintf("%v %v", n.GetTreeRaw(n.Fqdn), sp.State)
					tm.Printf(tm.Color(tm.Bold(line), tm.GREEN))
					tm.Flush()
				}
			}
		}()
		// Execute the sync
		n.Sync(repository, progressChannel)
		if err != nil {
			return err
		}
		return
	})
	return
}

func (s *Stage) Show() {
	s.Init()
	s.NodeTreeWalker(s.PulpRootNode, func(n *Node) {
		n.Show()
	})
}

// get a filtered stage.
func (s *Stage) Filter(nodeFqdns []string, nodeTags []string) (filteredStage *Stage) {
	filteredStage = s
	s.NodeTreeWalker(filteredStage.PulpRootNode, func(n *Node) {
		childsToKeep := []*Node{}
		for _, child := range n.Children {
			if child.FqdnsAreDescendant(nodeFqdns) ||
				child.MatchFqdns(nodeFqdns) ||
				child.TagsInDescendant(nodeTags) ||
				child.ContainsTags(nodeTags) {
				childsToKeep = append(childsToKeep, child)
			}
		}
		n.Children = childsToKeep
	})
	return filteredStage
}
<|endoftext|>


<commit_before>package collision

import (
	"errors"

	"bitbucket.org/oakmoundstudio/oak/event"
)

// A Phase is a struct that other structs who want to use PhaseCollision
// should be composed of
type Phase struct {
	OnCollisionS *Space
	// If allocating maps becomes an issue
	// we can have two constant maps that we
	// switch between on alternating frames
	Touching map[Label]bool
}

func (cp *Phase) getCollisionPhase() *Phase {
	return cp
}

type collisionPhase interface {
	getCollisionPhase() *Phase
}

// PhaseCollision binds to the entity behind the space's CID so that it will
// recieve CollisionStart and CollisionStop events, appropriately when
// entities begin to collide or stop colliding with the space.
func PhaseCollision(s *Space) error {
	switch t := event.GetEntity(int(s.CID)).(type) {
	case collisionPhase:
		oc := t.getCollisionPhase()
		oc.OnCollisionS = s
		s.CID.Bind(phaseCollisionEnter, "EnterFrame")
		return nil
	}
	return errors.New("This space's entity does not implement OnCollisionyThing")
}

func phaseCollisionEnter(id int, nothing interface{}) int {
	e := event.GetEntity(id).(collisionPhase)
	oc := e.getCollisionPhase()

	// check hits
	hits := Hits(oc.OnCollisionS)
	newTouching := map[Label]bool{}

	// if any are new, trigger on collision
	for _, h := range hits {
		l := h.Label
		if _, ok := oc.Touching[l]; !ok {
			event.CID(id).Trigger("CollisionStart", l)
		}
		newTouching[l] = true
	}

	// if we lost any, trigger off collision
	for l := range oc.Touching {
		if _, ok := newTouching[l]; !ok {
			event.CID(id).Trigger("CollisionStop", l)
		}
	}

	oc.Touching = newTouching

	return 0
}
<commit_msg>Gave a tree argument to phase collision<commit_after>package collision

import (
	"errors"

	"bitbucket.org/oakmoundstudio/oak/event"
)

// A Phase is a struct that other structs who want to use PhaseCollision
// should be composed of
type Phase struct {
	OnCollisionS *Space
	tree         *Tree
	// If allocating maps becomes an issue
	// we can have two constant maps that we
	// switch between on alternating frames
	Touching map[Label]bool
}

func (cp *Phase) getCollisionPhase() *Phase {
	return cp
}

type collisionPhase interface {
	getCollisionPhase() *Phase
}

// PhaseCollision binds to the entity behind the space's CID so that it will
// recieve CollisionStart and CollisionStop events, appropriately when
// entities begin to collide or stop colliding with the space.
func PhaseCollision(s *Space, trees ...*Tree) error {
	switch t := event.GetEntity(int(s.CID)).(type) {
	case collisionPhase:
		oc := t.getCollisionPhase()
		oc.OnCollisionS = s
		if len(trees) > 0 {
			oc.tree = trees[0]
		} else {
			oc.tree = DefTree
		}
		s.CID.Bind(phaseCollisionEnter, "EnterFrame")
		return nil
	}
	return errors.New("This space's entity does not implement OnCollisionyThing")
}

func phaseCollisionEnter(id int, nothing interface{}) int {
	e := event.GetEntity(id).(collisionPhase)
	oc := e.getCollisionPhase()

	// check hits
	hits := oc.tree.Hits(oc.OnCollisionS)
	newTouching := map[Label]bool{}

	// if any are new, trigger on collision
	for _, h := range hits {
		l := h.Label
		if _, ok := oc.Touching[l]; !ok {
			event.CID(id).Trigger("CollisionStart", l)
		}
		newTouching[l] = true
	}

	// if we lost any, trigger off collision
	for l := range oc.Touching {
		if _, ok := newTouching[l]; !ok {
			event.CID(id).Trigger("CollisionStop", l)
		}
	}

	oc.Touching = newTouching

	return 0
}
<|endoftext|>


<commit_before>package td

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"
	"sync/atomic"
	"time"

	"github.com/google/uuid"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"golang.org/x/oauth2"
	compute "google.golang.org/api/compute/v1"
)

const (
	// PubSub topic name for task driver metadata.
	PUBSUB_TOPIC = "task-driver"
	// PubSub topic name for task driver logs.
	PUBSUB_TOPIC_LOGS = "task-driver-logs"

	// Log ID for all Task Drivers. Logs are labeled with task ID and step
	// ID as well, and those labels should be used for filtering in most
	// cases.
	LOG_ID = "task-driver"

	// Special ID of the root step.
	STEP_ID_ROOT = "root"

	// Environment variables provided to all Swarming tasks.
	ENVVAR_SWARMING_BOT    = "SWARMING_BOT_ID"
	ENVVAR_SWARMING_SERVER = "SWARMING_SERVER"
	ENVVAR_SWARMING_TASK   = "SWARMING_TASK_ID"

	// PATH_VAR represents the PATH environment variable.
	PATH_VAR = "PATH"
)

var (
	BASE_ENV = []string{
		"CHROME_HEADLESS=1",
		"GIT_USER_AGENT=git/1.9.1", // I don't think this version matters.
		fmt.Sprintf("%s=%s", PATH_VAR, os.Getenv(PATH_VAR)),
	}

	// Auth scopes required for all task_drivers.
	SCOPES = []string{compute.CloudPlatformScope}
)

// RunProperties are properties for a single run of a Task Driver.
type RunProperties struct {
	Local          bool   `json:"local"`
	SwarmingBot    string `json:"swarmingBot,omitempty"`
	SwarmingServer string `json:"swarmingServer,omitempty"`
	SwarmingTask   string `json:"swarmingTask,omitempty"`
}

// Return an error if the RunProperties are not valid.
func (p *RunProperties) Validate() error {
	if p.Local {
		if p.SwarmingBot != "" {
			return errors.New("SwarmingBot must be empty for local runs!")
		}
		if p.SwarmingServer != "" {
			return errors.New("SwarmingServer must be empty for local runs!")
		}
		if p.SwarmingTask != "" {
			return errors.New("SwarmingTask must be empty for local runs!")
		}
	} else {
		if p.SwarmingBot == "" {
			return errors.New("SwarmingBot is required for non-local runs!")
		}
		if p.SwarmingServer == "" {
			return errors.New("SwarmingServer is required for non-local runs!")
		}
		if p.SwarmingTask == "" {
			return errors.New("SwarmingTask is required for non-local runs!")
		}
	}
	return nil
}

// Return a copy of the RunProperties.
func (p *RunProperties) Copy() *RunProperties {
	if p == nil {
		return nil
	}
	return &RunProperties{
		Local:          p.Local,
		SwarmingBot:    p.SwarmingBot,
		SwarmingServer: p.SwarmingServer,
		SwarmingTask:   p.SwarmingTask,
	}
}

// StartRunWithErr begins a new test automation run, returning any error which
// occurs.
func StartRunWithErr(projectId, taskId, taskName, output *string, local *bool) (context.Context, error) {
	common.Init()

	// TODO(borenet): Catch SIGINT, SIGKILL and report.

	// Gather RunProperties.
	swarmingBot := os.Getenv(ENVVAR_SWARMING_BOT)
	swarmingServer := os.Getenv(ENVVAR_SWARMING_SERVER)
	swarmingTask := os.Getenv(ENVVAR_SWARMING_TASK)

	// "reproduce" is supplied by "swarming.py reproduce" and indicates that
	// this is actually a local run, but --local won't have been provided
	// because the command was copied directly from the Swarming task.
	if swarmingTask == "reproduce" || swarmingBot == "reproduce" {
		*local = true
		swarmingBot = ""
		swarmingServer = ""
		swarmingTask = ""
	}
	if *local {
		// Check to make sure we're not actually running in production.
		// Note that the presence of SWARMING_SERVER does not indicate
		// that we're running in production, because it can be used with
		// swarming.py as an alternative to --swarming.
		errTmpl := "--local was supplied but %s environment variable was found. Was --local used by accident?"
		if swarmingBot != "" {
			return nil, fmt.Errorf(errTmpl, ENVVAR_SWARMING_BOT)
		} else if swarmingTask != "" {
			return nil, fmt.Errorf(errTmpl, ENVVAR_SWARMING_TASK)
		}

		// Prevent clobbering real task data for local tasks.
		hostname, err := os.Hostname()
		if err != nil {
			return nil, err
		}
		*taskId = fmt.Sprintf("%s_%s", hostname, uuid.New())
	} else {
		// Check to make sure that we're not running locally and the
		// user forgot to use --local.
		errTmpl := "--local was not supplied but environment variable %s was not found. Did you forget to use --local?"
		if swarmingBot == "" {
			return nil, fmt.Errorf(errTmpl, ENVVAR_SWARMING_BOT)
		} else if swarmingServer == "" {
			return nil, fmt.Errorf(errTmpl, ENVVAR_SWARMING_SERVER)
		} else if swarmingTask == "" {
			return nil, fmt.Errorf(errTmpl, ENVVAR_SWARMING_TASK)
		}
	}

	// Validate properties and flags.
	props := &RunProperties{
		Local:          *local,
		SwarmingBot:    swarmingBot,
		SwarmingServer: swarmingServer,
		SwarmingTask:   swarmingTask,
	}
	if err := props.Validate(); err != nil {
		return nil, err
	}
	if *projectId == "" {
		return nil, fmt.Errorf("Project ID is required.")
	}
	if *taskId == "" {
		return nil, fmt.Errorf("Task ID is required.")
	}
	if *taskName == "" {
		return nil, fmt.Errorf("Task name is required.")
	}

	// Create the token source.
	var ts oauth2.TokenSource
	if *local {
		var err error
		ts, err = auth.NewDefaultTokenSource(*local, SCOPES...)
		if err != nil {
			return nil, err
		}
	} else {
		var err error
		ts, err = auth.NewLUCIContextTokenSource(SCOPES...)
		if err != nil {
			return nil, fmt.Errorf("Failed to obtain LUCI TokenSource: %s", err)
		}
	}

	// Initialize Cloud Logging.
	labels := map[string]string{
		"taskId":   *taskId,
		"taskName": *taskName,
	}
	ctx := context.Background()
	logger, err := sklog.NewCloudLogger(ctx, *projectId, LOG_ID, ts, labels)
	if err != nil {
		return nil, err
	}
	sklog.SetLogger(logger)

	// Dump environment variables.
	sklog.Infof("Environment:\n%s", strings.Join(os.Environ(), "\n"))

	// Connect receivers.
	cloudLogging, err := NewCloudLoggingReceiver(logger.Logger())
	if err != nil {
		return nil, err
	}
	report := newReportReceiver(*output)
	receiver := MultiReceiver([]Receiver{
		cloudLogging,
		&DebugReceiver{},
		report,
	})

	// Set up and return the root-level Step.
	ctx = newRun(ctx, receiver, *taskId, *taskName, props)
	return ctx, nil
}

// StartRun begins a new test automation run, panicking if any setup fails.
func StartRun(projectId, taskId, taskName, output *string, local *bool) context.Context {
	ctx, err := StartRunWithErr(projectId, taskId, taskName, output, local)
	if err != nil {
		sklog.Fatalf("Failed task_driver.StartRun(): %s", err)
	}
	return ctx
}

// Perform any cleanup work for the run. Should be deferred in main().
func EndRun(ctx context.Context) {
	defer util.Close(getCtx(ctx).run)

	// Mark the root step as finished.
	finishStep(ctx, recover())
}

// run represents a full test automation run.
type run struct {
	receiver Receiver
	taskId   string
	msgIndex int32
}

// newRun returns a context.Context representing a Task Driver run, including
// creation of a root step.
func newRun(ctx context.Context, rec Receiver, taskId, taskName string, props *RunProperties) context.Context {
	r := &run{
		receiver: rec,
		taskId:   taskId,
	}
	r.send(&Message{
		Type: MSG_TYPE_RUN_STARTED,
		Run:  props,
	})
	ctx = context.WithValue(ctx, contextKey, &Context{
		run:     r,
		execRun: exec.DefaultRun,
	})
	ctx = newStep(ctx, STEP_ID_ROOT, nil, Props(taskName).Env(BASE_ENV))
	return ctx
}

// Send the given message to the receiver. Does not return an error, even if
// sending fails.
func (r *run) send(msg *Message) {
	msg.Index = int(atomic.AddInt32(&r.msgIndex, 1))
	msg.TaskId = r.taskId
	msg.Timestamp = time.Now().UTC()
	if err := msg.Validate(); err != nil {
		sklog.Error(err)
	}
	if err := r.receiver.HandleMessage(msg); err != nil {
		// Just log the error but don't return it.
		// TODO(borenet): How do we handle this?
		sklog.Error(err)
	}
}

// Send a Message indicating that a new step has started.
func (r *run) Start(props *StepProperties) {
	msg := &Message{
		Type:   MSG_TYPE_STEP_STARTED,
		StepId: props.Id,
		Step:   props,
	}
	r.send(msg)
}

// Send a Message with additional data for the current step.
func (r *run) AddStepData(id string, typ DataType, d interface{}) {
	msg := &Message{
		Type:     MSG_TYPE_STEP_DATA,
		StepId:   id,
		Data:     d,
		DataType: typ,
	}
	r.send(msg)
}

// Send a Message indicating that the current step has failed with the given
// error.
func (r *run) Failed(id string, err error) {
	msg := &Message{
		StepId: id,
		Error:  err.Error(),
	}
	if IsInfraError(err) {
		msg.Type = MSG_TYPE_STEP_EXCEPTION
	} else {
		msg.Type = MSG_TYPE_STEP_FAILED
	}
	r.send(msg)
}

// Send a Message indicating that the current step has finished.
func (r *run) Finish(id string) {
	msg := &Message{
		Type:   MSG_TYPE_STEP_FINISHED,
		StepId: id,
	}
	r.send(msg)
}

// Open a log stream.
func (r *run) LogStream(stepId, logName, severity string) io.Writer {
	logId := uuid.New().String() // TODO(borenet): Come up with a better ID.
	rv, err := r.receiver.LogStream(stepId, logId, severity)
	if err != nil {
		panic(err)
	}

	// Emit step data for the log stream.
	r.AddStepData(stepId, DATA_TYPE_LOG, &LogData{
		Name:     logName,
		Id:       logId,
		Severity: severity,
	})
	return rv
}

// Close the run.
func (r *run) Close() error {
	return r.receiver.Close()
}
<commit_msg>[task driver] Only send step data for logs which are actually used<commit_after>package td

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"
	"sync/atomic"
	"time"

	"github.com/google/uuid"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"golang.org/x/oauth2"
	compute "google.golang.org/api/compute/v1"
)

const (
	// PubSub topic name for task driver metadata.
	PUBSUB_TOPIC = "task-driver"
	// PubSub topic name for task driver logs.
	PUBSUB_TOPIC_LOGS = "task-driver-logs"

	// Log ID for all Task Drivers. Logs are labeled with task ID and step
	// ID as well, and those labels should be used for filtering in most
	// cases.
	LOG_ID = "task-driver"

	// Special ID of the root step.
	STEP_ID_ROOT = "root"

	// Environment variables provided to all Swarming tasks.
	ENVVAR_SWARMING_BOT    = "SWARMING_BOT_ID"
	ENVVAR_SWARMING_SERVER = "SWARMING_SERVER"
	ENVVAR_SWARMING_TASK   = "SWARMING_TASK_ID"

	// PATH_VAR represents the PATH environment variable.
	PATH_VAR = "PATH"
)

var (
	BASE_ENV = []string{
		"CHROME_HEADLESS=1",
		"GIT_USER_AGENT=git/1.9.1", // I don't think this version matters.
		fmt.Sprintf("%s=%s", PATH_VAR, os.Getenv(PATH_VAR)),
	}

	// Auth scopes required for all task_drivers.
	SCOPES = []string{compute.CloudPlatformScope}
)

// RunProperties are properties for a single run of a Task Driver.
type RunProperties struct {
	Local          bool   `json:"local"`
	SwarmingBot    string `json:"swarmingBot,omitempty"`
	SwarmingServer string `json:"swarmingServer,omitempty"`
	SwarmingTask   string `json:"swarmingTask,omitempty"`
}

// Return an error if the RunProperties are not valid.
func (p *RunProperties) Validate() error {
	if p.Local {
		if p.SwarmingBot != "" {
			return errors.New("SwarmingBot must be empty for local runs!")
		}
		if p.SwarmingServer != "" {
			return errors.New("SwarmingServer must be empty for local runs!")
		}
		if p.SwarmingTask != "" {
			return errors.New("SwarmingTask must be empty for local runs!")
		}
	} else {
		if p.SwarmingBot == "" {
			return errors.New("SwarmingBot is required for non-local runs!")
		}
		if p.SwarmingServer == "" {
			return errors.New("SwarmingServer is required for non-local runs!")
		}
		if p.SwarmingTask == "" {
			return errors.New("SwarmingTask is required for non-local runs!")
		}
	}
	return nil
}

// Return a copy of the RunProperties.
func (p *RunProperties) Copy() *RunProperties {
	if p == nil {
		return nil
	}
	return &RunProperties{
		Local:          p.Local,
		SwarmingBot:    p.SwarmingBot,
		SwarmingServer: p.SwarmingServer,
		SwarmingTask:   p.SwarmingTask,
	}
}

// StartRunWithErr begins a new test automation run, returning any error which
// occurs.
func StartRunWithErr(projectId, taskId, taskName, output *string, local *bool) (context.Context, error) {
	common.Init()

	// TODO(borenet): Catch SIGINT, SIGKILL and report.

	// Gather RunProperties.
	swarmingBot := os.Getenv(ENVVAR_SWARMING_BOT)
	swarmingServer := os.Getenv(ENVVAR_SWARMING_SERVER)
	swarmingTask := os.Getenv(ENVVAR_SWARMING_TASK)

	// "reproduce" is supplied by "swarming.py reproduce" and indicates that
	// this is actually a local run, but --local won't have been provided
	// because the command was copied directly from the Swarming task.
	if swarmingTask == "reproduce" || swarmingBot == "reproduce" {
		*local = true
		swarmingBot = ""
		swarmingServer = ""
		swarmingTask = ""
	}
	if *local {
		// Check to make sure we're not actually running in production.
		// Note that the presence of SWARMING_SERVER does not indicate
		// that we're running in production, because it can be used with
		// swarming.py as an alternative to --swarming.
		errTmpl := "--local was supplied but %s environment variable was found. Was --local used by accident?"
		if swarmingBot != "" {
			return nil, fmt.Errorf(errTmpl, ENVVAR_SWARMING_BOT)
		} else if swarmingTask != "" {
			return nil, fmt.Errorf(errTmpl, ENVVAR_SWARMING_TASK)
		}

		// Prevent clobbering real task data for local tasks.
		hostname, err := os.Hostname()
		if err != nil {
			return nil, err
		}
		*taskId = fmt.Sprintf("%s_%s", hostname, uuid.New())
	} else {
		// Check to make sure that we're not running locally and the
		// user forgot to use --local.
		errTmpl := "--local was not supplied but environment variable %s was not found. Did you forget to use --local?"
		if swarmingBot == "" {
			return nil, fmt.Errorf(errTmpl, ENVVAR_SWARMING_BOT)
		} else if swarmingServer == "" {
			return nil, fmt.Errorf(errTmpl, ENVVAR_SWARMING_SERVER)
		} else if swarmingTask == "" {
			return nil, fmt.Errorf(errTmpl, ENVVAR_SWARMING_TASK)
		}
	}

	// Validate properties and flags.
	props := &RunProperties{
		Local:          *local,
		SwarmingBot:    swarmingBot,
		SwarmingServer: swarmingServer,
		SwarmingTask:   swarmingTask,
	}
	if err := props.Validate(); err != nil {
		return nil, err
	}
	if *projectId == "" {
		return nil, fmt.Errorf("Project ID is required.")
	}
	if *taskId == "" {
		return nil, fmt.Errorf("Task ID is required.")
	}
	if *taskName == "" {
		return nil, fmt.Errorf("Task name is required.")
	}

	// Create the token source.
	var ts oauth2.TokenSource
	if *local {
		var err error
		ts, err = auth.NewDefaultTokenSource(*local, SCOPES...)
		if err != nil {
			return nil, err
		}
	} else {
		var err error
		ts, err = auth.NewLUCIContextTokenSource(SCOPES...)
		if err != nil {
			return nil, fmt.Errorf("Failed to obtain LUCI TokenSource: %s", err)
		}
	}

	// Initialize Cloud Logging.
	labels := map[string]string{
		"taskId":   *taskId,
		"taskName": *taskName,
	}
	ctx := context.Background()
	logger, err := sklog.NewCloudLogger(ctx, *projectId, LOG_ID, ts, labels)
	if err != nil {
		return nil, err
	}
	sklog.SetLogger(logger)

	// Dump environment variables.
	sklog.Infof("Environment:\n%s", strings.Join(os.Environ(), "\n"))

	// Connect receivers.
	cloudLogging, err := NewCloudLoggingReceiver(logger.Logger())
	if err != nil {
		return nil, err
	}
	report := newReportReceiver(*output)
	receiver := MultiReceiver([]Receiver{
		cloudLogging,
		&DebugReceiver{},
		report,
	})

	// Set up and return the root-level Step.
	ctx = newRun(ctx, receiver, *taskId, *taskName, props)
	return ctx, nil
}

// StartRun begins a new test automation run, panicking if any setup fails.
func StartRun(projectId, taskId, taskName, output *string, local *bool) context.Context {
	ctx, err := StartRunWithErr(projectId, taskId, taskName, output, local)
	if err != nil {
		sklog.Fatalf("Failed task_driver.StartRun(): %s", err)
	}
	return ctx
}

// Perform any cleanup work for the run. Should be deferred in main().
func EndRun(ctx context.Context) {
	defer util.Close(getCtx(ctx).run)

	// Mark the root step as finished.
	finishStep(ctx, recover())
}

// run represents a full test automation run.
type run struct {
	receiver Receiver
	taskId   string
	msgIndex int32
}

// newRun returns a context.Context representing a Task Driver run, including
// creation of a root step.
func newRun(ctx context.Context, rec Receiver, taskId, taskName string, props *RunProperties) context.Context {
	r := &run{
		receiver: rec,
		taskId:   taskId,
	}
	r.send(&Message{
		Type: MSG_TYPE_RUN_STARTED,
		Run:  props,
	})
	ctx = context.WithValue(ctx, contextKey, &Context{
		run:     r,
		execRun: exec.DefaultRun,
	})
	ctx = newStep(ctx, STEP_ID_ROOT, nil, Props(taskName).Env(BASE_ENV))
	return ctx
}

// Send the given message to the receiver. Does not return an error, even if
// sending fails.
func (r *run) send(msg *Message) {
	msg.Index = int(atomic.AddInt32(&r.msgIndex, 1))
	msg.TaskId = r.taskId
	msg.Timestamp = time.Now().UTC()
	if err := msg.Validate(); err != nil {
		sklog.Error(err)
	}
	if err := r.receiver.HandleMessage(msg); err != nil {
		// Just log the error but don't return it.
		// TODO(borenet): How do we handle this?
		sklog.Error(err)
	}
}

// Send a Message indicating that a new step has started.
func (r *run) Start(props *StepProperties) {
	msg := &Message{
		Type:   MSG_TYPE_STEP_STARTED,
		StepId: props.Id,
		Step:   props,
	}
	r.send(msg)
}

// Send a Message with additional data for the current step.
func (r *run) AddStepData(id string, typ DataType, d interface{}) {
	msg := &Message{
		Type:     MSG_TYPE_STEP_DATA,
		StepId:   id,
		Data:     d,
		DataType: typ,
	}
	r.send(msg)
}

// Send a Message indicating that the current step has failed with the given
// error.
func (r *run) Failed(id string, err error) {
	msg := &Message{
		StepId: id,
		Error:  err.Error(),
	}
	if IsInfraError(err) {
		msg.Type = MSG_TYPE_STEP_EXCEPTION
	} else {
		msg.Type = MSG_TYPE_STEP_FAILED
	}
	r.send(msg)
}

// Send a Message indicating that the current step has finished.
func (r *run) Finish(id string) {
	msg := &Message{
		Type:   MSG_TYPE_STEP_FINISHED,
		StepId: id,
	}
	r.send(msg)
}

// cbWriter is a wrapper around an io.Writer which runs a callback on the first
// call to Write().
type cbWriter struct {
	w  io.Writer
	cb func()
}

// See documentation for io.Writer interface.
func (w *cbWriter) Write(b []byte) (int, error) {
	if w.cb != nil {
		w.cb()
		w.cb = nil
	}
	return w.w.Write(b)
}

// Open a log stream.
func (r *run) LogStream(stepId, logName, severity string) io.Writer {
	logId := uuid.New().String() // TODO(borenet): Come up with a better ID.
	w, err := r.receiver.LogStream(stepId, logId, severity)
	if err != nil {
		panic(err)
	}

	// Wrap the io.Writer with a cbWrite so that we only send step data for
	// log streams which are actually used.
	return &cbWriter{
		w: w,
		cb: func() {
			// Emit step data for the log stream.
			r.AddStepData(stepId, DATA_TYPE_LOG, &LogData{
				Name:     logName,
				Id:       logId,
				Severity: severity,
			})
		},
	}
}

// Close the run.
func (r *run) Close() error {
	return r.receiver.Close()
}
<|endoftext|>