<commit_before>package commands

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/jbenet/go-ipfs/core"
	dag "github.com/jbenet/go-ipfs/merkledag"
)

// ObjectData takes a key string from args and writes out the raw bytes of that node (if there is one)
func ObjectData(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error {
	dagnode, err := n.Resolver.ResolvePath(args[0])
	if err != nil {
		return fmt.Errorf("objectData error: %v", err)
	}
	log.Debug("objectData: found dagnode %q (# of bytes: %d - # links: %d)", args[0], len(dagnode.Data), len(dagnode.Links))

	_, err = io.Copy(out, bytes.NewReader(dagnode.Data))
	return err
}

// ObjectLinks takes a key string from args and lists the links it points to
func ObjectLinks(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error {
	dagnode, err := n.Resolver.ResolvePath(args[0])
	if err != nil {
		return fmt.Errorf("objectLinks error: %v", err)
	}
	log.Debug("ObjectLinks: found dagnode %q (# of bytes: %d - # links: %d)", args[0], len(dagnode.Data), len(dagnode.Links))

	if len(dagnode.Links) == 0 {
		// TODO(cryptix): named error?
		return errors.New("objectLinks: no links in this node")
	}

	for _, link := range dagnode.Links {
		_, err = fmt.Fprintf(out, "%s %d %q\n", link.Hash.B58String(), link.Size, link.Name)
		if err != nil {
			break
		}
	}

	return err
}

// ErrUnknownObjectEnc is returned if a invalid encoding is supplied
var ErrUnknownObjectEnc = errors.New("unknown object encoding")

type objectEncoding string

const (
	objectEncodingJSON     objectEncoding = "json"
	objectEncodingProtobuf                = "protobuf"
)

func getObjectEnc(o interface{}) objectEncoding {
	v, ok := o.(string)
	if !ok {
		// chosen as default because it's human readable
		log.Warning("option is not a string - falling back to json")
		return objectEncodingJSON
	}

	return objectEncoding(v)
}

// ObjectGet takes a key string from args and a format option and serializes the dagnode to that format
func ObjectGet(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error {
	dagnode, err := n.Resolver.ResolvePath(args[0])
	if err != nil {
		return fmt.Errorf("ObjectGet error: %v", err)
	}
	log.Debug("objectGet: found dagnode %q (# of bytes: %d - # links: %d)", args[0], len(dagnode.Data), len(dagnode.Links))

	// sadly all encodings dont implement a common interface
	var data []byte
	switch getObjectEnc(opts["encoding"]) {
	case objectEncodingJSON:
		data, err = json.MarshalIndent(dagnode, "", "  ")

	case objectEncodingProtobuf:
		data, err = dagnode.Marshal()

	default:
		return ErrUnknownObjectEnc
	}

	if err != nil {
		return fmt.Errorf("ObjectGet error: %v", err)
	}

	_, err = io.Copy(out, bytes.NewReader(data))
	return err
}

// ObjectPut takes a format option, serilizes bytes from stdin and updates the dag with that data
func ObjectPut(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error {
	var (
		dagnode *dag.Node
		data    []byte
		err     error
	)

	data, err = ioutil.ReadAll(os.Stdin)
	if err != nil {
		return fmt.Errorf("ObjectPut error: %v", err)
	}

	switch getObjectEnc(opts["encoding"]) {
	case objectEncodingJSON:
		dagnode = new(dag.Node)
		err = json.Unmarshal(data, dagnode)

	case objectEncodingProtobuf:
		dagnode, err = dag.Decoded(data)

	default:
		return ErrUnknownObjectEnc
	}

	if err != nil {
		return fmt.Errorf("ObjectPut error: %v", err)
	}

	return addNode(n, dagnode, "stdin", out)
}
<commit_msg>no links, no error<commit_after>package commands

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/jbenet/go-ipfs/core"
	dag "github.com/jbenet/go-ipfs/merkledag"
)

// ObjectData takes a key string from args and writes out the raw bytes of that node (if there is one)
func ObjectData(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error {
	dagnode, err := n.Resolver.ResolvePath(args[0])
	if err != nil {
		return fmt.Errorf("objectData error: %v", err)
	}
	log.Debug("objectData: found dagnode %q (# of bytes: %d - # links: %d)", args[0], len(dagnode.Data), len(dagnode.Links))

	_, err = io.Copy(out, bytes.NewReader(dagnode.Data))
	return err
}

// ObjectLinks takes a key string from args and lists the links it points to
func ObjectLinks(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error {
	dagnode, err := n.Resolver.ResolvePath(args[0])
	if err != nil {
		return fmt.Errorf("objectLinks error: %v", err)
	}
	log.Debug("ObjectLinks: found dagnode %q (# of bytes: %d - # links: %d)", args[0], len(dagnode.Data), len(dagnode.Links))

	for _, link := range dagnode.Links {
		_, err = fmt.Fprintf(out, "%s %d %q\n", link.Hash.B58String(), link.Size, link.Name)
		if err != nil {
			break
		}
	}

	return err
}

// ErrUnknownObjectEnc is returned if a invalid encoding is supplied
var ErrUnknownObjectEnc = errors.New("unknown object encoding")

type objectEncoding string

const (
	objectEncodingJSON     objectEncoding = "json"
	objectEncodingProtobuf                = "protobuf"
)

func getObjectEnc(o interface{}) objectEncoding {
	v, ok := o.(string)
	if !ok {
		// chosen as default because it's human readable
		log.Warning("option is not a string - falling back to json")
		return objectEncodingJSON
	}

	return objectEncoding(v)
}

// ObjectGet takes a key string from args and a format option and serializes the dagnode to that format
func ObjectGet(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error {
	dagnode, err := n.Resolver.ResolvePath(args[0])
	if err != nil {
		return fmt.Errorf("ObjectGet error: %v", err)
	}
	log.Debug("objectGet: found dagnode %q (# of bytes: %d - # links: %d)", args[0], len(dagnode.Data), len(dagnode.Links))

	// sadly all encodings dont implement a common interface
	var data []byte
	switch getObjectEnc(opts["encoding"]) {
	case objectEncodingJSON:
		data, err = json.MarshalIndent(dagnode, "", "  ")

	case objectEncodingProtobuf:
		data, err = dagnode.Marshal()

	default:
		return ErrUnknownObjectEnc
	}

	if err != nil {
		return fmt.Errorf("ObjectGet error: %v", err)
	}

	_, err = io.Copy(out, bytes.NewReader(data))
	return err
}

// ObjectPut takes a format option, serilizes bytes from stdin and updates the dag with that data
func ObjectPut(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error {
	var (
		dagnode *dag.Node
		data    []byte
		err     error
	)

	data, err = ioutil.ReadAll(os.Stdin)
	if err != nil {
		return fmt.Errorf("ObjectPut error: %v", err)
	}

	switch getObjectEnc(opts["encoding"]) {
	case objectEncodingJSON:
		dagnode = new(dag.Node)
		err = json.Unmarshal(data, dagnode)

	case objectEncodingProtobuf:
		dagnode, err = dag.Decoded(data)

	default:
		return ErrUnknownObjectEnc
	}

	if err != nil {
		return fmt.Errorf("ObjectPut error: %v", err)
	}

	return addNode(n, dagnode, "stdin", out)
}
<|endoftext|>


<commit_before>package commands

import (
	"context"
	"encoding/binary"
	"fmt"
	"io"
	"net/http"
	"sync"
	"time"

	core "github.com/ipfs/go-ipfs/core"

	floodsub "gx/ipfs/QmSFihvoND3eDaAYRCeLgLPt62yCPgMZs1NSZmKFEtJQQw/go-libp2p-floodsub"
	pstore "gx/ipfs/QmXauCuJzmzapetmC6W4TuDJLL1yFFrVzSHoWv8YdbmnxH/go-libp2p-peerstore"
	cmds "gx/ipfs/QmabLouZTZwhfALuBcssPvkzhbYGMb4394huT7HY4LQ6d3/go-ipfs-cmds"
	cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
	cmdkit "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit"
	blocks "gx/ipfs/Qmej7nf81hi2x2tvjRBF3mcp74sQyuDH4VMYDGd1YtXjb2/go-block-format"
)

var PubsubCmd = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "An experimental publish-subscribe system on ipfs.",
		ShortDescription: `
ipfs pubsub allows you to publish messages to a given topic, and also to
subscribe to new messages on a given topic.

This is an experimental feature. It is not intended in its current state
to be used in a production environment.

To use, the daemon must be run with '--enable-pubsub-experiment'.
`,
	},
	Subcommands: map[string]*cmds.Command{
		"pub":   PubsubPubCmd,
		"sub":   PubsubSubCmd,
		"ls":    PubsubLsCmd,
		"peers": PubsubPeersCmd,
	},
}

var PubsubSubCmd = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "Subscribe to messages on a given topic.",
		ShortDescription: `
ipfs pubsub sub subscribes to messages on a given topic.

This is an experimental feature. It is not intended in its current state
to be used in a production environment.

To use, the daemon must be run with '--enable-pubsub-experiment'.
`,
		LongDescription: `
ipfs pubsub sub subscribes to messages on a given topic.

This is an experimental feature. It is not intended in its current state
to be used in a production environment.

To use, the daemon must be run with '--enable-pubsub-experiment'.

This command outputs data in the following encodings:
  * "json"
(Specified by the "--encoding" or "--enc" flag)
`,
	},
	Arguments: []cmdkit.Argument{
		cmdkit.StringArg("topic", true, false, "String name of topic to subscribe to."),
	},
	Options: []cmdkit.Option{
		cmdkit.BoolOption("discover", "try to discover other peers subscribed to the same topic"),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
		n, err := GetNode(env)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		// Must be online!
		if !n.OnlineMode() {
			res.SetError(errNotOnline, cmdkit.ErrClient)
			return
		}

		if n.Floodsub == nil {
			res.SetError(fmt.Errorf("experimental pubsub feature not enabled. Run daemon with --enable-pubsub-experiment to use."), cmdkit.ErrNormal)
			return
		}

		topic := req.Arguments[0]
		sub, err := n.Floodsub.Subscribe(topic)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}
		defer sub.Cancel()

		discover, _ := req.Options["discover"].(bool)
		if discover {
			go func() {
				blk := blocks.NewBlock([]byte("floodsub:" + topic))
				err := n.Blocks.AddBlock(blk)
				if err != nil {
					log.Error("pubsub discovery: ", err)
					return
				}

				connectToPubSubPeers(req.Context, n, blk.Cid())
			}()
		}

		if f, ok := res.(http.Flusher); ok {
			f.Flush()
		}

		for {
			msg, err := sub.Next(req.Context)
			if err == io.EOF || err == context.Canceled {
				return
			} else if err != nil {
				res.SetError(err, cmdkit.ErrNormal)
				return
			}

			res.Emit(msg)
		}
	},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
			m, ok := v.(*floodsub.Message)
			if !ok {
				return fmt.Errorf("unexpected type: %T", v)
			}

			_, err := w.Write(m.Data)
			return err
		}),
		"ndpayload": cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
			m, ok := v.(*floodsub.Message)
			if !ok {
				return fmt.Errorf("unexpected type: %T", v)
			}

			m.Data = append(m.Data, '\n')
			_, err := w.Write(m.Data)
			return err
		}),
		"lenpayload": cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
			m, ok := v.(*floodsub.Message)
			if !ok {
				return fmt.Errorf("unexpected type: %T", v)
			}

			buf := make([]byte, 8, len(m.Data)+8)

			n := binary.PutUvarint(buf, uint64(len(m.Data)))
			buf = append(buf[:n], m.Data...)
			_, err := w.Write(buf)
			return err
		}),
	},
	Type: floodsub.Message{},
}

func connectToPubSubPeers(ctx context.Context, n *core.IpfsNode, cid *cid.Cid) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	provs := n.Routing.FindProvidersAsync(ctx, cid, 10)
	wg := &sync.WaitGroup{}
	for p := range provs {
		wg.Add(1)
		go func(pi pstore.PeerInfo) {
			defer wg.Done()
			ctx, cancel := context.WithTimeout(ctx, time.Second*10)
			defer cancel()
			err := n.PeerHost.Connect(ctx, pi)
			if err != nil {
				log.Info("pubsub discover: ", err)
				return
			}
			log.Info("connected to pubsub peer:", pi.ID)
		}(p)
	}

	wg.Wait()
}

var PubsubPubCmd = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "Publish a message to a given pubsub topic.",
		ShortDescription: `
ipfs pubsub pub publishes a message to a specified topic.

This is an experimental feature. It is not intended in its current state
to be used in a production environment.

To use, the daemon must be run with '--enable-pubsub-experiment'.
`,
	},
	Arguments: []cmdkit.Argument{
		cmdkit.StringArg("topic", true, false, "Topic to publish to."),
		cmdkit.StringArg("data", true, true, "Payload of message to publish.").EnableStdin(),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
		n, err := GetNode(env)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		// Must be online!
		if !n.OnlineMode() {
			res.SetError(errNotOnline, cmdkit.ErrClient)
			return
		}

		if n.Floodsub == nil {
			res.SetError("experimental pubsub feature not enabled. Run daemon with --enable-pubsub-experiment to use.", cmdkit.ErrNormal)
			return
		}

		topic := req.Arguments[0]

		err = req.ParseBodyArgs()
		if err != nil && !cmds.IsAllArgsAlreadyCovered(err) {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		for _, data := range req.Arguments[1:] {
			if err := n.Floodsub.Publish(topic, []byte(data)); err != nil {
				res.SetError(err, cmdkit.ErrNormal)
				return
			}
		}
	},
}

var PubsubLsCmd = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "List subscribed topics by name.",
		ShortDescription: `
ipfs pubsub ls lists out the names of topics you are currently subscribed to.

This is an experimental feature. It is not intended in its current state
to be used in a production environment.

To use, the daemon must be run with '--enable-pubsub-experiment'.
`,
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
		n, err := GetNode(env)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		// Must be online!
		if !n.OnlineMode() {
			res.SetError(errNotOnline, cmdkit.ErrClient)
			return
		}

		if n.Floodsub == nil {
			res.SetError("experimental pubsub feature not enabled. Run daemon with --enable-pubsub-experiment to use.", cmdkit.ErrNormal)
			return
		}

		cmds.EmitOnce(res, stringList{n.Floodsub.GetTopics()})
	},
	Type: stringList{},
}

var PubsubPeersCmd = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "List peers we are currently pubsubbing with.",
		ShortDescription: `
ipfs pubsub peers with no arguments lists out the pubsub peers you are
currently connected to. If given a topic, it will list connected
peers who are subscribed to the named topic.

This is an experimental feature. It is not intended in its current state
to be used in a production environment.

To use, the daemon must be run with '--enable-pubsub-experiment'.
`,
	},
	Arguments: []cmdkit.Argument{
		cmdkit.StringArg("topic", false, false, "topic to list connected peers of"),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
		n, err := GetNode(env)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		// Must be online!
		if !n.OnlineMode() {
			res.SetError(errNotOnline, cmdkit.ErrClient)
			return
		}

		if n.Floodsub == nil {
			res.SetError(fmt.Errorf("experimental pubsub feature not enabled. Run daemon with --enable-pubsub-experiment to use."), cmdkit.ErrNormal)
			return
		}

		var topic string
		if len(req.Arguments) == 1 {
			topic = req.Arguments[0]
		}

		for _, peer := range n.Floodsub.ListPeers(topic) {
			res.Emit(peer.Pretty())
		}
	},
	Type: "",
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.Encoders[cmds.TextNewline],
	},
}
<commit_msg>cmd/pubsub: fix pubsub peers format brakage<commit_after>package commands

import (
	"context"
	"encoding/binary"
	"fmt"
	"io"
	"net/http"
	"sort"
	"sync"
	"time"

	core "github.com/ipfs/go-ipfs/core"
	e "github.com/ipfs/go-ipfs/core/commands/e"

	floodsub "gx/ipfs/QmSFihvoND3eDaAYRCeLgLPt62yCPgMZs1NSZmKFEtJQQw/go-libp2p-floodsub"
	pstore "gx/ipfs/QmXauCuJzmzapetmC6W4TuDJLL1yFFrVzSHoWv8YdbmnxH/go-libp2p-peerstore"
	cmds "gx/ipfs/QmabLouZTZwhfALuBcssPvkzhbYGMb4394huT7HY4LQ6d3/go-ipfs-cmds"
	cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
	cmdkit "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit"
	blocks "gx/ipfs/Qmej7nf81hi2x2tvjRBF3mcp74sQyuDH4VMYDGd1YtXjb2/go-block-format"
)

var PubsubCmd = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "An experimental publish-subscribe system on ipfs.",
		ShortDescription: `
ipfs pubsub allows you to publish messages to a given topic, and also to
subscribe to new messages on a given topic.

This is an experimental feature. It is not intended in its current state
to be used in a production environment.

To use, the daemon must be run with '--enable-pubsub-experiment'.
`,
	},
	Subcommands: map[string]*cmds.Command{
		"pub":   PubsubPubCmd,
		"sub":   PubsubSubCmd,
		"ls":    PubsubLsCmd,
		"peers": PubsubPeersCmd,
	},
}

var PubsubSubCmd = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "Subscribe to messages on a given topic.",
		ShortDescription: `
ipfs pubsub sub subscribes to messages on a given topic.

This is an experimental feature. It is not intended in its current state
to be used in a production environment.

To use, the daemon must be run with '--enable-pubsub-experiment'.
`,
		LongDescription: `
ipfs pubsub sub subscribes to messages on a given topic.

This is an experimental feature. It is not intended in its current state
to be used in a production environment.

To use, the daemon must be run with '--enable-pubsub-experiment'.

This command outputs data in the following encodings:
  * "json"
(Specified by the "--encoding" or "--enc" flag)
`,
	},
	Arguments: []cmdkit.Argument{
		cmdkit.StringArg("topic", true, false, "String name of topic to subscribe to."),
	},
	Options: []cmdkit.Option{
		cmdkit.BoolOption("discover", "try to discover other peers subscribed to the same topic"),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
		n, err := GetNode(env)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		// Must be online!
		if !n.OnlineMode() {
			res.SetError(errNotOnline, cmdkit.ErrClient)
			return
		}

		if n.Floodsub == nil {
			res.SetError(fmt.Errorf("experimental pubsub feature not enabled. Run daemon with --enable-pubsub-experiment to use."), cmdkit.ErrNormal)
			return
		}

		topic := req.Arguments[0]
		sub, err := n.Floodsub.Subscribe(topic)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}
		defer sub.Cancel()

		discover, _ := req.Options["discover"].(bool)
		if discover {
			go func() {
				blk := blocks.NewBlock([]byte("floodsub:" + topic))
				err := n.Blocks.AddBlock(blk)
				if err != nil {
					log.Error("pubsub discovery: ", err)
					return
				}

				connectToPubSubPeers(req.Context, n, blk.Cid())
			}()
		}

		if f, ok := res.(http.Flusher); ok {
			f.Flush()
		}

		for {
			msg, err := sub.Next(req.Context)
			if err == io.EOF || err == context.Canceled {
				return
			} else if err != nil {
				res.SetError(err, cmdkit.ErrNormal)
				return
			}

			res.Emit(msg)
		}
	},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
			m, ok := v.(*floodsub.Message)
			if !ok {
				return fmt.Errorf("unexpected type: %T", v)
			}

			_, err := w.Write(m.Data)
			return err
		}),
		"ndpayload": cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
			m, ok := v.(*floodsub.Message)
			if !ok {
				return fmt.Errorf("unexpected type: %T", v)
			}

			m.Data = append(m.Data, '\n')
			_, err := w.Write(m.Data)
			return err
		}),
		"lenpayload": cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
			m, ok := v.(*floodsub.Message)
			if !ok {
				return fmt.Errorf("unexpected type: %T", v)
			}

			buf := make([]byte, 8, len(m.Data)+8)

			n := binary.PutUvarint(buf, uint64(len(m.Data)))
			buf = append(buf[:n], m.Data...)
			_, err := w.Write(buf)
			return err
		}),
	},
	Type: floodsub.Message{},
}

func connectToPubSubPeers(ctx context.Context, n *core.IpfsNode, cid *cid.Cid) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	provs := n.Routing.FindProvidersAsync(ctx, cid, 10)
	wg := &sync.WaitGroup{}
	for p := range provs {
		wg.Add(1)
		go func(pi pstore.PeerInfo) {
			defer wg.Done()
			ctx, cancel := context.WithTimeout(ctx, time.Second*10)
			defer cancel()
			err := n.PeerHost.Connect(ctx, pi)
			if err != nil {
				log.Info("pubsub discover: ", err)
				return
			}
			log.Info("connected to pubsub peer:", pi.ID)
		}(p)
	}

	wg.Wait()
}

var PubsubPubCmd = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "Publish a message to a given pubsub topic.",
		ShortDescription: `
ipfs pubsub pub publishes a message to a specified topic.

This is an experimental feature. It is not intended in its current state
to be used in a production environment.

To use, the daemon must be run with '--enable-pubsub-experiment'.
`,
	},
	Arguments: []cmdkit.Argument{
		cmdkit.StringArg("topic", true, false, "Topic to publish to."),
		cmdkit.StringArg("data", true, true, "Payload of message to publish.").EnableStdin(),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
		n, err := GetNode(env)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		// Must be online!
		if !n.OnlineMode() {
			res.SetError(errNotOnline, cmdkit.ErrClient)
			return
		}

		if n.Floodsub == nil {
			res.SetError("experimental pubsub feature not enabled. Run daemon with --enable-pubsub-experiment to use.", cmdkit.ErrNormal)
			return
		}

		topic := req.Arguments[0]

		err = req.ParseBodyArgs()
		if err != nil && !cmds.IsAllArgsAlreadyCovered(err) {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		for _, data := range req.Arguments[1:] {
			if err := n.Floodsub.Publish(topic, []byte(data)); err != nil {
				res.SetError(err, cmdkit.ErrNormal)
				return
			}
		}
	},
}

var PubsubLsCmd = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "List subscribed topics by name.",
		ShortDescription: `
ipfs pubsub ls lists out the names of topics you are currently subscribed to.

This is an experimental feature. It is not intended in its current state
to be used in a production environment.

To use, the daemon must be run with '--enable-pubsub-experiment'.
`,
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
		n, err := GetNode(env)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		// Must be online!
		if !n.OnlineMode() {
			res.SetError(errNotOnline, cmdkit.ErrClient)
			return
		}

		if n.Floodsub == nil {
			res.SetError("experimental pubsub feature not enabled. Run daemon with --enable-pubsub-experiment to use.", cmdkit.ErrNormal)
			return
		}

		cmds.EmitOnce(res, stringList{n.Floodsub.GetTopics()})
	},
	Type: stringList{},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeEncoder(stringListEncoder),
	},
}

func stringListEncoder(req *cmds.Request, w io.Writer, v interface{}) error {
	list, ok := v.(*stringList)
	if !ok {
		return e.TypeErr(list, v)
	}
	for _, str := range list.Strings {
		_, err := fmt.Fprintf(w, "%s\n", str)
		if err != nil {
			return err
		}
	}
	return nil
}

var PubsubPeersCmd = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "List peers we are currently pubsubbing with.",
		ShortDescription: `
ipfs pubsub peers with no arguments lists out the pubsub peers you are
currently connected to. If given a topic, it will list connected
peers who are subscribed to the named topic.

This is an experimental feature. It is not intended in its current state
to be used in a production environment.

To use, the daemon must be run with '--enable-pubsub-experiment'.
`,
	},
	Arguments: []cmdkit.Argument{
		cmdkit.StringArg("topic", false, false, "topic to list connected peers of"),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
		n, err := GetNode(env)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		// Must be online!
		if !n.OnlineMode() {
			res.SetError(errNotOnline, cmdkit.ErrClient)
			return
		}

		if n.Floodsub == nil {
			res.SetError(fmt.Errorf("experimental pubsub feature not enabled. Run daemon with --enable-pubsub-experiment to use."), cmdkit.ErrNormal)
			return
		}

		var topic string
		if len(req.Arguments) == 1 {
			topic = req.Arguments[0]
		}

		peers := n.Floodsub.ListPeers(topic)
		list := &stringList{make([]string, 0, len(peers))}

		for _, peer := range peers {
			list.Strings = append(list.Strings, peer.Pretty())
		}
		sort.Strings(list.Strings)
		cmds.EmitOnce(res, list)
	},
	Type: stringList{},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeEncoder(stringListEncoder),
	},
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dns

import (
	"io/ioutil"
	"net"
	"strings"

	"github.com/golang/glog"
)

var defResolvConf = "/etc/resolv.conf"

// GetSystemNameServers returns the list of nameservers located in the file /etc/resolv.conf
func GetSystemNameServers() ([]net.IP, error) {
	var nameservers []net.IP
	file, err := ioutil.ReadFile(defResolvConf)
	if err != nil {
		return nameservers, err
	}

	// Lines of the form "nameserver 1.2.3.4" accumulate.
	lines := strings.Split(string(file), "\n")
	for l := range lines {
		trimmed := strings.TrimSpace(lines[l])
		if strings.HasPrefix(trimmed, "#") {
			continue
		}
		fields := strings.Fields(trimmed)
		if len(fields) == 0 {
			continue
		}
		if fields[0] == "nameserver" {
			ip := net.ParseIP(fields[1])
			if ip != nil {
				nameservers = append(nameservers, ip)
			}
		}
	}

	glog.V(3).Infof("nameservers IP address/es to use: %v", nameservers)
	return nameservers, nil
}
<commit_msg>check fields len in dns.go<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dns

import (
	"io/ioutil"
	"net"
	"strings"

	"github.com/golang/glog"
)

var defResolvConf = "/etc/resolv.conf"

// GetSystemNameServers returns the list of nameservers located in the file /etc/resolv.conf
func GetSystemNameServers() ([]net.IP, error) {
	var nameservers []net.IP
	file, err := ioutil.ReadFile(defResolvConf)
	if err != nil {
		return nameservers, err
	}

	// Lines of the form "nameserver 1.2.3.4" accumulate.
	lines := strings.Split(string(file), "\n")
	for l := range lines {
		trimmed := strings.TrimSpace(lines[l])
		if strings.HasPrefix(trimmed, "#") {
			continue
		}
		fields := strings.Fields(trimmed)
		if len(fields) < 2 {
			continue
		}
		if fields[0] == "nameserver" {
			ip := net.ParseIP(fields[1])
			if ip != nil {
				nameservers = append(nameservers, ip)
			}
		}
	}

	glog.V(3).Infof("nameservers IP address/es to use: %v", nameservers)
	return nameservers, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ssl

import (
	"crypto/sha1"
	"crypto/x509"
	"encoding/hex"
	"encoding/pem"
	"errors"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/golang/glog"

	"k8s.io/ingress/core/pkg/ingress"
)

// AddOrUpdateCertAndKey creates a .pem file wth the cert and the key with the specified name
func AddOrUpdateCertAndKey(name string, cert, key, ca []byte) (*ingress.SSLCert, error) {
	pemName := fmt.Sprintf("%v.pem", name)
	pemFileName := fmt.Sprintf("%v/%v", ingress.DefaultSSLDirectory, pemName)

	tempPemFile, err := ioutil.TempFile("", pemName)
	if err != nil {
		return nil, fmt.Errorf("could not create temp pem file %v: %v", tempPemFile.Name(), err)
	}

	_, err = tempPemFile.Write(cert)
	if err != nil {
		return nil, fmt.Errorf("could not write to pem file %v: %v", tempPemFile.Name(), err)
	}
	_, err = tempPemFile.Write([]byte("\n"))
	if err != nil {
		return nil, fmt.Errorf("could not write to pem file %v: %v", tempPemFile.Name(), err)
	}
	_, err = tempPemFile.Write(key)
	if err != nil {
		return nil, fmt.Errorf("could not write to pem file %v: %v", tempPemFile.Name(), err)
	}

	err = tempPemFile.Close()
	if err != nil {
		return nil, fmt.Errorf("could not close temp pem file %v: %v", tempPemFile.Name(), err)
	}

	pemCerts, err := ioutil.ReadFile(tempPemFile.Name())
	if err != nil {
		return nil, err
	}

	pembBock, _ := pem.Decode(pemCerts)
	if pembBock == nil {
		return nil, fmt.Errorf("No valid PEM formatted block found")
	}

	pemCert, err := x509.ParseCertificate(pembBock.Bytes)
	if err != nil {
		return nil, err
	}

	cn := []string{pemCert.Subject.CommonName}
	if len(pemCert.DNSNames) > 0 {
		cn = append(cn, pemCert.DNSNames...)
	}

	err = os.Rename(tempPemFile.Name(), pemFileName)
	if err != nil {
		return nil, fmt.Errorf("could not move temp pem file %v to destination %v: %v", tempPemFile.Name(), pemFileName, err)
	}

	if len(ca) > 0 {
		bundle := x509.NewCertPool()
		bundle.AppendCertsFromPEM(ca)
		opts := x509.VerifyOptions{
			Roots: bundle,
		}

		_, err := pemCert.Verify(opts)
		if err != nil {
			oe := fmt.Sprintf("failed to verify certificate chain: \n\t%s\n", err)
			return nil, errors.New(oe)
		}

		caName := fmt.Sprintf("ca-%v.pem", name)
		caFileName := fmt.Sprintf("%v/%v", ingress.DefaultSSLDirectory, caName)
		f, err := os.Create(caFileName)
		if err != nil {
			return nil, fmt.Errorf("could not create ca pem file %v: %v", caFileName, err)
		}
		defer f.Close()
		_, err = f.Write(ca)
		if err != nil {
			return nil, fmt.Errorf("could not create ca pem file %v: %v", caFileName, err)
		}
		f.Write([]byte("\n"))

		return &ingress.SSLCert{
			CAFileName:  caFileName,
			PemFileName: pemFileName,
			PemSHA:      pemSHA1(pemFileName),
			CN:          cn,
		}, nil
	}

	return &ingress.SSLCert{
		PemFileName: pemFileName,
		PemSHA:      pemSHA1(pemFileName),
		CN:          cn,
	}, nil
}

// SearchDHParamFile iterates all the secrets mounted inside the /etc/nginx-ssl directory
// in order to find a file with the name dhparam.pem. If such file exists it will
// returns the path. If not it just returns an empty string
func SearchDHParamFile(baseDir string) string {
	files, _ := ioutil.ReadDir(baseDir)
	for _, file := range files {
		if !file.IsDir() {
			continue
		}

		dhPath := fmt.Sprintf("%v/%v/dhparam.pem", baseDir, file.Name())
		if _, err := os.Stat(dhPath); err == nil {
			glog.Infof("using file '%v' for parameter ssl_dhparam", dhPath)
			return dhPath
		}
	}

	glog.Warning("no file dhparam.pem found in secrets")
	return ""
}

// pemSHA1 returns the SHA1 of a pem file. This is used to
// reload NGINX in case a secret with a SSL certificate changed.
func pemSHA1(filename string) string {
	hasher := sha1.New()
	s, err := ioutil.ReadFile(filename)
	if err != nil {
		return ""
	}

	hasher.Write(s)
	return hex.EncodeToString(hasher.Sum(nil))
}

const (
	snakeOilPem = "/etc/ssl/certs/ssl-cert-snakeoil.pem"
	snakeOilKey = "/etc/ssl/private/ssl-cert-snakeoil.key"
)

// GetFakeSSLCert returns the snake oil ssl certificate created by the command
// make-ssl-cert generate-default-snakeoil --force-overwrite
func GetFakeSSLCert() ([]byte, []byte) {
	cert, err := ioutil.ReadFile(snakeOilPem)
	if err != nil {
		return nil, nil
	}

	key, err := ioutil.ReadFile(snakeOilKey)
	if err != nil {
		return nil, nil
	}

	return cert, key
}
<commit_msg>Changes the SSL Temp file to something inside the same SSL Directory<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ssl

import (
	"crypto/sha1"
	"crypto/x509"
	"encoding/hex"
	"encoding/pem"
	"errors"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/golang/glog"

	"k8s.io/ingress/core/pkg/ingress"
)

// AddOrUpdateCertAndKey creates a .pem file wth the cert and the key with the specified name
func AddOrUpdateCertAndKey(name string, cert, key, ca []byte) (*ingress.SSLCert, error) {
	pemName := fmt.Sprintf("%v.pem", name)
	pemFileName := fmt.Sprintf("%v/%v", ingress.DefaultSSLDirectory, pemName)

	tempPemFile, err := ioutil.TempFile(ingress.DefaultSSLDirectory, pemName)
	if err != nil {
		return nil, fmt.Errorf("could not create temp pem file %v: %v", tempPemFile.Name(), err)
	}

	_, err = tempPemFile.Write(cert)
	if err != nil {
		return nil, fmt.Errorf("could not write to pem file %v: %v", tempPemFile.Name(), err)
	}
	_, err = tempPemFile.Write([]byte("\n"))
	if err != nil {
		return nil, fmt.Errorf("could not write to pem file %v: %v", tempPemFile.Name(), err)
	}
	_, err = tempPemFile.Write(key)
	if err != nil {
		return nil, fmt.Errorf("could not write to pem file %v: %v", tempPemFile.Name(), err)
	}

	err = tempPemFile.Close()
	if err != nil {
		return nil, fmt.Errorf("could not close temp pem file %v: %v", tempPemFile.Name(), err)
	}

	pemCerts, err := ioutil.ReadFile(tempPemFile.Name())
	if err != nil {
		return nil, err
	}

	pembBock, _ := pem.Decode(pemCerts)
	if pembBock == nil {
		return nil, fmt.Errorf("No valid PEM formatted block found")
	}

	pemCert, err := x509.ParseCertificate(pembBock.Bytes)
	if err != nil {
		return nil, err
	}

	cn := []string{pemCert.Subject.CommonName}
	if len(pemCert.DNSNames) > 0 {
		cn = append(cn, pemCert.DNSNames...)
	}

	err = os.Rename(tempPemFile.Name(), pemFileName)
	if err != nil {
		return nil, fmt.Errorf("could not move temp pem file %v to destination %v: %v", tempPemFile.Name(), pemFileName, err)
	}

	if len(ca) > 0 {
		bundle := x509.NewCertPool()
		bundle.AppendCertsFromPEM(ca)
		opts := x509.VerifyOptions{
			Roots: bundle,
		}

		_, err := pemCert.Verify(opts)
		if err != nil {
			oe := fmt.Sprintf("failed to verify certificate chain: \n\t%s\n", err)
			return nil, errors.New(oe)
		}

		caName := fmt.Sprintf("ca-%v.pem", name)
		caFileName := fmt.Sprintf("%v/%v", ingress.DefaultSSLDirectory, caName)
		f, err := os.Create(caFileName)
		if err != nil {
			return nil, fmt.Errorf("could not create ca pem file %v: %v", caFileName, err)
		}
		defer f.Close()
		_, err = f.Write(ca)
		if err != nil {
			return nil, fmt.Errorf("could not create ca pem file %v: %v", caFileName, err)
		}
		f.Write([]byte("\n"))

		return &ingress.SSLCert{
			CAFileName:  caFileName,
			PemFileName: pemFileName,
			PemSHA:      pemSHA1(pemFileName),
			CN:          cn,
		}, nil
	}

	return &ingress.SSLCert{
		PemFileName: pemFileName,
		PemSHA:      pemSHA1(pemFileName),
		CN:          cn,
	}, nil
}

// SearchDHParamFile iterates all the secrets mounted inside the /etc/nginx-ssl directory
// in order to find a file with the name dhparam.pem. If such file exists it will
// returns the path. If not it just returns an empty string
func SearchDHParamFile(baseDir string) string {
	files, _ := ioutil.ReadDir(baseDir)
	for _, file := range files {
		if !file.IsDir() {
			continue
		}

		dhPath := fmt.Sprintf("%v/%v/dhparam.pem", baseDir, file.Name())
		if _, err := os.Stat(dhPath); err == nil {
			glog.Infof("using file '%v' for parameter ssl_dhparam", dhPath)
			return dhPath
		}
	}

	glog.Warning("no file dhparam.pem found in secrets")
	return ""
}

// pemSHA1 returns the SHA1 of a pem file. This is used to
// reload NGINX in case a secret with a SSL certificate changed.
func pemSHA1(filename string) string {
	hasher := sha1.New()
	s, err := ioutil.ReadFile(filename)
	if err != nil {
		return ""
	}

	hasher.Write(s)
	return hex.EncodeToString(hasher.Sum(nil))
}

const (
	snakeOilPem = "/etc/ssl/certs/ssl-cert-snakeoil.pem"
	snakeOilKey = "/etc/ssl/private/ssl-cert-snakeoil.key"
)

// GetFakeSSLCert returns the snake oil ssl certificate created by the command
// make-ssl-cert generate-default-snakeoil --force-overwrite
func GetFakeSSLCert() ([]byte, []byte) {
	cert, err := ioutil.ReadFile(snakeOilPem)
	if err != nil {
		return nil, nil
	}

	key, err := ioutil.ReadFile(snakeOilKey)
	if err != nil {
		return nil, nil
	}

	return cert, key
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"bytes"
	"strings"
	"testing"
	"time"
)

func TestComputeCryptoHash(t *testing.T) {
	if bytes.Compare(ComputeCryptoHash([]byte("foobar")), ComputeCryptoHash([]byte("foobar"))) != 0 {
		t.Fatalf("Expected hashes to match, but they did not match")
	}
	if bytes.Compare(ComputeCryptoHash([]byte("foobar1")), ComputeCryptoHash([]byte("foobar2"))) == 0 {
		t.Fatalf("Expected hashes to be different, but they match")
	}
}

func TestUUIDGeneration(t *testing.T) {
	uuid := GenerateUUID()
	if len(uuid) != 36 {
		t.Fatalf("UUID length is not correct. Expected = 36, Got = %d", len(uuid))
	}
	uuid2 := GenerateUUID()
	if uuid == uuid2 {
		t.Fatalf("Two UUIDs are equal. This should never occur")
	}
}

func TestIntUUIDGeneration(t *testing.T) {
	uuid := GenerateIntUUID()

	uuid2 := GenerateIntUUID()
	if uuid == uuid2 {
		t.Fatalf("Two UUIDs are equal. This should never occur")
	}
}
func TestTimestamp(t *testing.T) {
	for i := 0; i < 10; i++ {
		t.Logf("timestamp now: %v", CreateUtcTimestamp())
		time.Sleep(200 * time.Millisecond)
	}
}

func TestGenerateHashFromSignature(t *testing.T) {
	if bytes.Compare(GenerateHashFromSignature("aPath", []byte("aCtor12")),
		GenerateHashFromSignature("aPath", []byte("aCtor12"))) != 0 {
		t.Fatalf("Expected hashes to match, but they did not match")
	}
	if bytes.Compare(GenerateHashFromSignature("aPath", []byte("aCtor12")),
		GenerateHashFromSignature("bPath", []byte("bCtor34"))) == 0 {
		t.Fatalf("Expected hashes to be different, but they match")
	}
}

func TestGenerateUUIDfromTxSHAHash(t *testing.T) {
	uuid := GenerateUUIDfromTxSHAHash([]byte("foobar"))
	if len(uuid) != 36 {
		t.Fatalf("UUID length is not correct. Expected = 36, Got = %d", len(uuid))
	}
	uuid2 := GenerateUUIDfromTxSHAHash([]byte("foobar1"))
	if uuid == uuid2 {
		t.Fatalf("Two UUIDs are equal. This should never occur")
	}
}

func TestGenerateIDWithAlg(t *testing.T) {
	_, err := GenerateIDWithAlg("sha256base64", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
	if err != nil {
		t.Fatalf("Decoder failure: %v", err)
	}
}

func TestFindMissingElements(t *testing.T) {
	all := []string{"a", "b", "c", "d"}
	some := []string{"b", "c"}
	expectedDelta := []string{"a", "d"}
	actualDelta := FindMissingElements(all, some)
	if len(expectedDelta) != len(actualDelta) {
		t.Fatalf("Got %v, expected %v", actualDelta, expectedDelta)
	}
	for i := range expectedDelta {
		if strings.Compare(expectedDelta[i], actualDelta[i]) != 0 {
			t.Fatalf("Got %v, expected %v", actualDelta, expectedDelta)
		}
	}
}
<commit_msg>Fix failing utils unit test<commit_after>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"bytes"
	"strings"
	"testing"
	"time"
)

func TestComputeCryptoHash(t *testing.T) {
	if bytes.Compare(ComputeCryptoHash([]byte("foobar")), ComputeCryptoHash([]byte("foobar"))) != 0 {
		t.Fatalf("Expected hashes to match, but they did not match")
	}
	if bytes.Compare(ComputeCryptoHash([]byte("foobar1")), ComputeCryptoHash([]byte("foobar2"))) == 0 {
		t.Fatalf("Expected hashes to be different, but they match")
	}
}

func TestUUIDGeneration(t *testing.T) {
	uuid := GenerateUUID()
	if len(uuid) != 36 {
		t.Fatalf("UUID length is not correct. Expected = 36, Got = %d", len(uuid))
	}
	uuid2 := GenerateUUID()
	if uuid == uuid2 {
		t.Fatalf("Two UUIDs are equal. This should never occur")
	}
}

func TestIntUUIDGeneration(t *testing.T) {
	uuid := GenerateIntUUID()

	uuid2 := GenerateIntUUID()
	if uuid == uuid2 {
		t.Fatalf("Two UUIDs are equal. This should never occur")
	}
}
func TestTimestamp(t *testing.T) {
	for i := 0; i < 10; i++ {
		t.Logf("timestamp now: %v", CreateUtcTimestamp())
		time.Sleep(200 * time.Millisecond)
	}
}

func TestGenerateHashFromSignature(t *testing.T) {
	if bytes.Compare(GenerateHashFromSignature("aPath", []byte("aCtor12")),
		GenerateHashFromSignature("aPath", []byte("aCtor12"))) != 0 {
		t.Fatalf("Expected hashes to match, but they did not match")
	}
	if bytes.Compare(GenerateHashFromSignature("aPath", []byte("aCtor12")),
		GenerateHashFromSignature("bPath", []byte("bCtor34"))) == 0 {
		t.Fatalf("Expected hashes to be different, but they match")
	}
}

func TestGeneratIDfromTxSHAHash(t *testing.T) {
	txid := GenerateIDfromTxSHAHash([]byte("foobar"))
	txid2 := GenerateIDfromTxSHAHash([]byte("foobar1"))
	if txid == txid2 {
		t.Fatalf("Two TxIDs are equal. This should never occur")
	}
}

func TestGenerateIDWithAlg(t *testing.T) {
	_, err := GenerateIDWithAlg("sha256", []byte{1, 1, 1, 1})
	if err != nil {
		t.Fatalf("Decoder failure: %v", err)
	}
}

func TestFindMissingElements(t *testing.T) {
	all := []string{"a", "b", "c", "d"}
	some := []string{"b", "c"}
	expectedDelta := []string{"a", "d"}
	actualDelta := FindMissingElements(all, some)
	if len(expectedDelta) != len(actualDelta) {
		t.Fatalf("Got %v, expected %v", actualDelta, expectedDelta)
	}
	for i := range expectedDelta {
		if strings.Compare(expectedDelta[i], actualDelta[i]) != 0 {
			t.Fatalf("Got %v, expected %v", actualDelta, expectedDelta)
		}
	}
}
<|endoftext|>


<commit_before>package failover

import (
	"github.com/gorilla/mux"
	"github.com/siddontang/go/log"
	"net"
	"net/http"
	"sync"
	"time"
)

type BeforeFailoverHandler func(downMaster string) error
type AfterFailoverHandler func(downMaster, newMaster string) error

type App struct {
	c *Config

	l net.Listener

	r *Raft

	masters *masterFSM

	gMutex sync.Mutex
	groups map[string]*Group

	quit chan struct{}
	wg   sync.WaitGroup

	hMutex         sync.Mutex
	beforeHandlers []BeforeFailoverHandler
	afterHandlers  []AfterFailoverHandler
}

func NewApp(c *Config) (*App, error) {
	var err error

	a := new(App)
	a.c = c
	a.quit = make(chan struct{})
	a.groups = make(map[string]*Group)

	a.masters = newMasterFSM()

	if c.MaxDownTime <= 0 {
		c.MaxDownTime = 3
	}

	if a.c.CheckInterval <= 0 {
		a.c.CheckInterval = 1000
	}

	if len(c.Addr) > 0 {
		a.l, err = net.Listen("tcp", c.Addr)
		if err != nil {
			return nil, err
		}
	}

	a.r, err = newRaft(c, a.masters)
	if err != nil {
		return nil, err
	}

	if c.MastersState == MastersStateNew {
		a.setMasters(c.Masters)
	} else {
		a.addMasters(c.Masters)
	}

	return a, nil
}

func (a *App) Close() {
	select {
	case <-a.quit:
		return
	default:
		break
	}

	if a.l != nil {
		a.l.Close()
	}

	if a.r != nil {
		a.r.Close()
	}

	close(a.quit)

	a.wg.Wait()
}

func (a *App) Run() {
	go a.startHTTP()

	a.wg.Add(1)
	t := time.NewTicker(time.Duration(a.c.CheckInterval) * time.Millisecond)
	defer func() {
		t.Stop()
		a.wg.Done()
	}()

	for {
		select {
		case <-t.C:
			a.check()
		case <-a.quit:
			return
		}
	}
}

func (a *App) check() {
	if a.r != nil && !a.r.IsLeader() {
		// is not leader, not check
		return
	}

	masters := a.masters.GetMasters()

	var wg sync.WaitGroup
	for _, master := range masters {
		a.gMutex.Lock()
		g, ok := a.groups[master]
		if !ok {
			g = newGroup(master)
			a.groups[master] = g
		}
		a.gMutex.Unlock()

		wg.Add(1)
		go a.checkMaster(&wg, g)
	}

	// wait all check done
	wg.Wait()

	a.gMutex.Lock()
	for master, g := range a.groups {
		if !a.masters.IsMaster(master) {
			delete(a.groups, master)
			g.Close()
		}
	}
	a.gMutex.Unlock()
}

func (a *App) checkMaster(wg *sync.WaitGroup, g *Group) {
	defer wg.Done()

	// later, add check strategy, like check failed n numbers in n seconds and do failover, etc.
	// now only check once.
	err := g.Check()
	if err == nil {
		return
	}

	oldMaster := g.Master.Addr

	if err == ErrNodeType {
		log.Errorf("server %s is not master now, we will skip it", oldMaster)

		// server is not master, we will not check it.
		a.delMasters([]string{oldMaster})
		return
	}

	errNum := time.Duration(g.CheckErrNum.Get())
	downTime := errNum * time.Duration(a.c.CheckInterval) * time.Millisecond
	if downTime < time.Duration(a.c.MaxDownTime)*time.Second {
		log.Warnf("check master %s err %v, down time: %0.2fs, retry check", oldMaster, err, downTime.Seconds())
		return
	}

	// If check error, we will remove it from saved masters and not check.
	// I just want to avoid some errors if below failover failed, at that time,
	// handling it manually seems a better way.
	// If you want to recheck it, please add it again.
	a.delMasters([]string{oldMaster})

	log.Errorf("check master %s err %v, do failover", oldMaster, err)

	a.onBeforeFailover(oldMaster)

	// first elect a candidate
	newMaster, err := g.Elect()
	if err != nil {
		// elect error
		return
	}

	log.Errorf("master is down, elect %s as new master, do failover", newMaster)

	// promote the candiate to master
	err = g.Promote(newMaster)

	if err != nil {
		log.Fatalf("do master %s failover err: %v", oldMaster, err)
		return
	}

	a.addMasters([]string{newMaster})

	a.onAfterFailover(oldMaster, newMaster)
}

func (a *App) startHTTP() {
	if a.l == nil {
		return
	}

	m := mux.NewRouter()

	m.Handle("/master", &masterHandler{a})

	s := http.Server{
		Handler: m,
	}

	s.Serve(a.l)
}

func (a *App) addMasters(addrs []string) error {
	if a.r != nil {
		return a.r.AddMasters(addrs, 0)
	} else {
		a.masters.AddMasters(addrs)
		return nil
	}
}

func (a *App) delMasters(addrs []string) error {
	if a.r != nil {
		return a.r.DelMasters(addrs, 0)
	} else {
		a.masters.DelMasters(addrs)
		return nil
	}
}

func (a *App) setMasters(addrs []string) error {
	if a.r != nil {
		return a.r.SetMasters(addrs, 0)
	} else {
		a.masters.SetMasters(addrs)
		return nil
	}

}

func (a *App) AddBeforeFailoverHandler(f BeforeFailoverHandler) {
	a.hMutex.Lock()
	a.beforeHandlers = append(a.beforeHandlers, f)
	a.hMutex.Unlock()
}

func (a *App) AddAfterFailoverHandler(f AfterFailoverHandler) {
	a.hMutex.Lock()
	a.afterHandlers = append(a.afterHandlers, f)
	a.hMutex.Unlock()
}

func (a *App) onBeforeFailover(downMaster string) {
	a.hMutex.Lock()
	for _, h := range a.beforeHandlers {
		if err := h(downMaster); err != nil {
			log.Errorf("do before failover handler for %s err: %v", downMaster, err)
		}
	}
	a.hMutex.Unlock()
}

func (a *App) onAfterFailover(downMaster string, newMaster string) {
	a.hMutex.Lock()
	for _, h := range a.afterHandlers {
		if err := h(downMaster, newMaster); err != nil {
			log.Errorf("do after failover handler for %s -> %s err: %v", downMaster, newMaster, err)
		}
	}

	a.hMutex.Unlock()
}
<commit_msg>add give up error<commit_after>package failover

import (
	"errors"
	"github.com/gorilla/mux"
	"github.com/siddontang/go/log"
	"net"
	"net/http"
	"sync"
	"time"
)

var (
	// If failover handler return this error, we will give up future handling.
	ErrGiveupFailover = errors.New("Give up failover handling")
)

type BeforeFailoverHandler func(downMaster string) error
type AfterFailoverHandler func(downMaster, newMaster string) error

type App struct {
	c *Config

	l net.Listener

	r *Raft

	masters *masterFSM

	gMutex sync.Mutex
	groups map[string]*Group

	quit chan struct{}
	wg   sync.WaitGroup

	hMutex         sync.Mutex
	beforeHandlers []BeforeFailoverHandler
	afterHandlers  []AfterFailoverHandler
}

func NewApp(c *Config) (*App, error) {
	var err error

	a := new(App)
	a.c = c
	a.quit = make(chan struct{})
	a.groups = make(map[string]*Group)

	a.masters = newMasterFSM()

	if c.MaxDownTime <= 0 {
		c.MaxDownTime = 3
	}

	if a.c.CheckInterval <= 0 {
		a.c.CheckInterval = 1000
	}

	if len(c.Addr) > 0 {
		a.l, err = net.Listen("tcp", c.Addr)
		if err != nil {
			return nil, err
		}
	}

	a.r, err = newRaft(c, a.masters)
	if err != nil {
		return nil, err
	}

	if c.MastersState == MastersStateNew {
		a.setMasters(c.Masters)
	} else {
		a.addMasters(c.Masters)
	}

	return a, nil
}

func (a *App) Close() {
	select {
	case <-a.quit:
		return
	default:
		break
	}

	if a.l != nil {
		a.l.Close()
	}

	if a.r != nil {
		a.r.Close()
	}

	close(a.quit)

	a.wg.Wait()
}

func (a *App) Run() {
	go a.startHTTP()

	a.wg.Add(1)
	t := time.NewTicker(time.Duration(a.c.CheckInterval) * time.Millisecond)
	defer func() {
		t.Stop()
		a.wg.Done()
	}()

	for {
		select {
		case <-t.C:
			a.check()
		case <-a.quit:
			return
		}
	}
}

func (a *App) check() {
	if a.r != nil && !a.r.IsLeader() {
		// is not leader, not check
		return
	}

	masters := a.masters.GetMasters()

	var wg sync.WaitGroup
	for _, master := range masters {
		a.gMutex.Lock()
		g, ok := a.groups[master]
		if !ok {
			g = newGroup(master)
			a.groups[master] = g
		}
		a.gMutex.Unlock()

		wg.Add(1)
		go a.checkMaster(&wg, g)
	}

	// wait all check done
	wg.Wait()

	a.gMutex.Lock()
	for master, g := range a.groups {
		if !a.masters.IsMaster(master) {
			delete(a.groups, master)
			g.Close()
		}
	}
	a.gMutex.Unlock()
}

func (a *App) checkMaster(wg *sync.WaitGroup, g *Group) {
	defer wg.Done()

	// later, add check strategy, like check failed n numbers in n seconds and do failover, etc.
	// now only check once.
	err := g.Check()
	if err == nil {
		return
	}

	oldMaster := g.Master.Addr

	if err == ErrNodeType {
		log.Errorf("server %s is not master now, we will skip it", oldMaster)

		// server is not master, we will not check it.
		a.delMasters([]string{oldMaster})
		return
	}

	errNum := time.Duration(g.CheckErrNum.Get())
	downTime := errNum * time.Duration(a.c.CheckInterval) * time.Millisecond
	if downTime < time.Duration(a.c.MaxDownTime)*time.Second {
		log.Warnf("check master %s err %v, down time: %0.2fs, retry check", oldMaster, err, downTime.Seconds())
		return
	}

	// If check error, we will remove it from saved masters and not check.
	// I just want to avoid some errors if below failover failed, at that time,
	// handling it manually seems a better way.
	// If you want to recheck it, please add it again.
	a.delMasters([]string{oldMaster})

	log.Errorf("check master %s err %v, do failover", oldMaster, err)

	if err := a.onBeforeFailover(oldMaster); err != nil {
		//give up failover
		return
	}

	// first elect a candidate
	newMaster, err := g.Elect()
	if err != nil {
		// elect error
		return
	}

	log.Errorf("master is down, elect %s as new master, do failover", newMaster)

	// promote the candiate to master
	err = g.Promote(newMaster)

	if err != nil {
		log.Fatalf("do master %s failover err: %v", oldMaster, err)
		return
	}

	a.addMasters([]string{newMaster})

	a.onAfterFailover(oldMaster, newMaster)
}

func (a *App) startHTTP() {
	if a.l == nil {
		return
	}

	m := mux.NewRouter()

	m.Handle("/master", &masterHandler{a})

	s := http.Server{
		Handler: m,
	}

	s.Serve(a.l)
}

func (a *App) addMasters(addrs []string) error {
	if a.r != nil {
		return a.r.AddMasters(addrs, 0)
	} else {
		a.masters.AddMasters(addrs)
		return nil
	}
}

func (a *App) delMasters(addrs []string) error {
	if a.r != nil {
		return a.r.DelMasters(addrs, 0)
	} else {
		a.masters.DelMasters(addrs)
		return nil
	}
}

func (a *App) setMasters(addrs []string) error {
	if a.r != nil {
		return a.r.SetMasters(addrs, 0)
	} else {
		a.masters.SetMasters(addrs)
		return nil
	}

}

func (a *App) AddBeforeFailoverHandler(f BeforeFailoverHandler) {
	a.hMutex.Lock()
	a.beforeHandlers = append(a.beforeHandlers, f)
	a.hMutex.Unlock()
}

func (a *App) AddAfterFailoverHandler(f AfterFailoverHandler) {
	a.hMutex.Lock()
	a.afterHandlers = append(a.afterHandlers, f)
	a.hMutex.Unlock()
}

func (a *App) onBeforeFailover(downMaster string) error {
	a.hMutex.Lock()
	defer a.hMutex.Unlock()

	for _, h := range a.beforeHandlers {
		if err := h(downMaster); err != nil {
			log.Errorf("do before failover handler for %s err: %v", downMaster, err)
			if err == ErrGiveupFailover {
				return ErrGiveupFailover
			}
		}
	}

	return nil
}

func (a *App) onAfterFailover(downMaster string, newMaster string) error {
	a.hMutex.Lock()
	defer a.hMutex.Unlock()

	for _, h := range a.afterHandlers {
		if err := h(downMaster, newMaster); err != nil {
			log.Errorf("do after failover handler for %s -> %s err: %v", downMaster, newMaster, err)
			if err == ErrGiveupFailover {
				return ErrGiveupFailover
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package raft

import (
	"fmt"
	"log"
	"net"
)

type followerReplication struct {
	peer     net.Addr
	inflight *inflight

	stopCh    chan uint64
	triggerCh chan struct{}

	currentTerm uint64
	matchIndex  uint64
	nextIndex   uint64
}

// replicate is a long running routine that is used to manage
// the process of replicating logs to our followers
func (r *Raft) replicate(s *followerReplication) {
	// Start an async heartbeating routing
	stopHeartbeat := make(chan struct{})
	defer close(stopHeartbeat)
	r.goFunc(func() { r.heartbeat(s, stopHeartbeat) })

	shouldStop := false
	for !shouldStop {
		select {
		case <-s.triggerCh:
			shouldStop = r.replicateTo(s, r.getLastLogIndex())
		case <-randomTimeout(r.conf.CommitTimeout):
			shouldStop = r.replicateTo(s, r.getLastLogIndex())
		case maxIndex := <-s.stopCh:
			// Make a best effort to replicate up to this index
			if maxIndex > 0 {
				r.replicateTo(s, maxIndex)
			}
			return
		}
	}
}

// replicateTo is used to replicate the logs up to a given last index.
// If the follower log is behind, we take care to bring them up to date
func (r *Raft) replicateTo(s *followerReplication, lastIndex uint64) (shouldStop bool) {
	// Create the base request
	var l Log
	var req AppendEntriesRequest
	var resp AppendEntriesResponse
	var maxIndex uint64
START:
	req = AppendEntriesRequest{
		Term:              s.currentTerm,
		Leader:            r.localAddr,
		LeaderCommitIndex: r.getCommitIndex(),
	}

	// Get the previous log entry based on the nextIndex.
	// Guard for the first index, since there is no 0 log entry
	if s.nextIndex > 1 {
		if err := r.logs.GetLog(s.nextIndex-1, &l); err != nil {
			if err == LogNotFound {
				goto SEND_SNAP
			}
			log.Printf("[ERR] Failed to get log at index %d: %v",
				s.nextIndex-1, err)
			return
		}

		// Set the previous index and term (0 if nextIndex is 1)
		req.PrevLogEntry = l.Index
		req.PrevLogTerm = l.Term
	} else {
		req.PrevLogEntry = 0
		req.PrevLogTerm = 0
	}

	// Append up to MaxAppendEntries or up to the lastIndex
	req.Entries = make([]*Log, 0, r.conf.MaxAppendEntries)
	maxIndex = min(s.nextIndex+uint64(r.conf.MaxAppendEntries)-1, lastIndex)
	for i := s.nextIndex; i <= maxIndex; i++ {
		oldLog := new(Log)
		if err := r.logs.GetLog(i, oldLog); err != nil {
			if err == LogNotFound {
				goto SEND_SNAP
			}
			log.Printf("[ERR] Failed to get log at index %d: %v", i, err)
			return
		}
		req.Entries = append(req.Entries, oldLog)
	}

	// Make the RPC call
	if err := r.trans.AppendEntries(s.peer, &req, &resp); err != nil {
		log.Printf("[ERR] Failed to AppendEntries to %v: %v", s.peer, err)
		return
	}

	// Check for a newer term, stop running
	if resp.Term > req.Term {
		return true
	}

	// Update the s based on success
	if resp.Success {
		// Mark any inflight logs as committed
		for i := s.matchIndex; i <= maxIndex; i++ {
			s.inflight.Commit(i, s.peer)
		}

		s.matchIndex = maxIndex
		s.nextIndex = maxIndex + 1
	} else {
		log.Printf("[WARN] AppendEntries to %v rejected, sending older logs", s.peer)
		s.nextIndex = max(min(s.nextIndex-1, resp.LastLog+1), 1)
		s.matchIndex = s.nextIndex - 1
	}

CHECK_MORE:
	// Check if there are more logs to replicate
	if s.nextIndex <= lastIndex {
		goto START
	}
	return

	// SEND_SNAP is used when we fail to get a log, usually because the follower
	// is too far behind, and we must ship a snapshot down instead
SEND_SNAP:
	stop, err := r.sendLatestSnapshot(s)

	// Check if we should stop
	if stop {
		return true
	}

	// Check for an error
	if err != nil {
		log.Printf("[ERR] Failed to send snapshot to %v: %v", s.peer, err)
		return
	}

	// Check if there is more to replicate
	goto CHECK_MORE
}

// sendLatestSnapshot is used to send the latest snapshot we have
// down to our follower
func (r *Raft) sendLatestSnapshot(s *followerReplication) (bool, error) {
	// Get the snapshots
	snapshots, err := r.snapshots.List()
	if err != nil {
		log.Printf("[ERR] Failed to list snapshots: %v", err)
		return false, err
	}

	// Check we have at least a single snapshot
	if len(snapshots) == 0 {
		return false, fmt.Errorf("no snapshots found")
	}

	// Open the most recent snapshot
	snapId := snapshots[0].ID
	meta, snapshot, err := r.snapshots.Open(snapId)
	if err != nil {
		log.Printf("[ERR] Failed to open snapshot %v: %v", snapId, err)
		return false, err
	}
	defer snapshot.Close()

	// Setup the request
	req := InstallSnapshotRequest{
		Term:         s.currentTerm,
		Leader:       r.localAddr,
		LastLogIndex: meta.Index,
		LastLogTerm:  meta.Term,
		Peers:        decodePeers(meta.Peers, r.trans),
		Size:         meta.Size,
	}

	// Make the call
	var resp InstallSnapshotResponse
	if err := r.trans.InstallSnapshot(s.peer, &req, &resp, snapshot); err != nil {
		log.Printf("[ERR] Failed to install snapshot %v: %v", snapId, err)
		return false, err
	}

	// Check for a newer term, stop running
	if resp.Term > req.Term {
		return true, nil
	}

	// Check for success
	if resp.Success {
		// Mark any inflight logs as committed
		for i := s.matchIndex; i <= meta.Index; i++ {
			s.inflight.Commit(i, s.peer)
		}

		// Update the indexes
		s.matchIndex = meta.Index
		s.nextIndex = s.matchIndex + 1
	} else {
		log.Printf("[WARN] InstallSnapshot to %v rejected", s.peer)
	}
	return false, nil
}

// hearbeat is used to periodically invoke AppendEntries on a peer
// to ensure they don't time out. This is done async of replicate(),
// since that routine could potentially be blocked on disk IO
func (r *Raft) heartbeat(s *followerReplication, stopCh chan struct{}) {
	req := AppendEntriesRequest{
		Term:   s.currentTerm,
		Leader: r.localAddr,
	}
	var resp AppendEntriesResponse
	for {
		select {
		case <-randomTimeout(r.conf.HeartbeatTimeout / 4):
			if err := r.trans.AppendEntries(s.peer, &req, &resp); err != nil {
				log.Printf("[ERR] Failed to heartbeat to %v: %v", s.peer, err)
			}
		case <-stopCh:
			return
		}
	}
}
<commit_msg>Properly encode peers<commit_after>package raft

import (
	"fmt"
	"log"
	"net"
)

type followerReplication struct {
	peer     net.Addr
	inflight *inflight

	stopCh    chan uint64
	triggerCh chan struct{}

	currentTerm uint64
	matchIndex  uint64
	nextIndex   uint64
}

// replicate is a long running routine that is used to manage
// the process of replicating logs to our followers
func (r *Raft) replicate(s *followerReplication) {
	// Start an async heartbeating routing
	stopHeartbeat := make(chan struct{})
	defer close(stopHeartbeat)
	r.goFunc(func() { r.heartbeat(s, stopHeartbeat) })

	shouldStop := false
	for !shouldStop {
		select {
		case <-s.triggerCh:
			shouldStop = r.replicateTo(s, r.getLastLogIndex())
		case <-randomTimeout(r.conf.CommitTimeout):
			shouldStop = r.replicateTo(s, r.getLastLogIndex())
		case maxIndex := <-s.stopCh:
			// Make a best effort to replicate up to this index
			if maxIndex > 0 {
				r.replicateTo(s, maxIndex)
			}
			return
		}
	}
}

// replicateTo is used to replicate the logs up to a given last index.
// If the follower log is behind, we take care to bring them up to date
func (r *Raft) replicateTo(s *followerReplication, lastIndex uint64) (shouldStop bool) {
	// Create the base request
	var l Log
	var req AppendEntriesRequest
	var resp AppendEntriesResponse
	var maxIndex uint64
START:
	req = AppendEntriesRequest{
		Term:              s.currentTerm,
		Leader:            r.trans.EncodePeer(r.localAddr),
		LeaderCommitIndex: r.getCommitIndex(),
	}

	// Get the previous log entry based on the nextIndex.
	// Guard for the first index, since there is no 0 log entry
	if s.nextIndex > 1 {
		if err := r.logs.GetLog(s.nextIndex-1, &l); err != nil {
			if err == LogNotFound {
				goto SEND_SNAP
			}
			log.Printf("[ERR] Failed to get log at index %d: %v",
				s.nextIndex-1, err)
			return
		}

		// Set the previous index and term (0 if nextIndex is 1)
		req.PrevLogEntry = l.Index
		req.PrevLogTerm = l.Term
	} else {
		req.PrevLogEntry = 0
		req.PrevLogTerm = 0
	}

	// Append up to MaxAppendEntries or up to the lastIndex
	req.Entries = make([]*Log, 0, r.conf.MaxAppendEntries)
	maxIndex = min(s.nextIndex+uint64(r.conf.MaxAppendEntries)-1, lastIndex)
	for i := s.nextIndex; i <= maxIndex; i++ {
		oldLog := new(Log)
		if err := r.logs.GetLog(i, oldLog); err != nil {
			if err == LogNotFound {
				goto SEND_SNAP
			}
			log.Printf("[ERR] Failed to get log at index %d: %v", i, err)
			return
		}
		req.Entries = append(req.Entries, oldLog)
	}

	// Make the RPC call
	if err := r.trans.AppendEntries(s.peer, &req, &resp); err != nil {
		log.Printf("[ERR] Failed to AppendEntries to %v: %v", s.peer, err)
		return
	}

	// Check for a newer term, stop running
	if resp.Term > req.Term {
		return true
	}

	// Update the s based on success
	if resp.Success {
		// Mark any inflight logs as committed
		for i := s.matchIndex; i <= maxIndex; i++ {
			s.inflight.Commit(i, s.peer)
		}

		s.matchIndex = maxIndex
		s.nextIndex = maxIndex + 1
	} else {
		log.Printf("[WARN] AppendEntries to %v rejected, sending older logs", s.peer)
		s.nextIndex = max(min(s.nextIndex-1, resp.LastLog+1), 1)
		s.matchIndex = s.nextIndex - 1
	}

CHECK_MORE:
	// Check if there are more logs to replicate
	if s.nextIndex <= lastIndex {
		goto START
	}
	return

	// SEND_SNAP is used when we fail to get a log, usually because the follower
	// is too far behind, and we must ship a snapshot down instead
SEND_SNAP:
	stop, err := r.sendLatestSnapshot(s)

	// Check if we should stop
	if stop {
		return true
	}

	// Check for an error
	if err != nil {
		log.Printf("[ERR] Failed to send snapshot to %v: %v", s.peer, err)
		return
	}

	// Check if there is more to replicate
	goto CHECK_MORE
}

// sendLatestSnapshot is used to send the latest snapshot we have
// down to our follower
func (r *Raft) sendLatestSnapshot(s *followerReplication) (bool, error) {
	// Get the snapshots
	snapshots, err := r.snapshots.List()
	if err != nil {
		log.Printf("[ERR] Failed to list snapshots: %v", err)
		return false, err
	}

	// Check we have at least a single snapshot
	if len(snapshots) == 0 {
		return false, fmt.Errorf("no snapshots found")
	}

	// Open the most recent snapshot
	snapId := snapshots[0].ID
	meta, snapshot, err := r.snapshots.Open(snapId)
	if err != nil {
		log.Printf("[ERR] Failed to open snapshot %v: %v", snapId, err)
		return false, err
	}
	defer snapshot.Close()

	// Setup the request
	req := InstallSnapshotRequest{
		Term:         s.currentTerm,
		Leader:       r.trans.EncodePeer(r.localAddr),
		LastLogIndex: meta.Index,
		LastLogTerm:  meta.Term,
		Peers:        meta.Peers,
		Size:         meta.Size,
	}

	// Make the call
	var resp InstallSnapshotResponse
	if err := r.trans.InstallSnapshot(s.peer, &req, &resp, snapshot); err != nil {
		log.Printf("[ERR] Failed to install snapshot %v: %v", snapId, err)
		return false, err
	}

	// Check for a newer term, stop running
	if resp.Term > req.Term {
		return true, nil
	}

	// Check for success
	if resp.Success {
		// Mark any inflight logs as committed
		for i := s.matchIndex; i <= meta.Index; i++ {
			s.inflight.Commit(i, s.peer)
		}

		// Update the indexes
		s.matchIndex = meta.Index
		s.nextIndex = s.matchIndex + 1
	} else {
		log.Printf("[WARN] InstallSnapshot to %v rejected", s.peer)
	}
	return false, nil
}

// hearbeat is used to periodically invoke AppendEntries on a peer
// to ensure they don't time out. This is done async of replicate(),
// since that routine could potentially be blocked on disk IO
func (r *Raft) heartbeat(s *followerReplication, stopCh chan struct{}) {
	req := AppendEntriesRequest{
		Term:   s.currentTerm,
		Leader: r.trans.EncodePeer(r.localAddr),
	}
	var resp AppendEntriesResponse
	for {
		select {
		case <-randomTimeout(r.conf.HeartbeatTimeout / 4):
			if err := r.trans.AppendEntries(s.peer, &req, &resp); err != nil {
				log.Printf("[ERR] Failed to heartbeat to %v: %v", s.peer, err)
			}
		case <-stopCh:
			return
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>log(scraping): include full line to be parsed<commit_after><|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package repo

import (
	"encoding/base64"
	"errors"
	"fmt"
	"io/ioutil"
	"path"
	"path/filepath"
	"strings"

	"github.com/go-martini/martini"

	"github.com/gogits/git"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/modules/auth"
	"github.com/gogits/gogs/modules/base"
	"github.com/gogits/gogs/modules/log"
	"github.com/gogits/gogs/modules/middleware"
)

func Create(ctx *middleware.Context) {
	ctx.Data["Title"] = "Create repository"
	ctx.Data["PageIsNewRepo"] = true
	ctx.Data["LanguageIgns"] = models.LanguageIgns
	ctx.Data["Licenses"] = models.Licenses
	ctx.HTML(200, "repo/create")
}

func CreatePost(ctx *middleware.Context, form auth.CreateRepoForm) {
	ctx.Data["Title"] = "Create repository"
	ctx.Data["PageIsNewRepo"] = true
	ctx.Data["LanguageIgns"] = models.LanguageIgns
	ctx.Data["Licenses"] = models.Licenses

	if ctx.HasError() {
		ctx.HTML(200, "repo/create")
		return
	}

	repo, err := models.CreateRepository(ctx.User, form.RepoName, form.Description,
		form.Language, form.License, form.Private, false, form.InitReadme)
	if err == nil {
		log.Trace("%s Repository created: %s/%s", ctx.Req.RequestURI, ctx.User.LowerName, form.RepoName)
		ctx.Redirect("/" + ctx.User.Name + "/" + form.RepoName)
		return
	} else if err == models.ErrRepoAlreadyExist {
		ctx.RenderWithErr("Repository name has already been used", "repo/create", &form)
		return
	} else if err == models.ErrRepoNameIllegal {
		ctx.RenderWithErr(models.ErrRepoNameIllegal.Error(), "repo/create", &form)
		return
	}

	if repo != nil {
		if errDelete := models.DeleteRepository(ctx.User.Id, repo.Id, ctx.User.Name); errDelete != nil {
			log.Error("repo.MigratePost(CreatePost): %v", errDelete)
		}
	}
	ctx.Handle(500, "repo.Create", err)
}

func Migrate(ctx *middleware.Context) {
	ctx.Data["Title"] = "Migrate repository"
	ctx.Data["PageIsNewRepo"] = true
	ctx.HTML(200, "repo/migrate")
}

func MigratePost(ctx *middleware.Context, form auth.MigrateRepoForm) {
	ctx.Data["Title"] = "Migrate repository"
	ctx.Data["PageIsNewRepo"] = true

	if ctx.HasError() {
		ctx.HTML(200, "repo/migrate")
		return
	}

	url := strings.Replace(form.Url, "://", fmt.Sprintf("://%s:%s@", form.AuthUserName, form.AuthPasswd), 1)
	repo, err := models.MigrateRepository(ctx.User, form.RepoName, form.Description, form.Private,
		form.Mirror, url)
	if err == nil {
		log.Trace("%s Repository migrated: %s/%s", ctx.Req.RequestURI, ctx.User.LowerName, form.RepoName)
		ctx.Redirect("/" + ctx.User.Name + "/" + form.RepoName)
		return
	} else if err == models.ErrRepoAlreadyExist {
		ctx.RenderWithErr("Repository name has already been used", "repo/migrate", &form)
		return
	} else if err == models.ErrRepoNameIllegal {
		ctx.RenderWithErr(models.ErrRepoNameIllegal.Error(), "repo/migrate", &form)
		return
	}

	if repo != nil {
		if errDelete := models.DeleteRepository(ctx.User.Id, repo.Id, ctx.User.Name); errDelete != nil {
			log.Error("repo.MigratePost(DeleteRepository): %v", errDelete)
		}
	}

	if strings.Contains(err.Error(), "Authentication failed") {
		ctx.RenderWithErr(err.Error(), "repo/migrate", &form)
		return
	}
	ctx.Handle(500, "repo.Migrate", err)
}

func Single(ctx *middleware.Context, params martini.Params) {
	branchName := ctx.Repo.BranchName
	userName := ctx.Repo.Owner.Name
	repoName := ctx.Repo.Repository.Name

	repoLink := ctx.Repo.RepoLink
	branchLink := ctx.Repo.RepoLink + "/src/" + branchName
	rawLink := ctx.Repo.RepoLink + "/raw/" + branchName

	// Get tree path
	treename := params["_1"]

	if len(treename) > 0 && treename[len(treename)-1] == '/' {
		ctx.Redirect(repoLink + "/src/" + branchName + "/" + treename[:len(treename)-1])
		return
	}

	ctx.Data["IsRepoToolbarSource"] = true

	isViewBranch := ctx.Repo.IsBranch
	ctx.Data["IsViewBranch"] = isViewBranch

	treePath := treename
	if len(treePath) != 0 {
		treePath = treePath + "/"
	}

	entry, err := ctx.Repo.Commit.GetTreeEntryByPath(treename)
	if err != nil && err != git.ErrNotExist {
		ctx.Handle(404, "repo.Single(GetTreeEntryByPath)", err)
		return
	}

	if len(treename) != 0 && entry == nil {
		ctx.Handle(404, "repo.Single", nil)
		return
	}

	if entry != nil && !entry.IsDir() {
		blob := entry.Blob()

		if dataRc, err := blob.Data(); err != nil {
			ctx.Handle(404, "repo.Single(blob.Data)", err)
		} else {
			ctx.Data["FileSize"] = blob.Size()
			ctx.Data["IsFile"] = true
			ctx.Data["FileName"] = blob.Name()
			ext := path.Ext(blob.Name())
			if len(ext) > 0 {
				ext = ext[1:]
			}
			ctx.Data["FileExt"] = ext
			ctx.Data["FileLink"] = rawLink + "/" + treename

			buf := make([]byte, 1024)
			n, _ := dataRc.Read(buf)
			if n > 0 {
				buf = buf[:n]
			}

			defer func() {
				dataRc.Close()
			}()

			_, isTextFile := base.IsTextFile(buf)
			_, isImageFile := base.IsImageFile(buf)
			ctx.Data["FileIsText"] = isTextFile

			if isImageFile {
				ctx.Data["IsImageFile"] = true
			} else {
				d, _ := ioutil.ReadAll(dataRc)
				buf = append(buf, d...)
				readmeExist := base.IsMarkdownFile(blob.Name()) || base.IsReadmeFile(blob.Name())
				ctx.Data["ReadmeExist"] = readmeExist
				if readmeExist {
					ctx.Data["FileContent"] = string(base.RenderMarkdown(buf, ""))
				} else {
					if isTextFile {
						ctx.Data["FileContent"] = string(buf)
					}
				}
			}
		}

	} else {
		// Directory and file list.
		tree, err := ctx.Repo.Commit.SubTree(treename)
		if err != nil {
			ctx.Handle(404, "repo.Single(SubTree)", err)
			return
		}
		entries := tree.ListEntries()
		entries.Sort()

		files := make([][]interface{}, 0, len(entries))

		for _, te := range entries {
			c, err := ctx.Repo.Commit.GetCommitOfRelPath(filepath.Join(treePath, te.Name()))
			if err != nil {
				ctx.Handle(404, "repo.Single(SubTree)", err)
				return
			}

			files = append(files, []interface{}{te, c})
		}

		ctx.Data["Files"] = files

		var readmeFile *git.Blob

		for _, f := range entries {
			if f.IsDir() || !base.IsReadmeFile(f.Name()) {
				continue
			} else {
				readmeFile = f.Blob()
				break
			}
		}

		if readmeFile != nil {
			ctx.Data["ReadmeInSingle"] = true
			ctx.Data["ReadmeExist"] = true
			if dataRc, err := readmeFile.Data(); err != nil {
				ctx.Handle(404, "repo.Single(readmeFile.LookupBlob)", err)
				return
			} else {

				buf := make([]byte, 1024)
				n, _ := dataRc.Read(buf)
				if n > 0 {
					buf = buf[:n]
				}
				defer func() {
					dataRc.Close()
				}()

				ctx.Data["FileSize"] = readmeFile.Size
				ctx.Data["FileLink"] = rawLink + "/" + treename
				_, isTextFile := base.IsTextFile(buf)
				ctx.Data["FileIsText"] = isTextFile
				ctx.Data["FileName"] = readmeFile.Name()
				if isTextFile {
					d, _ := ioutil.ReadAll(dataRc)
					buf = append(buf, d...)
					ctx.Data["FileContent"] = string(base.RenderMarkdown(buf, branchLink))
				}
			}
		}
	}

	ctx.Data["Username"] = userName
	ctx.Data["Reponame"] = repoName

	var treenames []string
	Paths := make([]string, 0)

	if len(treename) > 0 {
		treenames = strings.Split(treename, "/")
		for i, _ := range treenames {
			Paths = append(Paths, strings.Join(treenames[0:i+1], "/"))
		}

		ctx.Data["HasParentPath"] = true
		if len(Paths)-2 >= 0 {
			ctx.Data["ParentPath"] = "/" + Paths[len(Paths)-2]
		}
	}

	ctx.Data["LastCommit"] = ctx.Repo.Commit
	ctx.Data["Paths"] = Paths
	ctx.Data["TreeName"] = treename
	ctx.Data["Treenames"] = treenames
	ctx.Data["TreePath"] = treePath
	ctx.Data["BranchLink"] = branchLink
	ctx.HTML(200, "repo/single")
}

func basicEncode(username, password string) string {
	auth := username + ":" + password
	return base64.StdEncoding.EncodeToString([]byte(auth))
}

func basicDecode(encoded string) (user string, name string, err error) {
	var s []byte
	s, err = base64.StdEncoding.DecodeString(encoded)
	if err != nil {
		return
	}

	a := strings.Split(string(s), ":")
	if len(a) == 2 {
		user, name = a[0], a[1]
	} else {
		err = errors.New("decode failed")
	}
	return
}

func authRequired(ctx *middleware.Context) {
	ctx.ResponseWriter.Header().Set("WWW-Authenticate", "Basic realm=\".\"")
	ctx.Data["ErrorMsg"] = "no basic auth and digit auth"
	ctx.HTML(401, fmt.Sprintf("status/401"))
}

func Action(ctx *middleware.Context, params martini.Params) {
	var err error
	switch params["action"] {
	case "watch":
		err = models.WatchRepo(ctx.User.Id, ctx.Repo.Repository.Id, true)
	case "unwatch":
		err = models.WatchRepo(ctx.User.Id, ctx.Repo.Repository.Id, false)
	case "desc":
		if !ctx.Repo.IsOwner {
			ctx.Error(404)
			return
		}

		ctx.Repo.Repository.Description = ctx.Query("desc")
		ctx.Repo.Repository.Website = ctx.Query("site")
		err = models.UpdateRepository(ctx.Repo.Repository)
	}

	if err != nil {
		log.Error("repo.Action(%s): %v", params["action"], err)
		ctx.JSON(200, map[string]interface{}{
			"ok":  false,
			"err": err.Error(),
		})
		return
	}
	ctx.JSON(200, map[string]interface{}{
		"ok": true,
	})
}
<commit_msg>display large file<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package repo

import (
	"encoding/base64"
	"errors"
	"fmt"
	"io/ioutil"
	"path"
	"path/filepath"
	"strings"

	"github.com/go-martini/martini"

	"github.com/gogits/git"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/modules/auth"
	"github.com/gogits/gogs/modules/base"
	"github.com/gogits/gogs/modules/log"
	"github.com/gogits/gogs/modules/middleware"
)

func Create(ctx *middleware.Context) {
	ctx.Data["Title"] = "Create repository"
	ctx.Data["PageIsNewRepo"] = true
	ctx.Data["LanguageIgns"] = models.LanguageIgns
	ctx.Data["Licenses"] = models.Licenses
	ctx.HTML(200, "repo/create")
}

func CreatePost(ctx *middleware.Context, form auth.CreateRepoForm) {
	ctx.Data["Title"] = "Create repository"
	ctx.Data["PageIsNewRepo"] = true
	ctx.Data["LanguageIgns"] = models.LanguageIgns
	ctx.Data["Licenses"] = models.Licenses

	if ctx.HasError() {
		ctx.HTML(200, "repo/create")
		return
	}

	repo, err := models.CreateRepository(ctx.User, form.RepoName, form.Description,
		form.Language, form.License, form.Private, false, form.InitReadme)
	if err == nil {
		log.Trace("%s Repository created: %s/%s", ctx.Req.RequestURI, ctx.User.LowerName, form.RepoName)
		ctx.Redirect("/" + ctx.User.Name + "/" + form.RepoName)
		return
	} else if err == models.ErrRepoAlreadyExist {
		ctx.RenderWithErr("Repository name has already been used", "repo/create", &form)
		return
	} else if err == models.ErrRepoNameIllegal {
		ctx.RenderWithErr(models.ErrRepoNameIllegal.Error(), "repo/create", &form)
		return
	}

	if repo != nil {
		if errDelete := models.DeleteRepository(ctx.User.Id, repo.Id, ctx.User.Name); errDelete != nil {
			log.Error("repo.MigratePost(CreatePost): %v", errDelete)
		}
	}
	ctx.Handle(500, "repo.Create", err)
}

func Migrate(ctx *middleware.Context) {
	ctx.Data["Title"] = "Migrate repository"
	ctx.Data["PageIsNewRepo"] = true
	ctx.HTML(200, "repo/migrate")
}

func MigratePost(ctx *middleware.Context, form auth.MigrateRepoForm) {
	ctx.Data["Title"] = "Migrate repository"
	ctx.Data["PageIsNewRepo"] = true

	if ctx.HasError() {
		ctx.HTML(200, "repo/migrate")
		return
	}

	url := strings.Replace(form.Url, "://", fmt.Sprintf("://%s:%s@", form.AuthUserName, form.AuthPasswd), 1)
	repo, err := models.MigrateRepository(ctx.User, form.RepoName, form.Description, form.Private,
		form.Mirror, url)
	if err == nil {
		log.Trace("%s Repository migrated: %s/%s", ctx.Req.RequestURI, ctx.User.LowerName, form.RepoName)
		ctx.Redirect("/" + ctx.User.Name + "/" + form.RepoName)
		return
	} else if err == models.ErrRepoAlreadyExist {
		ctx.RenderWithErr("Repository name has already been used", "repo/migrate", &form)
		return
	} else if err == models.ErrRepoNameIllegal {
		ctx.RenderWithErr(models.ErrRepoNameIllegal.Error(), "repo/migrate", &form)
		return
	}

	if repo != nil {
		if errDelete := models.DeleteRepository(ctx.User.Id, repo.Id, ctx.User.Name); errDelete != nil {
			log.Error("repo.MigratePost(DeleteRepository): %v", errDelete)
		}
	}

	if strings.Contains(err.Error(), "Authentication failed") {
		ctx.RenderWithErr(err.Error(), "repo/migrate", &form)
		return
	}
	ctx.Handle(500, "repo.Migrate", err)
}

func Single(ctx *middleware.Context, params martini.Params) {
	branchName := ctx.Repo.BranchName
	userName := ctx.Repo.Owner.Name
	repoName := ctx.Repo.Repository.Name

	repoLink := ctx.Repo.RepoLink
	branchLink := ctx.Repo.RepoLink + "/src/" + branchName
	rawLink := ctx.Repo.RepoLink + "/raw/" + branchName

	// Get tree path
	treename := params["_1"]

	if len(treename) > 0 && treename[len(treename)-1] == '/' {
		ctx.Redirect(repoLink + "/src/" + branchName + "/" + treename[:len(treename)-1])
		return
	}

	ctx.Data["IsRepoToolbarSource"] = true

	isViewBranch := ctx.Repo.IsBranch
	ctx.Data["IsViewBranch"] = isViewBranch

	treePath := treename
	if len(treePath) != 0 {
		treePath = treePath + "/"
	}

	entry, err := ctx.Repo.Commit.GetTreeEntryByPath(treename)
	if err != nil && err != git.ErrNotExist {
		ctx.Handle(404, "repo.Single(GetTreeEntryByPath)", err)
		return
	}

	if len(treename) != 0 && entry == nil {
		ctx.Handle(404, "repo.Single", nil)
		return
	}

	if entry != nil && !entry.IsDir() {
		blob := entry.Blob()

		if dataRc, err := blob.Data(); err != nil {
			ctx.Handle(404, "repo.Single(blob.Data)", err)
		} else {
			ctx.Data["FileSize"] = blob.Size()
			ctx.Data["IsFile"] = true
			ctx.Data["FileName"] = blob.Name()
			ext := path.Ext(blob.Name())
			if len(ext) > 0 {
				ext = ext[1:]
			}
			ctx.Data["FileExt"] = ext
			ctx.Data["FileLink"] = rawLink + "/" + treename

			buf := make([]byte, 1024)
			n, _ := dataRc.Read(buf)
			if n > 0 {
				buf = buf[:n]
			}

			defer func() {
				dataRc.Close()
			}()

			_, isTextFile := base.IsTextFile(buf)
			_, isImageFile := base.IsImageFile(buf)
			ctx.Data["FileIsText"] = isTextFile

			switch {
			case isImageFile:
				ctx.Data["IsImageFile"] = true
			case isTextFile:
				d, _ := ioutil.ReadAll(dataRc)
				buf = append(buf, d...)
				readmeExist := base.IsMarkdownFile(blob.Name()) || base.IsReadmeFile(blob.Name())
				ctx.Data["ReadmeExist"] = readmeExist
				if readmeExist {
					ctx.Data["FileContent"] = string(base.RenderMarkdown(buf, ""))
				} else {
					ctx.Data["FileContent"] = string(buf)
				}
			}
		}

	} else {
		// Directory and file list.
		tree, err := ctx.Repo.Commit.SubTree(treename)
		if err != nil {
			ctx.Handle(404, "repo.Single(SubTree)", err)
			return
		}
		entries := tree.ListEntries()
		entries.Sort()

		files := make([][]interface{}, 0, len(entries))

		for _, te := range entries {
			c, err := ctx.Repo.Commit.GetCommitOfRelPath(filepath.Join(treePath, te.Name()))
			if err != nil {
				ctx.Handle(404, "repo.Single(SubTree)", err)
				return
			}

			files = append(files, []interface{}{te, c})
		}

		ctx.Data["Files"] = files

		var readmeFile *git.Blob

		for _, f := range entries {
			if f.IsDir() || !base.IsReadmeFile(f.Name()) {
				continue
			} else {
				readmeFile = f.Blob()
				break
			}
		}

		if readmeFile != nil {
			ctx.Data["ReadmeInSingle"] = true
			ctx.Data["ReadmeExist"] = true
			if dataRc, err := readmeFile.Data(); err != nil {
				ctx.Handle(404, "repo.Single(readmeFile.LookupBlob)", err)
				return
			} else {

				buf := make([]byte, 1024)
				n, _ := dataRc.Read(buf)
				if n > 0 {
					buf = buf[:n]
				}
				defer func() {
					dataRc.Close()
				}()

				ctx.Data["FileSize"] = readmeFile.Size
				ctx.Data["FileLink"] = rawLink + "/" + treename
				_, isTextFile := base.IsTextFile(buf)
				ctx.Data["FileIsText"] = isTextFile
				ctx.Data["FileName"] = readmeFile.Name()
				if isTextFile {
					d, _ := ioutil.ReadAll(dataRc)
					buf = append(buf, d...)
					ctx.Data["FileContent"] = string(base.RenderMarkdown(buf, branchLink))
				}
			}
		}
	}

	ctx.Data["Username"] = userName
	ctx.Data["Reponame"] = repoName

	var treenames []string
	Paths := make([]string, 0)

	if len(treename) > 0 {
		treenames = strings.Split(treename, "/")
		for i, _ := range treenames {
			Paths = append(Paths, strings.Join(treenames[0:i+1], "/"))
		}

		ctx.Data["HasParentPath"] = true
		if len(Paths)-2 >= 0 {
			ctx.Data["ParentPath"] = "/" + Paths[len(Paths)-2]
		}
	}

	ctx.Data["LastCommit"] = ctx.Repo.Commit
	ctx.Data["Paths"] = Paths
	ctx.Data["TreeName"] = treename
	ctx.Data["Treenames"] = treenames
	ctx.Data["TreePath"] = treePath
	ctx.Data["BranchLink"] = branchLink
	ctx.HTML(200, "repo/single")
}

func basicEncode(username, password string) string {
	auth := username + ":" + password
	return base64.StdEncoding.EncodeToString([]byte(auth))
}

func basicDecode(encoded string) (user string, name string, err error) {
	var s []byte
	s, err = base64.StdEncoding.DecodeString(encoded)
	if err != nil {
		return
	}

	a := strings.Split(string(s), ":")
	if len(a) == 2 {
		user, name = a[0], a[1]
	} else {
		err = errors.New("decode failed")
	}
	return
}

func authRequired(ctx *middleware.Context) {
	ctx.ResponseWriter.Header().Set("WWW-Authenticate", "Basic realm=\".\"")
	ctx.Data["ErrorMsg"] = "no basic auth and digit auth"
	ctx.HTML(401, fmt.Sprintf("status/401"))
}

func Action(ctx *middleware.Context, params martini.Params) {
	var err error
	switch params["action"] {
	case "watch":
		err = models.WatchRepo(ctx.User.Id, ctx.Repo.Repository.Id, true)
	case "unwatch":
		err = models.WatchRepo(ctx.User.Id, ctx.Repo.Repository.Id, false)
	case "desc":
		if !ctx.Repo.IsOwner {
			ctx.Error(404)
			return
		}

		ctx.Repo.Repository.Description = ctx.Query("desc")
		ctx.Repo.Repository.Website = ctx.Query("site")
		err = models.UpdateRepository(ctx.Repo.Repository)
	}

	if err != nil {
		log.Error("repo.Action(%s): %v", params["action"], err)
		ctx.JSON(200, map[string]interface{}{
			"ok":  false,
			"err": err.Error(),
		})
		return
	}
	ctx.JSON(200, map[string]interface{}{
		"ok": true,
	})
}
<|endoftext|>


<commit_before>// Copyright 2017 The qurl Authors. All rights reserved.

// Package routes implements all the HTTP entry points for this microservice.
package routes

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/labstack/echo"
)

func TestQuery(t *testing.T) {
	// Create a request to pass to our handler. We don't have any query
	// parameters for now, so we'll pass 'nil' as the third parameter.
	req, err := http.NewRequest("GET", "/q?url=https://www.example.com", nil)
	if err != nil {
		t.Fatal(err)
	}

	// We create a ResponseRecorder (which satisfies http.ResponseWriter) to
	// record the response.
	rec := httptest.NewRecorder()

	e := echo.New()
	c := e.NewContext(req, rec)
	err = Query(c)
	if err != nil {
		t.Fatalf("Query() failed %v", err)
	}

	// Check the status code is what we expect.
	if status := rec.Code; status != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusOK)
	}

	// Check the response body is what we expect.
	expected := "{\"url\":\"https://www.example.com\",\"status\":200}"
	if rec.Body.String() != expected {
		t.Errorf("handler returned unexpected body: got %v want %v", rec.Body.String(), expected)
	}
}

func TestQueryInvalidURL(t *testing.T) {
	// Create a request to pass to our handler. We don't have any query
	// parameters for now, so we'll pass 'nil' as the third parameter.
	req, err := http.NewRequest("GET", "/q?url=invalidurl", nil)
	if err != nil {
		t.Fatal(err)
	}

	// We create a ResponseRecorder (which satisfies http.ResponseWriter) to
	// record the response.
	rec := httptest.NewRecorder()

	e := echo.New()
	c := e.NewContext(req, rec)
	err = Query(c)
	if err != nil {
		t.Fatalf("Query() failed %v", err)
	}

	// Check the status code is what we expect.
	if status := rec.Code; status != http.StatusBadRequest {
		t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusBadRequest)
	}

	// Check the response body is what we expect.
	expected := "{\"url\":\"invalidurl\",\"status\":400}"
	if rec.Body.String() != expected {
		t.Errorf("handler returned unexpected body: got %v want %v", rec.Body.String(), expected)
	}
}
<commit_msg>Test for failing fetch url<commit_after>// Copyright 2017 The qurl Authors. All rights reserved.

// Package routes implements all the HTTP entry points for this microservice.
package routes

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/labstack/echo"
)

func TestQuery(t *testing.T) {
	// Create a request to pass to our handler. We don't have any query
	// parameters for now, so we'll pass 'nil' as the third parameter.
	req, err := http.NewRequest("GET", "/q?url=https://www.example.com", nil)
	if err != nil {
		t.Fatal(err)
	}

	// We create a ResponseRecorder (which satisfies http.ResponseWriter) to
	// record the response.
	rec := httptest.NewRecorder()

	e := echo.New()
	c := e.NewContext(req, rec)
	err = Query(c)
	if err != nil {
		t.Fatalf("Query() failed %v", err)
	}

	// Check the status code is what we expect.
	if status := rec.Code; status != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusOK)
	}

	// Check the response body is what we expect.
	expected := "{\"url\":\"https://www.example.com\",\"status\":200}"
	if rec.Body.String() != expected {
		t.Errorf("handler returned unexpected body: got %v want %v", rec.Body.String(), expected)
	}
}

func TestQueryInvalidURL(t *testing.T) {
	req, err := http.NewRequest("GET", "/q?url=invalidurl", nil)
	if err != nil {
		t.Fatal(err)
	}

	rec := httptest.NewRecorder()

	e := echo.New()
	c := e.NewContext(req, rec)
	err = Query(c)
	if err != nil {
		t.Fatalf("Query() failed %v", err)
	}

	// Check the status code is what we expect.
	if status := rec.Code; status != http.StatusBadRequest {
		t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusBadRequest)
	}

	// Check the response body is what we expect.
	expected := "{\"url\":\"invalidurl\",\"status\":400}"
	if rec.Body.String() != expected {
		t.Errorf("handler returned unexpected body: got %v want %v", rec.Body.String(), expected)
	}
}

func TestFailFetchURL(t *testing.T) {
	req, err := http.NewRequest("GET", "/q?url=http://localhost", nil)
	if err != nil {
		t.Fatal(err)
	}

	rec := httptest.NewRecorder()

	e := echo.New()
	c := e.NewContext(req, rec)
	err = Query(c)
	if err != nil {
		t.Fatalf("Query() failed %v", err)
	}

	// Check the status code is what we expect.
	if status := rec.Code; status != http.StatusInternalServerError {
		t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusInternalServerError)
	}

	// Check the response body is what we expect.
	expected := "{\"url\":\"http://localhost\",\"status\":500}"
	if rec.Body.String() != expected {
		t.Errorf("handler returned unexpected body: got %v want %v", rec.Body.String(), expected)
	}
}
<|endoftext|>