<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/urfave/cli"
)

var ErrWrongStateFile = errors.New("fconf: wrong state file")

type WifiState struct {
	Enabled bool  `json:"enabled"`
	Configg *Wifi `json:"config"`
}

func WifiClientCMD(ctx *cli.Context) error {
	if ctx.IsSet(enableFlag) {
		return EnableWifiClient(ctx)
	}
	if ctx.IsSet(disableFlag) {
		return DisableWifi(ctx)
	}
	if ctx.IsSet(removeFlag) {
		return RemoveWifi(ctx)
	}
	if ctx.IsSet(configFlag) {
		return configWifiClient(ctx)
	}
	return nil
}

//EnableWifiClient enables wifi client. If the config flag is set, wifi is
//configured before being enabled.
func EnableWifiClient(ctx *cli.Context) error {
	if ctx.IsSet(configFlag) {
		err := configWifiClient(ctx)
		if err != nil {
			return err
		}
	}
	i := getInterface(ctx)
	if i == "" {
		return errors.New("missing interface, you must specify interface")
	}
	w, err := wifiClientState(i)
	if err != nil {
		return err
	}
	if w.Configg.Interface == "" {
		w.Configg.Interface = "wlan0"
	}
	service := "wpa_supplicant@" + w.Configg.Interface
	err = restartService(service)
	if err != nil {
		return err
	}
	err = enableService(service)
	if err != nil {
		return err
	}
	err = restartService("systemd-networkd")
	if err != nil {
		return err
	}
	w.Enabled = true
	data, err := json.Marshal(w)
	if err != nil {
		return err
	}
	return keepState(
		fmt.Sprintf(defaultWifiClientConfig, w.Configg.Interface), data)

}

func wifiClientState(i string) (*WifiState, error) {
	dir := os.Getenv("FCONF_CONFIGDIR")
	if dir == "" {
		dir = fconfConfigDir
	}
	b, err := ioutil.ReadFile(filepath.Join(dir,
		fmt.Sprintf(defaultWifiClientConfig, i)))
	if err != nil {
		return nil, err
	}
	w := &WifiState{}
	err = json.Unmarshal(b, w)
	if err != nil {
		return nil, err
	}
	if w.Configg == nil {
		return nil, ErrWrongStateFile
	}
	return w, nil
}

func configWifiClient(ctx *cli.Context) error {
	base := ctx.String("dir")
	name := ctx.String("name")
	src := ctx.String("config")
	if src == "" {
		return errors.New("fconf: missing configuration source file")
	}
	var b []byte
	var err error
	if src == "stdin" {
		b, err = ReadFromStdin()
		if err != nil {
			return err
		}
	} else {
		b, err = ioutil.ReadFile(src)
		if err != nil {
			return err
		}
	}
	e := Wifi{}
	err = json.Unmarshal(b, &e)
	if err != nil {
		return err
	}
	if e.Interface == "" {
		e.Interface = "wlan0"
	}
	err = checkDir(base)
	if err != nil {
		return err
	}
	if strings.Contains(name, "%s") {
		name = fmt.Sprintf(name, e.Interface)
	}
	filename := filepath.Join(base, name)
	err = CreateSystemdFile(e, filename, 0644)
	if err != nil {
		return err
	}
	fmt.Printf("successful written wifi configuration to %s \n", filename)
	path := "/etc/wpa_supplicant/"
	err = checkDir(path)
	if err != nil {
		return err
	}
	cname := "wpa_supplicant-" + e.Interface + ".conf"
	s, err := wifiConfig(e.Username, e.Password)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(filepath.Join(path, cname), []byte(s), 0644)
	if err != nil {
		return err
	}
	state := &WifiState{Configg: &e}
	ws, err := wifiClientState(e.Interface)
	if err == nil {
		state.Enabled = ws.Enabled
	}
	b, _ = json.Marshal(state)
	fmt.Printf("successful written wifi connection  configuration to %s \n", filepath.Join(path, cname))
	setInterface(ctx, e.Interface)
	return keepState(
		fmt.Sprintf(defaultWifiClientConfig, e.Interface), b)
}

func wifiConfig(username, password string) (string, error) {
	cmd := "/usr/bin/wpa_passphrase"
	firstLine := "ctrl_interface=/run/wpa_supplicant_fconf"
	o, err := exec.Command(cmd, username, password).Output()
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("%s \n \n%s\n", firstLine, string(o)), nil
}

func DisableWifi(ctx *cli.Context) error {
	if ctx.IsSet(configFlag) {
		fmt.Println("WARN: config flag will be ignored when diable flag is used")
	}
	i := getInterface(ctx)
	if i == "" {
		return errors.New("missing interface, you must specify interface")
	}
	w, err := wifiClientState(i)
	if err != nil {
		return err
	}
	service := "wpa_supplicant@" + w.Configg.Interface
	err = disableService(service)
	if err != nil {
		return err
	}
	err = stopService(service)
	if err != nil {
		return err
	}
	err = restartService("systemd-networkd")
	if err != nil {
		return err
	}
	w.Enabled = false
	data, err := json.Marshal(w)
	if err != nil {
		return err
	}
	return keepState(
		fmt.Sprintf(defaultWifiClientConfig, i), data)
}

func RemoveWifi(ctx *cli.Context) error {
	i := getInterface(ctx)
	if i == "" {
		return errors.New("missing interface, you must specify interface")
	}
	w, err := wifiClientState(i)
	if err != nil {
		return err
	}

	// remove systemd file
	unit := filepath.Join(networkBase, wirelessService)
	err = removeFile(unit)
	if err != nil {
		return err
	}
	path := "/etc/wpa_supplicant/"
	cname := "wpa_supplicant-" + w.Configg.Interface + ".conf"

	// remove client connection
	err = removeFile(filepath.Join(path, cname))
	if err != nil {
		return err
	}
	err = DisableWifi(ctx)
	if err != nil {
		return err
	}

	// Remove any interface settings
	err = FlushInterface(w.Configg.Interface)
	if err != nil {
		return err
	}

	// remove the state file
	stateFile := filepath.Join(stateDir(),
		fmt.Sprintf(defaultWifiClientConfig, i))
	return removeFile(stateFile)
}
<commit_msg>wifi: flush and remove systemd file on disable<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/urfave/cli"
)

var ErrWrongStateFile = errors.New("fconf: wrong state file")

type WifiState struct {
	Enabled bool  `json:"enabled"`
	Configg *Wifi `json:"config"`
}

func WifiClientCMD(ctx *cli.Context) error {
	if ctx.IsSet(enableFlag) {
		return EnableWifiClient(ctx)
	}
	if ctx.IsSet(disableFlag) {
		return DisableWifi(ctx)
	}
	if ctx.IsSet(removeFlag) {
		return RemoveWifi(ctx)
	}
	if ctx.IsSet(configFlag) {
		return configWifiClient(ctx)
	}
	return nil
}

//EnableWifiClient enables wifi client. If the config flag is set, wifi is
//configured before being enabled.
func EnableWifiClient(ctx *cli.Context) error {
	if ctx.IsSet(configFlag) {
		err := configWifiClient(ctx)
		if err != nil {
			return err
		}
	}
	i := getInterface(ctx)
	if i == "" {
		return errors.New("missing interface, you must specify interface")
	}
	w, err := wifiClientState(i)
	if err != nil {
		return err
	}
	if w.Configg.Interface == "" {
		w.Configg.Interface = "wlan0"
	}
	service := "wpa_supplicant@" + w.Configg.Interface
	err = restartService(service)
	if err != nil {
		return err
	}
	err = enableService(service)
	if err != nil {
		return err
	}
	err = restartService("systemd-networkd")
	if err != nil {
		return err
	}
	w.Enabled = true
	data, err := json.Marshal(w)
	if err != nil {
		return err
	}
	return keepState(
		fmt.Sprintf(defaultWifiClientConfig, w.Configg.Interface), data)

}

func wifiClientState(i string) (*WifiState, error) {
	dir := os.Getenv("FCONF_CONFIGDIR")
	if dir == "" {
		dir = fconfConfigDir
	}
	b, err := ioutil.ReadFile(filepath.Join(dir,
		fmt.Sprintf(defaultWifiClientConfig, i)))
	if err != nil {
		return nil, err
	}
	w := &WifiState{}
	err = json.Unmarshal(b, w)
	if err != nil {
		return nil, err
	}
	if w.Configg == nil {
		return nil, ErrWrongStateFile
	}
	return w, nil
}

func configWifiClient(ctx *cli.Context) error {
	base := ctx.String("dir")
	name := ctx.String("name")
	src := ctx.String("config")
	if src == "" {
		return errors.New("fconf: missing configuration source file")
	}
	var b []byte
	var err error
	if src == "stdin" {
		b, err = ReadFromStdin()
		if err != nil {
			return err
		}
	} else {
		b, err = ioutil.ReadFile(src)
		if err != nil {
			return err
		}
	}
	e := Wifi{}
	err = json.Unmarshal(b, &e)
	if err != nil {
		return err
	}
	if e.Interface == "" {
		e.Interface = "wlan0"
	}
	err = checkDir(base)
	if err != nil {
		return err
	}
	if strings.Contains(name, "%s") {
		name = fmt.Sprintf(name, e.Interface)
	}
	filename := filepath.Join(base, name)
	err = CreateSystemdFile(e, filename, 0644)
	if err != nil {
		return err
	}
	fmt.Printf("successful written wifi configuration to %s \n", filename)
	path := "/etc/wpa_supplicant/"
	err = checkDir(path)
	if err != nil {
		return err
	}
	cname := "wpa_supplicant-" + e.Interface + ".conf"
	s, err := wifiConfig(e.Username, e.Password)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(filepath.Join(path, cname), []byte(s), 0644)
	if err != nil {
		return err
	}
	state := &WifiState{Configg: &e}
	ws, err := wifiClientState(e.Interface)
	if err == nil {
		state.Enabled = ws.Enabled
	}
	b, _ = json.Marshal(state)
	fmt.Printf("successful written wifi connection  configuration to %s \n", filepath.Join(path, cname))
	setInterface(ctx, e.Interface)
	return keepState(
		fmt.Sprintf(defaultWifiClientConfig, e.Interface), b)
}

func wifiConfig(username, password string) (string, error) {
	cmd := "/usr/bin/wpa_passphrase"
	firstLine := "ctrl_interface=/run/wpa_supplicant_fconf"
	o, err := exec.Command(cmd, username, password).Output()
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("%s \n \n%s\n", firstLine, string(o)), nil
}

func DisableWifi(ctx *cli.Context) error {
	if ctx.IsSet(configFlag) {
		fmt.Println("WARN: config flag will be ignored when diable flag is used")
	}
	i := getInterface(ctx)
	if i == "" {
		return errors.New("missing interface, you must specify interface")
	}
	w, err := wifiClientState(i)
	if err != nil {
		return err
	}
	service := "wpa_supplicant@" + w.Configg.Interface
	err = disableService(service)
	if err != nil {
		return err
	}
	err = stopService(service)
	if err != nil {
		return err
	}
	err = restartService("systemd-networkd")
	if err != nil {
		return err
	}
	w.Enabled = false
	data, err := json.Marshal(w)
	if err != nil {
		return err
	}
	_, err = exec.Command("ip", "addr", "flush", "dev", w.Configg.Interface).Output()
	if err != nil {
		return fmt.Errorf("ERROR: running ip addr flush dev %s %v",
			w.Configg.Interface, err,
		)
	}
	unit := filepath.Join(networkBase,
		fmt.Sprintf(wirelessService, w.Configg.Interface))
	err = removeFile(unit)
	if err != nil {
		return err
	}
	return keepState(
		fmt.Sprintf(defaultWifiClientConfig, i), data)
}

func RemoveWifi(ctx *cli.Context) error {
	i := getInterface(ctx)
	if i == "" {
		return errors.New("missing interface, you must specify interface")
	}
	w, err := wifiClientState(i)
	if err != nil {
		return err
	}

	// remove systemd file
	unit := filepath.Join(networkBase, wirelessService)
	err = removeFile(unit)
	if err != nil {
		return err
	}
	path := "/etc/wpa_supplicant/"
	cname := "wpa_supplicant-" + w.Configg.Interface + ".conf"

	// remove client connection
	err = removeFile(filepath.Join(path, cname))
	if err != nil {
		return err
	}
	err = DisableWifi(ctx)
	if err != nil {
		return err
	}

	// Remove any interface settings
	err = FlushInterface(w.Configg.Interface)
	if err != nil {
		return err
	}

	// remove the state file
	stateFile := filepath.Join(stateDir(),
		fmt.Sprintf(defaultWifiClientConfig, i))
	return removeFile(stateFile)
}
<|endoftext|>


<commit_before>// myLG is command line looking glass that written with Go language
// it tries from its own icmp and external looking glasses tools
package main

import (
	"errors"
	"github.com/briandowns/spinner"
	"net"
	"regexp"
	"strings"
	"time"

	"github.com/mehrdadrad/mylg/cli"
	"github.com/mehrdadrad/mylg/icmp"
	"github.com/mehrdadrad/mylg/lg"
	"github.com/mehrdadrad/mylg/ns"
	"github.com/mehrdadrad/mylg/peeringdb"
	"github.com/mehrdadrad/mylg/ripe"
)

// Provider represents looking glass
type Provider interface {
	Set(host, version string)
	GetDefaultNode() string
	GetNodes() []string
	ChangeNode(node string)
	Ping() (string, error)
	Trace() chan string
	BGP() chan string
}

// Whois represents whois providers
type Whois interface {
	Set(r string)
	GetData() bool
	PrettyPrint()
}

var (
	// register looking glass hosts
	providers = map[string]Provider{"telia": new(lg.Telia), "level3": new(lg.Level3), "cogent": new(lg.Cogent)}
	whois     = map[string]Whois{"asn": new(ripe.ASN), "prefix": new(ripe.Prefix)}
	pNames    = providerNames()
	nsr       *ns.Request
)

// providerName
func providerNames() []string {
	pNames := []string{}
	for p := range providers {
		pNames = append(pNames, p)
	}
	return pNames
}

// validateProvider
func validateProvider(p string) (string, error) {
	pNames := []string{}
	match, _ := regexp.MatchString("("+strings.Join(pNames, "|")+")", p)
	p = strings.ToLower(p)
	if match {
		return p, nil
	}
	return "", errors.New("provider not support")

}
func init() {
	// Initialize name server data
	nsr = ns.NewRequest()
	go nsr.Init()
}

func main() {
	var (
		err     error
		request string
		loop    = true
		cPName  = "local"
	)

	rep := make(chan string, 1)
	req := make(chan string, 1)
	nxt := make(chan struct{}, 1)

	c := cli.Init("local")
	go c.Run(req, nxt)

	r, _ := regexp.Compile(`(ping|trace|bgp|lg|ns|dig|whois|peering|connect|node|local|mode|help|exit|quit)\s{0,1}(.*)`)
	s := spinner.New(spinner.CharSets[26], 220*time.Millisecond)

	for loop {
		select {
		case request, loop = <-req:
			if !loop {
				break
			}
			if len(request) < 1 {
				c.Next()
				continue
			}
			subReq := r.FindStringSubmatch(request)
			if len(subReq) == 0 {
				println("syntax error")
				c.Next()
				continue
			}
			prompt := c.GetPrompt()
			cmd := strings.TrimSpace(subReq[1])
			args := strings.TrimSpace(subReq[2])
			switch {
			case cmd == "ping" && cPName == "local":
				p := icmp.NewPing()
				ra, err := net.ResolveIPAddr("ip", args)
				if err != nil {
					println("cannot resolve", args, ": Unknown host")
					c.Next()
					continue
				}
				p.IP(ra.String())
				for n := 0; n < 4; n++ {
					p.Ping(rep)
					println(<-rep)
				}
				c.Next()
			case cmd == "ping":
				s.Prefix = "please wait "
				s.Start()
				providers[cPName].Set(args, "ipv4")
				m, _ := providers[cPName].Ping()
				s.Stop()
				println(m)
				c.Next()
			case cmd == "trace":
				switch {
				case strings.HasPrefix(prompt, "local"):
					trace := icmp.Trace{}
					trace.Run(args)
				case strings.HasPrefix(prompt, "lg"):
					providers[cPName].Set(args, "ipv4")
					for l := range providers[cPName].Trace() {
						println(l)
					}
				}
				c.Next()
			case cmd == "bgp":
				providers[cPName].Set(args, "ipv4")
				for l := range providers[cPName].BGP() {
					println(l)
				}
				c.Next()
			case cmd == "dig":
				nsr.Dig(args)
				c.Next()
			case cmd == "node":
				switch {
				case strings.HasPrefix(prompt, "lg"):
					if _, ok := providers[cPName]; ok {
						providers[cPName].ChangeNode(args)
						c.UpdatePromptN(args, 3)
					} else {
						println("the specified node doesn't support")
					}
				case strings.HasPrefix(prompt, "ns"):
					if !nsr.ChkNode(args) {
						println("error: argument is not valid")
						continue
					}
					c.UpdatePromptN(args, 3)
				}
				c.Next()
			case cmd == "local":
				nsr.Local()
				cPName = "local"
				c.SetPrompt(cPName)
				c.Next()
			case cmd == "lg":
				c.SetPrompt("lg")
				c.UpdateCompleter("connect", pNames)
				c.Next()
			case cmd == "connect":
				switch {
				case strings.HasPrefix(prompt, "lg"):
					var pName string
					if pName, err = validateProvider(args); err != nil {
						println("provider not available")
						c.Next()
						continue
					}
					cPName = pName
					if _, ok := providers[cPName]; ok {
						c.UpdatePromptN(cPName+"/"+providers[cPName].GetDefaultNode(), 2)
						go func() {
							c.UpdateCompleter("node", providers[cPName].GetNodes())
						}()
					} else {
						println("it doesn't support")
					}
				case strings.HasPrefix(prompt, "ns"):
					if !nsr.ChkCountry(args) {
						println("error: argument is not valid")
						continue
					}
					c.SetPrompt("ns/" + args)
					nsr.SetNodeList(c)

				}
				c.Next()
			case cmd == "ns":
				nsr.SetCountryList(c)
				nsr.ResetNodeList(c)
				c.SetPrompt("ns")
				c.Next()
			case cmd == "whois":
				if ripe.IsASN(args) {
					whois["asn"].Set(args)
					whois["asn"].GetData()
					whois["asn"].PrettyPrint()
				} else {
					whois["prefix"].Set(args)
					whois["prefix"].GetData()
					whois["prefix"].PrettyPrint()
				}
				c.Next()
			case cmd == "peering":
				peeringdb.Search(args)
				c.Next()
			case cmd == "mode":
				if args == "vim" {
					c.SetVim()
				} else if args == "emacs" {
					c.SetEmacs()
				} else {
					println("the request mode doesn't support")
				}
				c.Next()
			case cmd == "help":
				c.Help()
				c.Next()
			case cmd == "exit", cmd == "quit":
				c.Close(nxt)
				close(req)
				// todo
			}
		}
	}
}
<commit_msg>fixed bgp once runs on local, added default command handler<commit_after>// myLG is command line looking glass that written with Go language
// it tries from its own icmp and external looking glasses tools
package main

import (
	"errors"
	"github.com/briandowns/spinner"
	"net"
	"regexp"
	"strings"
	"time"

	"github.com/mehrdadrad/mylg/cli"
	"github.com/mehrdadrad/mylg/icmp"
	"github.com/mehrdadrad/mylg/lg"
	"github.com/mehrdadrad/mylg/ns"
	"github.com/mehrdadrad/mylg/peeringdb"
	"github.com/mehrdadrad/mylg/ripe"
)

// Provider represents looking glass
type Provider interface {
	Set(host, version string)
	GetDefaultNode() string
	GetNodes() []string
	ChangeNode(node string)
	Ping() (string, error)
	Trace() chan string
	BGP() chan string
}

// Whois represents whois providers
type Whois interface {
	Set(r string)
	GetData() bool
	PrettyPrint()
}

var (
	// register looking glass hosts
	providers = map[string]Provider{"telia": new(lg.Telia), "level3": new(lg.Level3), "cogent": new(lg.Cogent)}
	whois     = map[string]Whois{"asn": new(ripe.ASN), "prefix": new(ripe.Prefix)}
	pNames    = providerNames()
	nsr       *ns.Request
)

// providerName
func providerNames() []string {
	pNames := []string{}
	for p := range providers {
		pNames = append(pNames, p)
	}
	return pNames
}

// validateProvider
func validateProvider(p string) (string, error) {
	pNames := []string{}
	match, _ := regexp.MatchString("("+strings.Join(pNames, "|")+")", p)
	p = strings.ToLower(p)
	if match {
		return p, nil
	}
	return "", errors.New("provider not support")

}
func init() {
	// Initialize name server data
	nsr = ns.NewRequest()
	go nsr.Init()
}

func main() {
	var (
		err     error
		request string
		loop    = true
		cPName  = "local"
	)

	rep := make(chan string, 1)
	req := make(chan string, 1)
	nxt := make(chan struct{}, 1)

	c := cli.Init("local")
	go c.Run(req, nxt)

	r, _ := regexp.Compile(`(ping|trace|bgp|lg|ns|dig|whois|peering|connect|node|local|mode|help|exit|quit)\s{0,1}(.*)`)
	s := spinner.New(spinner.CharSets[26], 220*time.Millisecond)

	for loop {
		select {
		case request, loop = <-req:
			if !loop {
				break
			}
			if len(request) < 1 {
				c.Next()
				continue
			}
			subReq := r.FindStringSubmatch(request)
			if len(subReq) == 0 {
				println("syntax error")
				c.Next()
				continue
			}
			prompt := c.GetPrompt()
			cmd := strings.TrimSpace(subReq[1])
			args := strings.TrimSpace(subReq[2])
			switch {
			case cmd == "ping" && cPName == "local":
				p := icmp.NewPing()
				ra, err := net.ResolveIPAddr("ip", args)
				if err != nil {
					println("cannot resolve", args, ": Unknown host")
					c.Next()
					continue
				}
				p.IP(ra.String())
				for n := 0; n < 4; n++ {
					p.Ping(rep)
					println(<-rep)
				}
				c.Next()
			case cmd == "ping":
				s.Prefix = "please wait "
				s.Start()
				providers[cPName].Set(args, "ipv4")
				m, _ := providers[cPName].Ping()
				s.Stop()
				println(m)
				c.Next()
			case cmd == "trace":
				switch {
				case strings.HasPrefix(prompt, "local"):
					trace := icmp.Trace{}
					trace.Run(args)
				case strings.HasPrefix(prompt, "lg"):
					providers[cPName].Set(args, "ipv4")
					for l := range providers[cPName].Trace() {
						println(l)
					}
				}
				c.Next()
			case cmd == "bgp" && cPName != "local":
				providers[cPName].Set(args, "ipv4")
				for l := range providers[cPName].BGP() {
					println(l)
				}
				c.Next()
			case cmd == "dig":
				nsr.Dig(args)
				c.Next()
			case cmd == "node":
				switch {
				case strings.HasPrefix(prompt, "lg"):
					if _, ok := providers[cPName]; ok {
						providers[cPName].ChangeNode(args)
						c.UpdatePromptN(args, 3)
					} else {
						println("the specified node doesn't support")
					}
				case strings.HasPrefix(prompt, "ns"):
					if !nsr.ChkNode(args) {
						println("error: argument is not valid")
						continue
					}
					c.UpdatePromptN(args, 3)
				}
				c.Next()
			case cmd == "local":
				nsr.Local()
				cPName = "local"
				c.SetPrompt(cPName)
				c.Next()
			case cmd == "lg":
				c.SetPrompt("lg")
				c.UpdateCompleter("connect", pNames)
				c.Next()
			case cmd == "connect":
				switch {
				case strings.HasPrefix(prompt, "lg"):
					var pName string
					if pName, err = validateProvider(args); err != nil {
						println("provider not available")
						c.Next()
						continue
					}
					cPName = pName
					if _, ok := providers[cPName]; ok {
						c.UpdatePromptN(cPName+"/"+providers[cPName].GetDefaultNode(), 2)
						go func() {
							c.UpdateCompleter("node", providers[cPName].GetNodes())
						}()
					} else {
						println("it doesn't support")
					}
				case strings.HasPrefix(prompt, "ns"):
					if !nsr.ChkCountry(args) {
						println("error: argument is not valid")
						continue
					}
					c.SetPrompt("ns/" + args)
					nsr.SetNodeList(c)

				}
				c.Next()
			case cmd == "ns":
				nsr.SetCountryList(c)
				nsr.ResetNodeList(c)
				c.SetPrompt("ns")
				c.Next()
			case cmd == "whois":
				if ripe.IsASN(args) {
					whois["asn"].Set(args)
					whois["asn"].GetData()
					whois["asn"].PrettyPrint()
				} else {
					whois["prefix"].Set(args)
					whois["prefix"].GetData()
					whois["prefix"].PrettyPrint()
				}
				c.Next()
			case cmd == "peering":
				peeringdb.Search(args)
				c.Next()
			case cmd == "mode":
				if args == "vim" {
					c.SetVim()
				} else if args == "emacs" {
					c.SetEmacs()
				} else {
					println("the request mode doesn't support")
				}
				c.Next()
			case cmd == "help":
				c.Help()
				c.Next()
			case cmd == "exit", cmd == "quit":
				c.Close(nxt)
				close(req)
			// todo
			default:
				c.Next()
			}
		}
	}
}
<|endoftext|>


<commit_before>package analytics

import (
	"encoding/json"
	"strconv"
	"time"

	"github.com/boltdb/bolt"
)

// batchInsert is effectively a specialized version of SetContentMulti from the
// db package, iterating over a []apiRequest instead of []url.Values
func batchInsert(requests chan apiRequest) error {
	var reqs []apiRequest
	batchSize := len(requestChan)

	for i := 0; i < batchSize; i++ {
		reqs = append(reqs, <-requestChan)
	}

	err := store.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte("requests"))
		if err != nil {
			return err
		}

		for _, apiReq := range reqs {
			// get the next available ID and convert to string
			// also set effectedID to int of ID
			id, err := b.NextSequence()
			if err != nil {
				return err
			}
			cid := strconv.FormatUint(id, 10)

			j, err := json.Marshal(apiReq)
			if err != nil {
				return err
			}

			err = b.Put([]byte(cid), j)
			if err != nil {
				return err
			}
		}

		return nil

	})
	if err != nil {
		return err
	}

	return nil
}

// batchPrune takes a duration to evaluate apiRequest dates against. If any of
// the apiRequest timestamps are before the threshold, they are removed.
// TODO: add feature to alternatively backup old analytics to cloud
func batchPrune(threshold time.Duration) error {
	now := time.Now()
	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
	max := today.Add(threshold)

	// iterate through all request data
	err := store.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("requests"))

		err := b.ForEach(func(k, v []byte) error {
			var r apiRequest
			err := json.Unmarshal(v, &r)
			if err != nil {
				return err
			}

			// delete if timestamp is below or equal to max
			ts := time.Unix(r.Timestamp/1000, 0)
			if ts.Equal(max) || ts.Before(max) {
				err := b.Delete(k)
				if err != nil {
					return err
				}
			}

			return nil
		})
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>fix leftover db specifier name changes<commit_after>package analytics

import (
	"encoding/json"
	"strconv"
	"time"

	"github.com/boltdb/bolt"
)

// batchInsert is effectively a specialized version of SetContentMulti from the
// db package, iterating over a []apiRequest instead of []url.Values
func batchInsert(requests chan apiRequest) error {
	var reqs []apiRequest
	batchSize := len(requestChan)

	for i := 0; i < batchSize; i++ {
		reqs = append(reqs, <-requestChan)
	}

	err := store.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte("__requests"))
		if err != nil {
			return err
		}

		for _, apiReq := range reqs {
			// get the next available ID and convert to string
			// also set effectedID to int of ID
			id, err := b.NextSequence()
			if err != nil {
				return err
			}
			cid := strconv.FormatUint(id, 10)

			j, err := json.Marshal(apiReq)
			if err != nil {
				return err
			}

			err = b.Put([]byte(cid), j)
			if err != nil {
				return err
			}
		}

		return nil

	})
	if err != nil {
		return err
	}

	return nil
}

// batchPrune takes a duration to evaluate apiRequest dates against. If any of
// the apiRequest timestamps are before the threshold, they are removed.
// TODO: add feature to alternatively backup old analytics to cloud
func batchPrune(threshold time.Duration) error {
	now := time.Now()
	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
	max := today.Add(threshold)

	// iterate through all request data
	err := store.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("__requests"))

		err := b.ForEach(func(k, v []byte) error {
			var r apiRequest
			err := json.Unmarshal(v, &r)
			if err != nil {
				return err
			}

			// delete if timestamp is below or equal to max
			ts := time.Unix(r.Timestamp/1000, 0)
			if ts.Equal(max) || ts.Before(max) {
				err := b.Delete(k)
				if err != nil {
					return err
				}
			}

			return nil
		})
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package common

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/skeswa/gophr/common/semver"
)

const (
	errorRefsFetchNoSuchRepo       = "Could not find a Github repository at %s"
	errorRefsFetchGithubError      = "Github responded with an error: %v"
	errorRefsFetchGithubParseError = "Cannot read refs from Github: %v"
	errorRefsFetchNetworkFailure   = "Could not reach Github at the moment; Please try again later"
	errorRefsParseSizeFormat       = "Could not parse refs line size: %s"
	errorRefsParseIncompleteRefs   = "Incomplete refs data received from GitHub"
)

const (
	versionRefRegexIndexLabel             = 1
	versionRefRegexIndexMajorVersion      = 2
	versionRefRegexIndexMinorVersion      = 3
	versionRefRegexIndexPatchVersion      = 4
	versionRefRegexIndexPrereleaseLabel   = 5
	versionRefRegexIndexPrereleaseVersion = 6
)

const (
	refsHead                                  = "HEAD"
	refsLineCap                               = "\n\x00"
	refsSpaceChar                             = ' '
	refsHeadPrefix                            = "refs/heads/"
	refsLineFormat                            = "%04x%s"
	refsHeadMaster                            = "refs/heads/master"
	githubRootTemplate                        = "github.com/%s/%s"
	refsMasterLineFormat                      = "%s refs/heads/master\n"
	refsSymRefAssignment                      = "symref="
	refsOldRefAssignment                      = "oldref="
	refsFetchURLTemplate                      = "https://%s.git/info/refs?service=git-upload-pack"
	refsAugmentedHeadLineFormat               = "%s HEAD\n"
	refsAugmentedSymrefHeadLineFormat         = "%s HEAD\x00symref=HEAD:%s\n"
	refsAugmentedHeadLineWithCapsFormat       = "%s HEAD\x00%s\n"
	refsAugmentedSymrefHeadLineWithCapsFormat = "%s HEAD\x00symref=HEAD:%s %s\n"
)

var (
	httpClient      = &http.Client{Timeout: 10 * time.Second}
	versionRefRegex = regexp.MustCompile(`^refs\/(?:tags|heads)\/(v?([0-9]+)(?:\.([0-9]+))?(?:\.([0-9]+))?(?:\-([a-zA-Z0-9\-_]+))?(?:\.([0-9]+))?)(?:\^\{\})?`)
)

// Refs collects information about git references for one specific repository.
type Refs struct {
	Data                 []byte
	DataStr              string
	DataLen              int
	DataStrLen           int
	Candidates           semver.SemverCandidateList
	MasterRefHash        string
	IndexHeadLineEnd     int
	IndexHeadLineStart   int
	IndexMasterLineEnd   int
	IndexMasterLineStart int
}

// NewRefs creates a new Refs instance from raw refs data fetched from Github
// (or elsewhere).
func NewRefs(data []byte) (Refs, error) {
	var (
		dataStr    = string(data)
		dataLen    = len(data)
		dataStrLen = len(dataStr)

		masterRefHash                                 string
		indexHashStart, indexHashEnd                  int
		indexNameStart, indexNameEnd                  int
		indexHeadLineStart, indexHeadLineEnd          int
		indexMasterLineStart, indexMasterLineEnd      int
		versionCandidates, sanitizedVersionCandidates []semver.SemverCandidate
	)

	for i, j := 0, 0; i < dataLen; i = j {
		// Calculate the size by reading and parsing the size string
		size, err := strconv.ParseInt(dataStr[i:i+4], 16, 32)

		// If we can't read the hex, conclude that it was invalid
		if err != nil {
			return Refs{}, fmt.Errorf(errorRefsParseSizeFormat, string(data[i:i+4]))
		}

		// If we found that the size was zero, advance it by 4 since 4 is the
		// acceptable minimum
		if size == 0 {
			size = 4
		}

		// Advance the second cursor so the next token is bounded by the two cursors
		j = i + int(size)

		// If the second cursor exceeds the string boundary, then conclude that the
		// refs data is incomplete
		if j > len(dataStr) {
			return Refs{}, errors.New(errorRefsParseIncompleteRefs)
		}

		// TODO(skeswa): figure out why this line is here
		if dataStr[0] == '#' {
			continue
		}

		// Use the cursors to get the indices of the hash
		indexHashStart = i + 4
		indexHashEnd = strings.IndexByte(
			dataStr[indexHashStart:j],
			refsSpaceChar,
		)

		// Check for invalid hash end
		if indexHashEnd < 0 || indexHashEnd != 40 {
			continue
		}

		// TODO(skeswa): figure out why this line is here
		indexHashEnd += indexHashStart

		// Use the cursors to get the indices of the name
		indexNameStart = indexHashEnd + 1
		indexNameEnd = strings.IndexAny(
			dataStr[indexNameStart:j],
			refsLineCap,
		)

		// Check for invalid name end
		if indexNameEnd < 0 {
			indexNameEnd = j
		} else {
			indexNameEnd += indexNameStart
		}

		// Get the name and hash respectively as strings
		hash := dataStr[indexHashStart:indexHashEnd]
		name := dataStr[indexNameStart:indexNameEnd]

		// Process the name and hash according to whether the name is relevant
		if name == refsHead {
			indexHeadLineStart = i
			indexHeadLineEnd = j
		} else if name == refsHeadMaster {
			indexMasterLineStart = i
			indexMasterLineEnd = j
			masterRefHash = hash
		} else if captureGroups := versionRefRegex.FindStringSubmatch(name); captureGroups != nil {
			var (
				gitRefLabel       = captureGroups[versionRefRegexIndexLabel]
				majorVersion      = captureGroups[versionRefRegexIndexMajorVersion]
				minorVersion      = captureGroups[versionRefRegexIndexMinorVersion]
				patchVersion      = captureGroups[versionRefRegexIndexPatchVersion]
				prereleaseLabel   = captureGroups[versionRefRegexIndexPrereleaseLabel]
				prereleaseVersion = captureGroups[versionRefRegexIndexPrereleaseVersion]
			)

			// Annotated tag is peeled off and overrides the same version just parsed
			if strings.HasSuffix(name, "^{}") {
				name = name[:len(name)-3]
			}

			versionCandidate, err := semver.NewSemverCandidate(
				hash,
				name,
				gitRefLabel,
				majorVersion,
				minorVersion,
				patchVersion,
				prereleaseLabel,
				prereleaseVersion)
			if err == nil {
				versionCandidates = append(versionCandidates, versionCandidate)
			}
		}
	}

	if versionCandidates != nil && len(versionCandidates) > 0 {
		// First attach the sortable type to the slice of candidates.
		versionCandidatesList := semver.SemverCandidateList(versionCandidates)
		// Sort the list of candidates.
		sort.Sort(versionCandidatesList)
		// Remove duplicates by adding them to a new slice altogether.
		var lastInsertedCandidate semver.SemverCandidate
		for i, versionCandidate := range versionCandidatesList {
			if i == 0 || versionCandidate.CompareTo(lastInsertedCandidate) != 0 {
				sanitizedVersionCandidates = append(sanitizedVersionCandidates, versionCandidate)
				lastInsertedCandidate = versionCandidate
			}
		}
	}

	return Refs{
		Data:                 data,
		DataStr:              dataStr,
		DataLen:              dataLen,
		DataStrLen:           dataStrLen,
		Candidates:           sanitizedVersionCandidates,
		MasterRefHash:        masterRefHash,
		IndexHeadLineEnd:     indexHeadLineEnd,
		IndexMasterLineEnd:   indexMasterLineEnd,
		IndexHeadLineStart:   indexHeadLineStart,
		IndexMasterLineStart: indexMasterLineStart,
	}, nil
}

// FetchRefs downloads and processes refs data from Github and ultimately
// contructs a Refs instance with it.
func FetchRefs(author, repo string) (Refs, error) {
	githubRoot := fmt.Sprintf(
		githubRootTemplate,
		author,
		repo,
	)

	res, err := httpClient.Get(fmt.Sprintf(refsFetchURLTemplate, githubRoot))
	if err != nil {
		return Refs{}, errors.New(errorRefsFetchNetworkFailure)
	}

	defer res.Body.Close()

	if res.StatusCode >= 400 && res.StatusCode < 500 {
		return Refs{}, fmt.Errorf(errorRefsFetchNoSuchRepo, githubRoot)
	} else if res.StatusCode >= 500 {
		// FYI no reliable way to get test coverage here; this never happens
		return Refs{}, fmt.Errorf(errorRefsFetchGithubError, res.Status)
	}

	data, err := ioutil.ReadAll(res.Body)
	if err != nil {
		// FYI no reliable way to get test coverage here; this never happens
		return Refs{}, fmt.Errorf(errorRefsFetchGithubParseError, err)
	}

	return NewRefs(data)
}

// Reserialize changes the refs data to incorporate the selected version as
// the HEAD instead of the default HEAD.
//
// This code was written by Gustavo Niemeyer, Nathan Youngman and
// Geert-Johan Riemer.
func (refsData Refs) Reserialize(versionRefName, versionRefHash string) []byte {
	var (
		buf bytes.Buffer

		data                 = refsData.Data
		dataLen              = refsData.DataLen
		dataStr              = refsData.DataStr
		indexHeadLineEnd     = refsData.IndexHeadLineEnd
		indexHeadLineStart   = refsData.IndexHeadLineStart
		indexMasterLineEnd   = refsData.IndexMasterLineEnd
		indexMasterLineStart = refsData.IndexMasterLineStart
	)

	// Size the buffer to be a little bigger than
	buf.Grow(dataLen + 256)

	// Copy the header as-is.
	buf.Write(data[:indexHeadLineStart])

	// Extract the original capabilities.
	caps := ""
	indexNullByte := strings.Index(
		dataStr[indexHeadLineStart:indexHeadLineEnd],
		"\x00",
	)

	// IF we found a zero byte, replace the symref with an oldref
	if indexNullByte > 0 {
		caps = strings.Replace(
			dataStr[indexHeadLineStart+indexNullByte+1:indexHeadLineEnd-1],
			refsSymRefAssignment,
			refsOldRefAssignment,
			-1,
		)
	}

	// Insert the HEAD reference line with the right hash and a proper symref
	// capability.
	var line string
	if strings.HasPrefix(versionRefName, refsHeadPrefix) {
		if caps == "" {
			line = fmt.Sprintf(
				refsAugmentedSymrefHeadLineFormat,
				versionRefHash,
				versionRefName,
			)
		} else {
			line = fmt.Sprintf(
				refsAugmentedSymrefHeadLineWithCapsFormat,
				versionRefHash,
				versionRefName,
				caps,
			)
		}
	} else {
		if caps == "" {
			line = fmt.Sprintf(refsAugmentedHeadLineFormat, versionRefHash)
		} else {
			line = fmt.Sprintf(refsAugmentedHeadLineWithCapsFormat, versionRefHash, caps)
		}
	}
	fmt.Fprintf(&buf, "%04x%s", 4+len(line), line)

	// Insert the master reference line.
	line = fmt.Sprintf(refsMasterLineFormat, versionRefHash)
	fmt.Fprintf(&buf, refsLineFormat, 4+len(line), line)

	// Append the rest, dropping the original master line if necessary.
	if indexMasterLineStart > 0 {
		buf.Write(data[indexHeadLineEnd:indexMasterLineStart])
		buf.Write(data[indexMasterLineEnd:])
	} else {
		buf.Write(data[indexHeadLineEnd:])
	}

	return buf.Bytes()
}
<commit_msg>created CheckIfRefExists to determine if ref has already been versioned<commit_after>package common

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/skeswa/gophr/common/semver"
)

const (
	errorRefsFetchNoSuchRepo       = "Could not find a Github repository at %s"
	errorRefsFetchGithubError      = "Github responded with an error: %v"
	errorRefsFetchGithubParseError = "Cannot read refs from Github: %v"
	errorRefsFetchNetworkFailure   = "Could not reach Github at the moment; Please try again later"
	errorRefsParseSizeFormat       = "Could not parse refs line size: %s"
	errorRefsParseIncompleteRefs   = "Incomplete refs data received from GitHub"
)

const (
	versionRefRegexIndexLabel             = 1
	versionRefRegexIndexMajorVersion      = 2
	versionRefRegexIndexMinorVersion      = 3
	versionRefRegexIndexPatchVersion      = 4
	versionRefRegexIndexPrereleaseLabel   = 5
	versionRefRegexIndexPrereleaseVersion = 6
)

const (
	refsHead                                  = "HEAD"
	refsLineCap                               = "\n\x00"
	refsSpaceChar                             = ' '
	refsHeadPrefix                            = "refs/heads/"
	refsLineFormat                            = "%04x%s"
	refsHeadMaster                            = "refs/heads/master"
	githubRootTemplate                        = "github.com/%s/%s"
	refsMasterLineFormat                      = "%s refs/heads/master\n"
	refsSymRefAssignment                      = "symref="
	refsOldRefAssignment                      = "oldref="
	refsFetchURLTemplate                      = "https://%s.git/info/refs?service=git-upload-pack"
	refsAugmentedHeadLineFormat               = "%s HEAD\n"
	refsAugmentedSymrefHeadLineFormat         = "%s HEAD\x00symref=HEAD:%s\n"
	refsAugmentedHeadLineWithCapsFormat       = "%s HEAD\x00%s\n"
	refsAugmentedSymrefHeadLineWithCapsFormat = "%s HEAD\x00symref=HEAD:%s %s\n"
)

var (
	httpClient      = &http.Client{Timeout: 10 * time.Second}
	versionRefRegex = regexp.MustCompile(`^refs\/(?:tags|heads)\/(v?([0-9]+)(?:\.([0-9]+))?(?:\.([0-9]+))?(?:\-([a-zA-Z0-9\-_]+))?(?:\.([0-9]+))?)(?:\^\{\})?`)
)

// Refs collects information about git references for one specific repository.
type Refs struct {
	Data                 []byte
	DataStr              string
	DataLen              int
	DataStrLen           int
	Candidates           semver.SemverCandidateList
	MasterRefHash        string
	IndexHeadLineEnd     int
	IndexHeadLineStart   int
	IndexMasterLineEnd   int
	IndexMasterLineStart int
}

// NewRefs creates a new Refs instance from raw refs data fetched from Github
// (or elsewhere).
func NewRefs(data []byte) (Refs, error) {
	var (
		dataStr    = string(data)
		dataLen    = len(data)
		dataStrLen = len(dataStr)

		masterRefHash                                 string
		indexHashStart, indexHashEnd                  int
		indexNameStart, indexNameEnd                  int
		indexHeadLineStart, indexHeadLineEnd          int
		indexMasterLineStart, indexMasterLineEnd      int
		versionCandidates, sanitizedVersionCandidates []semver.SemverCandidate
	)

	for i, j := 0, 0; i < dataLen; i = j {
		// Calculate the size by reading and parsing the size string
		size, err := strconv.ParseInt(dataStr[i:i+4], 16, 32)

		// If we can't read the hex, conclude that it was invalid
		if err != nil {
			return Refs{}, fmt.Errorf(errorRefsParseSizeFormat, string(data[i:i+4]))
		}

		// If we found that the size was zero, advance it by 4 since 4 is the
		// acceptable minimum
		if size == 0 {
			size = 4
		}

		// Advance the second cursor so the next token is bounded by the two cursors
		j = i + int(size)

		// If the second cursor exceeds the string boundary, then conclude that the
		// refs data is incomplete
		if j > len(dataStr) {
			return Refs{}, errors.New(errorRefsParseIncompleteRefs)
		}

		// TODO(skeswa): figure out why this line is here
		if dataStr[0] == '#' {
			continue
		}

		// Use the cursors to get the indices of the hash
		indexHashStart = i + 4
		indexHashEnd = strings.IndexByte(
			dataStr[indexHashStart:j],
			refsSpaceChar,
		)

		// Check for invalid hash end
		if indexHashEnd < 0 || indexHashEnd != 40 {
			continue
		}

		// TODO(skeswa): figure out why this line is here
		indexHashEnd += indexHashStart

		// Use the cursors to get the indices of the name
		indexNameStart = indexHashEnd + 1
		indexNameEnd = strings.IndexAny(
			dataStr[indexNameStart:j],
			refsLineCap,
		)

		// Check for invalid name end
		if indexNameEnd < 0 {
			indexNameEnd = j
		} else {
			indexNameEnd += indexNameStart
		}

		// Get the name and hash respectively as strings
		hash := dataStr[indexHashStart:indexHashEnd]
		name := dataStr[indexNameStart:indexNameEnd]

		// Process the name and hash according to whether the name is relevant
		if name == refsHead {
			indexHeadLineStart = i
			indexHeadLineEnd = j
		} else if name == refsHeadMaster {
			indexMasterLineStart = i
			indexMasterLineEnd = j
			masterRefHash = hash
		} else if captureGroups := versionRefRegex.FindStringSubmatch(name); captureGroups != nil {
			var (
				gitRefLabel       = captureGroups[versionRefRegexIndexLabel]
				majorVersion      = captureGroups[versionRefRegexIndexMajorVersion]
				minorVersion      = captureGroups[versionRefRegexIndexMinorVersion]
				patchVersion      = captureGroups[versionRefRegexIndexPatchVersion]
				prereleaseLabel   = captureGroups[versionRefRegexIndexPrereleaseLabel]
				prereleaseVersion = captureGroups[versionRefRegexIndexPrereleaseVersion]
			)

			// Annotated tag is peeled off and overrides the same version just parsed
			if strings.HasSuffix(name, "^{}") {
				name = name[:len(name)-3]
			}

			versionCandidate, err := semver.NewSemverCandidate(
				hash,
				name,
				gitRefLabel,
				majorVersion,
				minorVersion,
				patchVersion,
				prereleaseLabel,
				prereleaseVersion)
			if err == nil {
				versionCandidates = append(versionCandidates, versionCandidate)
			}
		}
	}

	if versionCandidates != nil && len(versionCandidates) > 0 {
		// First attach the sortable type to the slice of candidates.
		versionCandidatesList := semver.SemverCandidateList(versionCandidates)
		// Sort the list of candidates.
		sort.Sort(versionCandidatesList)
		// Remove duplicates by adding them to a new slice altogether.
		var lastInsertedCandidate semver.SemverCandidate
		for i, versionCandidate := range versionCandidatesList {
			if i == 0 || versionCandidate.CompareTo(lastInsertedCandidate) != 0 {
				sanitizedVersionCandidates = append(sanitizedVersionCandidates, versionCandidate)
				lastInsertedCandidate = versionCandidate
			}
		}
	}

	return Refs{
		Data:                 data,
		DataStr:              dataStr,
		DataLen:              dataLen,
		DataStrLen:           dataStrLen,
		Candidates:           sanitizedVersionCandidates,
		MasterRefHash:        masterRefHash,
		IndexHeadLineEnd:     indexHeadLineEnd,
		IndexMasterLineEnd:   indexMasterLineEnd,
		IndexHeadLineStart:   indexHeadLineStart,
		IndexMasterLineStart: indexMasterLineStart,
	}, nil
}

// FetchRefs downloads and processes refs data from Github and ultimately
// contructs a Refs instance with it.
func FetchRefs(author, repo string) (Refs, error) {
	githubRoot := fmt.Sprintf(
		githubRootTemplate,
		author,
		repo,
	)

	res, err := httpClient.Get(fmt.Sprintf(refsFetchURLTemplate, githubRoot))
	if err != nil {
		return Refs{}, errors.New(errorRefsFetchNetworkFailure)
	}

	defer res.Body.Close()

	if res.StatusCode >= 400 && res.StatusCode < 500 {
		return Refs{}, fmt.Errorf(errorRefsFetchNoSuchRepo, githubRoot)
	} else if res.StatusCode >= 500 {
		// FYI no reliable way to get test coverage here; this never happens
		return Refs{}, fmt.Errorf(errorRefsFetchGithubError, res.Status)
	}

	data, err := ioutil.ReadAll(res.Body)
	if err != nil {
		// FYI no reliable way to get test coverage here; this never happens
		return Refs{}, fmt.Errorf(errorRefsFetchGithubParseError, err)
	}

	return NewRefs(data)
}

// Reserialize changes the refs data to incorporate the selected version as
// the HEAD instead of the default HEAD.
//
// This code was written by Gustavo Niemeyer, Nathan Youngman and
// Geert-Johan Riemer.
func (refsData Refs) Reserialize(versionRefName, versionRefHash string) []byte {
	var (
		buf bytes.Buffer

		data                 = refsData.Data
		dataLen              = refsData.DataLen
		dataStr              = refsData.DataStr
		indexHeadLineEnd     = refsData.IndexHeadLineEnd
		indexHeadLineStart   = refsData.IndexHeadLineStart
		indexMasterLineEnd   = refsData.IndexMasterLineEnd
		indexMasterLineStart = refsData.IndexMasterLineStart
	)

	// Size the buffer to be a little bigger than
	buf.Grow(dataLen + 256)

	// Copy the header as-is.
	buf.Write(data[:indexHeadLineStart])

	// Extract the original capabilities.
	caps := ""
	indexNullByte := strings.Index(
		dataStr[indexHeadLineStart:indexHeadLineEnd],
		"\x00",
	)

	// IF we found a zero byte, replace the symref with an oldref
	if indexNullByte > 0 {
		caps = strings.Replace(
			dataStr[indexHeadLineStart+indexNullByte+1:indexHeadLineEnd-1],
			refsSymRefAssignment,
			refsOldRefAssignment,
			-1,
		)
	}

	// Insert the HEAD reference line with the right hash and a proper symref
	// capability.
	var line string
	if strings.HasPrefix(versionRefName, refsHeadPrefix) {
		if caps == "" {
			line = fmt.Sprintf(
				refsAugmentedSymrefHeadLineFormat,
				versionRefHash,
				versionRefName,
			)
		} else {
			line = fmt.Sprintf(
				refsAugmentedSymrefHeadLineWithCapsFormat,
				versionRefHash,
				versionRefName,
				caps,
			)
		}
	} else {
		if caps == "" {
			line = fmt.Sprintf(refsAugmentedHeadLineFormat, versionRefHash)
		} else {
			line = fmt.Sprintf(refsAugmentedHeadLineWithCapsFormat, versionRefHash, caps)
		}
	}
	fmt.Fprintf(&buf, "%04x%s", 4+len(line), line)

	// Insert the master reference line.
	line = fmt.Sprintf(refsMasterLineFormat, versionRefHash)
	fmt.Fprintf(&buf, refsLineFormat, 4+len(line), line)

	// Append the rest, dropping the original master line if necessary.
	if indexMasterLineStart > 0 {
		buf.Write(data[indexHeadLineEnd:indexMasterLineStart])
		buf.Write(data[indexMasterLineEnd:])
	} else {
		buf.Write(data[indexHeadLineEnd:])
	}

	return buf.Bytes()
}

// CheckIfRefExists downloads and processes refs data from Github and checks
// whether a given ref exists in the remote refs list.
func CheckIfRefExists(author, repo string, ref string) (bool, error) {
	githubRoot := fmt.Sprintf(
		githubRootTemplate,
		author,
		repo,
	)

	res, err := httpClient.Get(fmt.Sprintf(refsFetchURLTemplate, githubRoot))
	if err != nil {
		return false, errors.New(errorRefsFetchNetworkFailure)
	}

	defer res.Body.Close()

	if res.StatusCode >= 400 && res.StatusCode < 500 {
		return false, fmt.Errorf(errorRefsFetchNoSuchRepo, githubRoot)
	} else if res.StatusCode >= 500 {
		// FYI no reliable way to get test coverage here; this never happens
		return false, fmt.Errorf(errorRefsFetchGithubError, res.Status)
	}

	data, err := ioutil.ReadAll(res.Body)
	if err != nil {
		// FYI no reliable way to get test coverage here; this never happens
		return false, fmt.Errorf(errorRefsFetchGithubParseError, err)
	}

	refsString := string(data)
	refExists := strings.Contains(refsString, ref)

	return refExists, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"http"
	"io"
	"json"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"camli/blobref"
	"camli/blobserver"
	"camli/httputil"
	"camli/jsonconfig"
	"camli/schema"
)

var _ = log.Printf

var staticFilePattern = regexp.MustCompile(`^([a-zA-Z0-9\-\_]+\.(html|js|css|png|jpg|gif))$`)
var identPattern = regexp.MustCompile(`^[a-zA-Z\_]+$`)

// Download URL suffix:
//   $1: blobref (checked in download handler)
//   $2: optional "/filename" to be sent as recommended download name,
//       if sane looking
var downloadPattern = regexp.MustCompile(`^download/([^/]+)(/.*)?$`)

// UIHandler handles serving the UI and discovery JSON.
type UIHandler struct {
	// URL prefixes (path or full URL) to the primary blob and
	// search root.  Only used by the UI and thus necessary if UI
	// is true.
	BlobRoot     string
	SearchRoot   string
	JSONSignRoot string

	FilesDir string

	Storage blobserver.Storage // of BlobRoot
	Cache   blobserver.Storage // or nil
}

func defaultFilesDir() string {
	dir, _ := filepath.Split(os.Args[0])
	return filepath.Join(dir, "ui")
}

func init() {
	blobserver.RegisterHandlerConstructor("ui", newUiFromConfig)
}

func newUiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err os.Error) {
	ui := &UIHandler{}
	ui.BlobRoot = conf.OptionalString("blobRoot", "")
	ui.SearchRoot = conf.OptionalString("searchRoot", "")
	ui.JSONSignRoot = conf.OptionalString("jsonSignRoot", "")
	cachePrefix := conf.OptionalString("cache", "")
	ui.FilesDir = conf.OptionalString("staticFiles", defaultFilesDir())
	if err = conf.Validate(); err != nil {
		return
	}

	checkType := func(key string, htype string) {
		v := conf.OptionalString(key, "")
		if v == "" {
			return
		}
		ct := ld.GetHandlerType(v)
		if ct == "" {
			err = fmt.Errorf("UI handler's %q references non-existant %q", key, v)
		} else if ct != htype {
			err = fmt.Errorf("UI handler's %q references %q of type %q; expected type %q", key, v,
				ct, htype)
		}
	}
	checkType("searchRoot", "search")
	checkType("jsonSignRoot", "jsonsign")
	if err != nil {
		return
	}

	if ui.BlobRoot != "" {
		bs, err := ld.GetStorage(ui.BlobRoot)
		if err != nil {
			return nil, fmt.Errorf("UI handler's blobRoot of %q error: %v", ui.BlobRoot, err)
		}
		ui.Storage = bs
	}

	if cachePrefix != "" {
		bs, err := ld.GetStorage(cachePrefix)
		if err != nil {
			return nil, fmt.Errorf("UI handler's cache of %q error: %v", ui.BlobRoot, err)
		}
		ui.Cache = bs
	}

	fi, sterr := os.Stat(ui.FilesDir)
	if sterr != nil || !fi.IsDirectory() {
		err = fmt.Errorf("UI handler's \"staticFiles\" of %q is invalid", ui.FilesDir)
		return
	}
	return ui, nil
}

func camliMode(req *http.Request) string {
	// TODO-GO: this is too hard to get at the GET Query args on a
	// POST request.
	m, err := http.ParseQuery(req.URL.RawQuery)
	if err != nil {
		return ""
	}
	if mode, ok := m["camli.mode"]; ok && len(mode) > 0 {
		return mode[0]
	}
	return ""
}

func wantsDiscovery(req *http.Request) bool {
	return req.Method == "GET" &&
		(req.Header.Get("Accept") == "text/x-camli-configuration" ||
			camliMode(req) == "config")
}

func wantsUploadHelper(req *http.Request) bool {
	return req.Method == "POST" && camliMode(req) == "uploadhelper"
}

func wantsPermanode(req *http.Request) bool {
	return req.Method == "GET" && blobref.Parse(req.FormValue("p")) != nil
}

func wantsBlobInfo(req *http.Request) bool {
	return req.Method == "GET" && blobref.Parse(req.FormValue("b")) != nil
}

func (ui *UIHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	base := req.Header.Get("X-PrefixHandler-PathBase")
	suffix := req.Header.Get("X-PrefixHandler-PathSuffix")

	rw.Header().Set("Vary", "Accept")
	switch {
	case wantsDiscovery(req):
		ui.serveDiscovery(rw, req)
	case wantsUploadHelper(req):
		ui.serveUploadHelper(rw, req)
	case strings.HasPrefix(suffix, "download/"):
		ui.serveDownload(rw, req)
	default:
		file := ""
		if m := staticFilePattern.FindStringSubmatch(suffix); m != nil {
			file = m[1]
		} else {
			switch {
			case wantsPermanode(req):
				file = "permanode.html"
			case wantsBlobInfo(req):
				file = "blobinfo.html"
			case req.URL.Path == base:
				file = "index.html"
			default:
				http.Error(rw, "Illegal URL.", 404)
				return
			}
		}
		http.ServeFile(rw, req, filepath.Join(ui.FilesDir, file))
	}
}

func (ui *UIHandler) serveDiscovery(rw http.ResponseWriter, req *http.Request) {
	rw.Header().Set("Content-Type", "text/javascript")
	inCb := false
	if cb := req.FormValue("cb"); identPattern.MatchString(cb) {
		fmt.Fprintf(rw, "%s(", cb)
		inCb = true
	}
	bytes, _ := json.Marshal(map[string]interface{}{
		"blobRoot":     ui.BlobRoot,
		"searchRoot":   ui.SearchRoot,
		"jsonSignRoot": ui.JSONSignRoot,
		"uploadHelper": "?camli.mode=uploadhelper", // hack; remove with better javascript
	})
	rw.Write(bytes)
	if inCb {
		rw.Write([]byte{')'})
	}
}

func (ui *UIHandler) storageSeekFetcher() (blobref.SeekFetcher, os.Error) {
	return blobref.SeekerFromStreamingFetcher(ui.Storage)
}

func (ui *UIHandler) serveDownload(rw http.ResponseWriter, req *http.Request) {
	if req.Method != "GET" && req.Method != "HEAD" {
		http.Error(rw, "Invalid download method", 400)
		return
	}
	if ui.Storage == nil {
		http.Error(rw, "No BlobRoot configured", 500)
		return
	}

	fetchSeeker, err := ui.storageSeekFetcher()
	if err != nil {
		http.Error(rw, err.String(), 500)
		return
	}

	suffix := req.Header.Get("X-PrefixHandler-PathSuffix")

	m := downloadPattern.FindStringSubmatch(suffix)
	if m == nil {
		httputil.ErrorRouting(rw, req)
		return
	}

	fbr := blobref.Parse(m[1])
	if fbr == nil {
		http.Error(rw, "Invalid blobref", 400)
		return
	}

	filename := m[2]
	if len(filename) > 0 {
		filename = filename[1:] // remove leading slash
	}

	fr, err := schema.NewFileReader(fetchSeeker, fbr)
	if err != nil {
		http.Error(rw, "Can't serve file: "+err.String(), 500)
		return
	}
	defer fr.Close()

	// TODO: fr.FileSchema() and guess a mime type?  For now:
	schema := fr.FileSchema()
	rw.Header().Set("Content-Type", "application/octet-stream")
	rw.Header().Set("Content-Length", fmt.Sprintf("%d", schema.Size))

	if req.Method == "HEAD" {
		vbr := blobref.Parse(req.FormValue("verifycontents"))
		if vbr == nil {
			return
		}
		hash := vbr.Hash()
		if hash == nil {
			return
		}
		io.Copy(hash, fr) // ignore errors, caught later
		if vbr.HashMatches(hash) {
			rw.Header().Set("X-Camli-Contents", vbr.String())
		}
		return
	}

	n, err := io.Copy(rw, fr)
	log.Printf("For %q request of %s: copied %d, %v", req.Method, req.URL.RawPath, n, err)
	if err != nil {
		log.Printf("error serving download of file schema %s: %v", fbr, err)
		return
	}
	if n != int64(schema.Size) {
		log.Printf("error serving download of file schema %s: sent %d, expected size of %d",
			fbr, n, schema.Size)
		return
	}
}
<commit_msg>advertise download helper in discovery<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"http"
	"io"
	"json"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"camli/blobref"
	"camli/blobserver"
	"camli/httputil"
	"camli/jsonconfig"
	"camli/schema"
)

var _ = log.Printf

var staticFilePattern = regexp.MustCompile(`^([a-zA-Z0-9\-\_]+\.(html|js|css|png|jpg|gif))$`)
var identPattern = regexp.MustCompile(`^[a-zA-Z\_]+$`)

// Download URL suffix:
//   $1: blobref (checked in download handler)
//   $2: optional "/filename" to be sent as recommended download name,
//       if sane looking
var downloadPattern = regexp.MustCompile(`^download/([^/]+)(/.*)?$`)

// UIHandler handles serving the UI and discovery JSON.
type UIHandler struct {
	// URL prefixes (path or full URL) to the primary blob and
	// search root.  Only used by the UI and thus necessary if UI
	// is true.
	BlobRoot     string
	SearchRoot   string
	JSONSignRoot string

	FilesDir string

	Storage blobserver.Storage // of BlobRoot
	Cache   blobserver.Storage // or nil
}

func defaultFilesDir() string {
	dir, _ := filepath.Split(os.Args[0])
	return filepath.Join(dir, "ui")
}

func init() {
	blobserver.RegisterHandlerConstructor("ui", newUiFromConfig)
}

func newUiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err os.Error) {
	ui := &UIHandler{}
	ui.BlobRoot = conf.OptionalString("blobRoot", "")
	ui.SearchRoot = conf.OptionalString("searchRoot", "")
	ui.JSONSignRoot = conf.OptionalString("jsonSignRoot", "")
	cachePrefix := conf.OptionalString("cache", "")
	ui.FilesDir = conf.OptionalString("staticFiles", defaultFilesDir())
	if err = conf.Validate(); err != nil {
		return
	}

	checkType := func(key string, htype string) {
		v := conf.OptionalString(key, "")
		if v == "" {
			return
		}
		ct := ld.GetHandlerType(v)
		if ct == "" {
			err = fmt.Errorf("UI handler's %q references non-existant %q", key, v)
		} else if ct != htype {
			err = fmt.Errorf("UI handler's %q references %q of type %q; expected type %q", key, v,
				ct, htype)
		}
	}
	checkType("searchRoot", "search")
	checkType("jsonSignRoot", "jsonsign")
	if err != nil {
		return
	}

	if ui.BlobRoot != "" {
		bs, err := ld.GetStorage(ui.BlobRoot)
		if err != nil {
			return nil, fmt.Errorf("UI handler's blobRoot of %q error: %v", ui.BlobRoot, err)
		}
		ui.Storage = bs
	}

	if cachePrefix != "" {
		bs, err := ld.GetStorage(cachePrefix)
		if err != nil {
			return nil, fmt.Errorf("UI handler's cache of %q error: %v", ui.BlobRoot, err)
		}
		ui.Cache = bs
	}

	fi, sterr := os.Stat(ui.FilesDir)
	if sterr != nil || !fi.IsDirectory() {
		err = fmt.Errorf("UI handler's \"staticFiles\" of %q is invalid", ui.FilesDir)
		return
	}
	return ui, nil
}

func camliMode(req *http.Request) string {
	// TODO-GO: this is too hard to get at the GET Query args on a
	// POST request.
	m, err := http.ParseQuery(req.URL.RawQuery)
	if err != nil {
		return ""
	}
	if mode, ok := m["camli.mode"]; ok && len(mode) > 0 {
		return mode[0]
	}
	return ""
}

func wantsDiscovery(req *http.Request) bool {
	return req.Method == "GET" &&
		(req.Header.Get("Accept") == "text/x-camli-configuration" ||
			camliMode(req) == "config")
}

func wantsUploadHelper(req *http.Request) bool {
	return req.Method == "POST" && camliMode(req) == "uploadhelper"
}

func wantsPermanode(req *http.Request) bool {
	return req.Method == "GET" && blobref.Parse(req.FormValue("p")) != nil
}

func wantsBlobInfo(req *http.Request) bool {
	return req.Method == "GET" && blobref.Parse(req.FormValue("b")) != nil
}

func (ui *UIHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	base := req.Header.Get("X-PrefixHandler-PathBase")
	suffix := req.Header.Get("X-PrefixHandler-PathSuffix")

	rw.Header().Set("Vary", "Accept")
	switch {
	case wantsDiscovery(req):
		ui.serveDiscovery(rw, req)
	case wantsUploadHelper(req):
		ui.serveUploadHelper(rw, req)
	case strings.HasPrefix(suffix, "download/"):
		ui.serveDownload(rw, req)
	default:
		file := ""
		if m := staticFilePattern.FindStringSubmatch(suffix); m != nil {
			file = m[1]
		} else {
			switch {
			case wantsPermanode(req):
				file = "permanode.html"
			case wantsBlobInfo(req):
				file = "blobinfo.html"
			case req.URL.Path == base:
				file = "index.html"
			default:
				http.Error(rw, "Illegal URL.", 404)
				return
			}
		}
		http.ServeFile(rw, req, filepath.Join(ui.FilesDir, file))
	}
}

func (ui *UIHandler) serveDiscovery(rw http.ResponseWriter, req *http.Request) {
	rw.Header().Set("Content-Type", "text/javascript")
	inCb := false
	if cb := req.FormValue("cb"); identPattern.MatchString(cb) {
		fmt.Fprintf(rw, "%s(", cb)
		inCb = true
	}
	bytes, _ := json.Marshal(map[string]interface{}{
		"blobRoot":     ui.BlobRoot,
		"searchRoot":   ui.SearchRoot,
		"jsonSignRoot": ui.JSONSignRoot,
		"uploadHelper": "?camli.mode=uploadhelper", // hack; remove with better javascript
		"downloadHelper": "./download/",
	})
	rw.Write(bytes)
	if inCb {
		rw.Write([]byte{')'})
	}
}

func (ui *UIHandler) storageSeekFetcher() (blobref.SeekFetcher, os.Error) {
	return blobref.SeekerFromStreamingFetcher(ui.Storage)
}

func (ui *UIHandler) serveDownload(rw http.ResponseWriter, req *http.Request) {
	if req.Method != "GET" && req.Method != "HEAD" {
		http.Error(rw, "Invalid download method", 400)
		return
	}
	if ui.Storage == nil {
		http.Error(rw, "No BlobRoot configured", 500)
		return
	}

	fetchSeeker, err := ui.storageSeekFetcher()
	if err != nil {
		http.Error(rw, err.String(), 500)
		return
	}

	suffix := req.Header.Get("X-PrefixHandler-PathSuffix")

	m := downloadPattern.FindStringSubmatch(suffix)
	if m == nil {
		httputil.ErrorRouting(rw, req)
		return
	}

	fbr := blobref.Parse(m[1])
	if fbr == nil {
		http.Error(rw, "Invalid blobref", 400)
		return
	}

	filename := m[2]
	if len(filename) > 0 {
		filename = filename[1:] // remove leading slash
	}

	fr, err := schema.NewFileReader(fetchSeeker, fbr)
	if err != nil {
		http.Error(rw, "Can't serve file: "+err.String(), 500)
		return
	}
	defer fr.Close()

	// TODO: fr.FileSchema() and guess a mime type?  For now:
	schema := fr.FileSchema()
	rw.Header().Set("Content-Type", "application/octet-stream")
	rw.Header().Set("Content-Length", fmt.Sprintf("%d", schema.Size))

	if req.Method == "HEAD" {
		vbr := blobref.Parse(req.FormValue("verifycontents"))
		if vbr == nil {
			return
		}
		hash := vbr.Hash()
		if hash == nil {
			return
		}
		io.Copy(hash, fr) // ignore errors, caught later
		if vbr.HashMatches(hash) {
			rw.Header().Set("X-Camli-Contents", vbr.String())
		}
		return
	}

	n, err := io.Copy(rw, fr)
	log.Printf("For %q request of %s: copied %d, %v", req.Method, req.URL.RawPath, n, err)
	if err != nil {
		log.Printf("error serving download of file schema %s: %v", fbr, err)
		return
	}
	if n != int64(schema.Size) {
		log.Printf("error serving download of file schema %s: sent %d, expected size of %d",
			fbr, n, schema.Size)
		return
	}
}
<|endoftext|>


<commit_before>package atlas

import (
	"testing"
)

func TestSetAuth(t *testing.T) {

	SetAuth("foo")

	if APIKey != "foo" {
		t.Errorf("APIKey is not set")
	}
}

func TestWantAuth(t *testing.T) {

	var (
		key string
		ok bool
	)

	SetAuth("")
	if key, ok = HasAPIKey(); ok != false {
		t.Errorf("WantAuth() should be true")
	}

	SetAuth("foo")
	if key, ok = HasAPIKey(); ok != true {
		t.Errorf("ok should be true")
	}

	if key != "foo" {
		t.Errorf("APIKey should be %s but is %s", "foo", key)
	}
}

func TestGetPageNum(t *testing.T) {
	url := "https://foo.example.com/"

	n := getPageNum(url)
	if n != "" {
		t.Errorf("n=%s should be ''", n)
	}
	url = "https://foo.example.com/?page=42"
	n = getPageNum(url)
	if n != "42" {
		t.Errorf("n=%s should be 42", n)
	}
	url = "https://foo.example.com/?country=fr&page=43"
	n = getPageNum(url)
	if n != "43" {
		t.Errorf("n=%s should be 43", n)
	}
	url = "https://foo.example.com/?country=fr&page=666&bar=1"
	n = getPageNum(url)
	if n != "666" {
		t.Errorf("n=%s should be 666", n)
	}

	url = ""
	n = getPageNum(url)
	if n != "" {
		t.Errorf("n=%d should be ''", n)
	}
}
<commit_msg>Fix format string.<commit_after>package atlas

import (
	"testing"
)

func TestSetAuth(t *testing.T) {

	SetAuth("foo")

	if APIKey != "foo" {
		t.Errorf("APIKey is not set")
	}
}

func TestWantAuth(t *testing.T) {

	var (
		key string
		ok bool
	)

	SetAuth("")
	if key, ok = HasAPIKey(); ok != false {
		t.Errorf("WantAuth() should be true")
	}

	SetAuth("foo")
	if key, ok = HasAPIKey(); ok != true {
		t.Errorf("ok should be true")
	}

	if key != "foo" {
		t.Errorf("APIKey should be %s but is %s", "foo", key)
	}
}

func TestGetPageNum(t *testing.T) {
	url := "https://foo.example.com/"

	n := getPageNum(url)
	if n != "" {
		t.Errorf("n=%s should be ''", n)
	}
	url = "https://foo.example.com/?page=42"
	n = getPageNum(url)
	if n != "42" {
		t.Errorf("n=%s should be 42", n)
	}
	url = "https://foo.example.com/?country=fr&page=43"
	n = getPageNum(url)
	if n != "43" {
		t.Errorf("n=%s should be 43", n)
	}
	url = "https://foo.example.com/?country=fr&page=666&bar=1"
	n = getPageNum(url)
	if n != "666" {
		t.Errorf("n=%s should be 666", n)
	}

	url = ""
	n = getPageNum(url)
	if n != "" {
		t.Errorf("n=%s should be ''", n)
	}
}
<|endoftext|>


<commit_before>/*
	This file is part of go-ethereum

	go-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU Lesser General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	go-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU Lesser General Public License
	along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
/**
 * @authors
 * 	Gustav Simonsson <gustav.simonsson@gmail.com>
 * @date 2015
 *
 */

package crypto

import (
	"bytes"
	"crypto/ecdsa"
	"crypto/elliptic"
	"encoding/json"
	"io"

	"code.google.com/p/go-uuid/uuid"
)

type Key struct {
	Id uuid.UUID // Version 4 "random" for unique id not derived from key data
	// to simplify lookups we also store the address
	Address []byte
	// we only store privkey as pubkey/address can be derived from it
	// privkey in this struct is always in plaintext
	PrivateKey *ecdsa.PrivateKey
}

type plainKeyJSON struct {
	Id         []byte
	Address    []byte
	PrivateKey []byte
}

type cipherJSON struct {
	Salt       []byte
	IV         []byte
	CipherText []byte
}

type encryptedKeyJSON struct {
	Id      []byte
	Address []byte
	Crypto  cipherJSON
}

func (k *Key) MarshalJSON() (j []byte, err error) {
	jStruct := plainKeyJSON{
		k.Id,
		k.Address,
		FromECDSA(k.PrivateKey),
	}
	j, err = json.Marshal(jStruct)
	return j, err
}

func (k *Key) UnmarshalJSON(j []byte) (err error) {
	keyJSON := new(plainKeyJSON)
	err = json.Unmarshal(j, &keyJSON)
	if err != nil {
		return err
	}

	u := new(uuid.UUID)
	*u = keyJSON.Id
	k.Id = *u
	k.Address = keyJSON.Address
	k.PrivateKey = ToECDSA(keyJSON.PrivateKey)

	return err
}

func NewKey(rand io.Reader) *Key {
	randBytes := make([]byte, 32)
	_, err := rand.Read(randBytes)
	if err != nil {
		panic("key generation: could not read from random source: " + err.Error())
	}
	reader := bytes.NewReader(randBytes)
	_, x, y, err := elliptic.GenerateKey(S256(), reader)
	if err != nil {
		panic("key generation: elliptic.GenerateKey failed: " + err.Error())
	}
	privateKeyMarshalled := elliptic.Marshal(S256(), x, y)
	privateKeyECDSA := ToECDSA(privateKeyMarshalled)

	id := uuid.NewRandom()
	key := &Key{
		Id:         id,
		Address:    PubkeyToAddress(privateKeyECDSA.PublicKey),
		PrivateKey: privateKeyECDSA,
	}
	return key
}
<commit_msg>Use ECDSA instead of elliptic<commit_after>/*
	This file is part of go-ethereum

	go-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU Lesser General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	go-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU Lesser General Public License
	along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
/**
 * @authors
 * 	Gustav Simonsson <gustav.simonsson@gmail.com>
 * @date 2015
 *
 */

package crypto

import (
	"bytes"
	"crypto/ecdsa"
	"encoding/json"
	"io"

	"code.google.com/p/go-uuid/uuid"
)

type Key struct {
	Id uuid.UUID // Version 4 "random" for unique id not derived from key data
	// to simplify lookups we also store the address
	Address []byte
	// we only store privkey as pubkey/address can be derived from it
	// privkey in this struct is always in plaintext
	PrivateKey *ecdsa.PrivateKey
}

type plainKeyJSON struct {
	Id         []byte
	Address    []byte
	PrivateKey []byte
}

type cipherJSON struct {
	Salt       []byte
	IV         []byte
	CipherText []byte
}

type encryptedKeyJSON struct {
	Id      []byte
	Address []byte
	Crypto  cipherJSON
}

func (k *Key) MarshalJSON() (j []byte, err error) {
	jStruct := plainKeyJSON{
		k.Id,
		k.Address,
		FromECDSA(k.PrivateKey),
	}
	j, err = json.Marshal(jStruct)
	return j, err
}

func (k *Key) UnmarshalJSON(j []byte) (err error) {
	keyJSON := new(plainKeyJSON)
	err = json.Unmarshal(j, &keyJSON)
	if err != nil {
		return err
	}

	u := new(uuid.UUID)
	*u = keyJSON.Id
	k.Id = *u
	k.Address = keyJSON.Address
	k.PrivateKey = ToECDSA(keyJSON.PrivateKey)

	return err
}

func NewKey(rand io.Reader) *Key {
	randBytes := make([]byte, 64)
	_, err := rand.Read(randBytes)
	if err != nil {
		panic("key generation: could not read from random source: " + err.Error())
	}
	reader := bytes.NewReader(randBytes)
	privateKeyECDSA, err := ecdsa.GenerateKey(S256(), reader)
	if err != nil {
		panic("key generation: ecdsa.GenerateKey failed: " + err.Error())
	}

	id := uuid.NewRandom()
	key := &Key{
		Id:         id,
		Address:    PubkeyToAddress(privateKeyECDSA.PublicKey),
		PrivateKey: privateKeyECDSA,
	}
	return key
}
<|endoftext|>


<commit_before>package services

import (
	"jbrodriguez/mediagui/server/src/dto"
	"jbrodriguez/mediagui/server/src/lib"
	"jbrodriguez/mediagui/server/src/model"
	"jbrodriguez/mediagui/server/src/proto"

	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/jbrodriguez/actor"
	"github.com/jbrodriguez/mlog"
	"github.com/jbrodriguez/pubsub"
	"github.com/micro/go-micro/client"

	"golang.org/x/net/context"
)

// Core -
type Core struct {
	bus      *pubsub.PubSub
	settings *lib.Settings

	actor *actor.Actor
	re    *regexp.Regexp
	maps  map[string]bool

	wg sync.WaitGroup
}

// NewCore -
func NewCore(bus *pubsub.PubSub, settings *lib.Settings) *Core {
	core := &Core{
		bus:      bus,
		settings: settings,
		actor:    actor.NewActor(bus),
	}
	return core
}

// Start -
func (c *Core) Start() {
	mlog.Info("Starting service Core ...")

	c.actor.Register("/get/config", c.getConfig)
	c.actor.Register("/post/import", c.importMovies)
	c.actor.Register("/post/add", c.addMovie)
	c.actor.Register("/post/prune", c.pruneMovies)
	c.actor.Register("/put/config/folder", c.addMediaFolder)
	c.actor.Register("/put/movies/fix", c.fixMovie)

	c.actor.Register("/event/movie/found", c.doMovieFound)
	c.actor.Register("/event/movie/tmdbnotfound", c.doMovieTmdbNotFound)
	c.actor.Register("/event/movie/scraped", c.doMovieScraped)
	c.actor.Register("/event/movie/rescraped", c.doMovieReScraped)
	// c.actor.Register("/event/movie/updated", c.doMovieUpdated)
	// c.actor.Register("/event/movie/cached/forced", c.doMovieCachedForced)

	c.actor.Register("/event/workunit/done", c.doWorkUnitDone)

	c.re = regexp.MustCompile(`(?i)/Volumes/(.*?)/.*`)
	c.maps = make(map[string]bool)

	// maps["/Volumes/wopr-films"] = true // for example
	for _, folder := range c.settings.MediaFolders {
		c.maps[folder] = false
		if _, err := os.Stat(folder); err == nil {
			c.maps[folder] = true
		}
	}

	go c.actor.React()
}

// Stop -
func (c *Core) Stop() {
	mlog.Info("Stopped service Core ...")
}

func (c *Core) getConfig(msg *pubsub.Message) {
	msg.Reply <- &c.settings.Config
	mlog.Info("Sent config")
}

func (c *Core) importMovies(msg *pubsub.Message) {
	t0 := time.Now()
	// mlog.Info("Begin movie scanning ...")
	lib.Notify(c.bus, "import:begin", "Import process started")

	c.wg.Add(1)

	c.bus.Pub(nil, "/command/movie/scan")
	//	msg.Reply <- &c.settings.Config
	// mlog.Info("Import finished")

	go func() {
		c.wg.Wait()
		// 𝛥t := float64(time.Since(t0)) / 1e9
		lib.Notify(c.bus, "import:end", fmt.Sprintf("Import process finished (%s elapsed)", time.Since(t0).String()))
	}()

}

func (c *Core) addMovie(msg *pubsub.Message) {
	// I'm expecting movie to have the following fields filled
	// Title
	movie := msg.Payload.(*model.Movie)

	// Since it's coming in through this endpoint, we know it's a stub
	movie.File_Title = movie.Title
	movie.Resolution = "n/a"
	movie.FileType = "n/a"
	movie.Location = "n/a"
	movie.Stub = 1

	// 3 operations, rescrape, update and cache
	c.wg.Add(1)

	// rescrape
	scrape := &pubsub.Message{Payload: movie, Reply: make(chan interface{}, 3)}
	c.bus.Pub(scrape, "/command/movie/scrape")

	go func() {
		c.wg.Wait()
		msg.Reply <- movie
	}()
}

func (c *Core) doMovieFound(msg *pubsub.Message) {
	movie := msg.Payload.(*model.Movie)

	check := &pubsub.Message{Payload: movie, Reply: make(chan interface{}, 3)}
	c.bus.Pub(check, "/command/movie/exists")

	reply := <-check.Reply
	exists := reply.(bool)

	if exists {
		mlog.Info("SKIPPED: exists [%s] (%s)", movie.Title, movie.Location)
	} else {
		lib.Notify(c.bus, "import:progress", fmt.Sprintf("NEW: [%s] (%s)", movie.Title, movie.Location))

		c.wg.Add(1)
		c.bus.Pub(msg, "/command/movie/scrape")
	}
}

func (c *Core) fixMovie(msg *pubsub.Message) {
	movie := msg.Payload.(*model.Movie)

	// 3 operations, rescrape, update and cache
	c.wg.Add(1)

	// rescrape
	scrape := &pubsub.Message{Payload: movie, Reply: make(chan interface{}, 3)}
	c.bus.Pub(scrape, "/command/movie/rescrape")

	go func() {
		c.wg.Wait()
		msg.Reply <- movie
	}()
	// go c.waitFixMovie(msg.Reply, movie)
}

func (c *Core) doMovieTmdbNotFound(msg *pubsub.Message) {
	dto := msg.Payload.(*dto.Scrape)

	store := &pubsub.Message{Payload: dto.Movie, Reply: make(chan interface{}, 3)}
	c.bus.Pub(store, "/command/movie/partialstore")
}

func (c *Core) doMovieScraped(msg *pubsub.Message) {
	dto := msg.Payload.(*dto.Scrape)

	mlog.Info("ScrapeDTO: %+v", dto)

	// I treat the following two commands as one, for the sake of the wg
	// now there are two oustanding locks, which will be decreased by each
	// responding service
	c.wg.Add(1)

	store := &pubsub.Message{Payload: dto.Movie, Reply: make(chan interface{}, 3)}
	c.bus.Pub(store, "/command/movie/store")

	cache := &pubsub.Message{Payload: dto, Reply: make(chan interface{}, 3)}
	c.bus.Pub(cache, "/command/movie/cache")

	// mlog.Info("ScrapeDTO: %+v", dto)
}

func (c *Core) doMovieReScraped(msg *pubsub.Message) {
	dto := msg.Payload.(*dto.Scrape)

	mlog.Info("ScrapeDTO: %+v", dto)

	// I treat the following two commands as one, for the sake of the wg
	// now there are two oustanding locks, which will be decreased by each
	// responding service
	c.wg.Add(1)

	store := &pubsub.Message{Payload: dto.Movie, Reply: make(chan interface{}, 3)}
	c.bus.Pub(store, "/command/movie/update")

	cache := &pubsub.Message{Payload: dto, Reply: make(chan interface{}, 3)}
	c.bus.Pub(cache, "/command/movie/cache")

	// mlog.Info("ScrapeDTO: %+v", dto)
}

func (c *Core) pruneMovies(msg *pubsub.Message) {
	t0 := time.Now()

	lib.Notify(c.bus, "prune:begin", "Started Prune Process")

	options := &lib.Options{Offset: 0, Limit: 99999999999999, SortBy: "title", SortOrder: "asc"}
	all := &pubsub.Message{Payload: options, Reply: make(chan interface{}, capacity)}
	c.bus.Pub(all, "/get/movies")

	reply := <-all.Reply
	dto := reply.(*model.MoviesDTO)

	if c.settings.UnraidMode {
		for _, item := range dto.Items {
			// mlog.Info("Item is %s (%s)", item.Title, item.Location)

			index := strings.Index(item.Location, ":")
			if index == -1 {
				// a valid location is wopr:/mnt/user/films/bluray/22 Bullets (2010)/22.Bullets_BLURAY.iso
				// if a ':' isn't found, then this must be a stub
				continue
			}

			host := item.Location[:index]
			location := item.Location[index+1:]

			req := client.NewRequest("io.jbrodriguez.mediagui.agent."+host, "Agent.Exists", &agent.ExistsReq{
				Location: location,
			})

			rsp := &agent.ExistsRsp{}

			if err := client.Call(context.Background(), req, rsp); err != nil {
				mlog.Warning("Unable to connect to service (%s): %s", "io.jbrodriguez.mediagui.agent."+host, err)
				// lib.Notify(s.bus, "import:progress", "Unable to connect to host "+host)
				// lib.Notify(s.bus, "import:end", "Import process finished")
				// return
				continue
			}

			if !rsp.Exists {
				lib.Notify(c.bus, "prune:selected", fmt.Sprintf("UP FOR DELETION: [%d] %s (%s))", item.Id, item.Title, item.Location))

				movie := &pubsub.Message{Payload: item, Reply: make(chan interface{}, capacity)}
				c.bus.Pub(movie, "/command/movie/delete")
			}
		}
	} else {
		for _, item := range dto.Items {
			matches := c.re.FindStringSubmatch(item.Location)
			if len(matches) == 0 {
				continue
			}

			folder := filepath.Join("/Volumes", matches[1])
			if !c.maps[folder] {
				// mlog.Info("Folder not mapped (%s): %s", folder, item.Location)
				continue
			}

			// mlog.Info("Folder mapped (%s): %s", folder, item.Location)

			if _, err := os.Stat(item.Location); err != nil {
				if os.IsNotExist(err) {
					lib.Notify(c.bus, "prune:selected", fmt.Sprintf("UP FOR DELETION: [%d] %s (%s))", item.Id, item.Title, item.Location))

					movie := &pubsub.Message{Payload: item, Reply: make(chan interface{}, capacity)}
					c.bus.Pub(movie, "/command/movie/delete")
				}
			}
		}
	}

	// for _, folder := range c.settings.MediaFolders {
	// 	mlog.Info("folder: %s", folder)
	// 	if _, err := os.Stat(folder); err != nil {
	// 		if os.IsNotExist(err) {
	// 			lib.Notify(c.bus, "prune:error", fmt.Sprintf("Folder %s is not present. Aborting Prune process.", folder))
	// 			return
	// 		}
	// 	}
	// }

	// lib.Notify(c.bus, "prune:end", "Finished Prune Process")
	lib.Notify(c.bus, "prune:end", fmt.Sprintf("Prune process finished (%s elapsed)", time.Since(t0).String()))

}

func (c *Core) addMediaFolder(msg *pubsub.Message) {
	folder := msg.Payload.(string)
	mlog.Info("addMediaFolder: %s", folder)

	c.settings.MediaFolders = append(c.settings.MediaFolders, folder)
	c.settings.Save()

	cfg := &pubsub.Message{Payload: c.settings}
	c.bus.Pub(cfg, "/event/config/changed")

	msg.Reply <- &c.settings.Config
	// mlog.Info("Sent config")
}

// func (c *Core) waitFixMovie(ch chan interface{}, movie *model.Movie) {
// 	c.wg.Wait()
// 	ch <- movie
// }

// func (c *Core) doMovieUpdated(msg *pubsub.Message) {
// 	c.wg.Done()
// }

// func (c *Core) doMovieCachedForced(msg *pubsub.Message) {
// 	c.wg.Done()
// }

func (c *Core) doWorkUnitDone(msg *pubsub.Message) {
	c.wg.Done()
}
<commit_msg>Closes #34<commit_after>package services

import (
	"jbrodriguez/mediagui/server/src/dto"
	"jbrodriguez/mediagui/server/src/lib"
	"jbrodriguez/mediagui/server/src/model"
	"jbrodriguez/mediagui/server/src/proto"

	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/jbrodriguez/actor"
	"github.com/jbrodriguez/mlog"
	"github.com/jbrodriguez/pubsub"
	"github.com/micro/go-micro/client"

	"golang.org/x/net/context"
)

// Core -
type Core struct {
	bus      *pubsub.PubSub
	settings *lib.Settings

	actor *actor.Actor
	re    *regexp.Regexp
	maps  map[string]bool

	wg sync.WaitGroup
}

// NewCore -
func NewCore(bus *pubsub.PubSub, settings *lib.Settings) *Core {
	core := &Core{
		bus:      bus,
		settings: settings,
		actor:    actor.NewActor(bus),
	}
	return core
}

// Start -
func (c *Core) Start() {
	mlog.Info("Starting service Core ...")

	c.actor.Register("/get/config", c.getConfig)
	c.actor.Register("/post/import", c.importMovies)
	c.actor.Register("/post/add", c.addMovie)
	c.actor.Register("/post/prune", c.pruneMovies)
	c.actor.Register("/put/config/folder", c.addMediaFolder)
	c.actor.Register("/put/movies/fix", c.fixMovie)

	c.actor.Register("/event/movie/found", c.doMovieFound)
	c.actor.Register("/event/movie/tmdbnotfound", c.doMovieTmdbNotFound)
	c.actor.Register("/event/movie/scraped", c.doMovieScraped)
	c.actor.Register("/event/movie/rescraped", c.doMovieReScraped)
	// c.actor.Register("/event/movie/updated", c.doMovieUpdated)
	// c.actor.Register("/event/movie/cached/forced", c.doMovieCachedForced)

	c.actor.Register("/event/workunit/done", c.doWorkUnitDone)

	c.re = regexp.MustCompile(`(?i)/Volumes/(.*?)/.*`)
	c.maps = make(map[string]bool)

	// maps["/Volumes/wopr-films"] = true // for example
	for _, folder := range c.settings.MediaFolders {
		c.maps[folder] = false
		if _, err := os.Stat(folder); err == nil {
			c.maps[folder] = true
		}
	}

	go c.actor.React()
}

// Stop -
func (c *Core) Stop() {
	mlog.Info("Stopped service Core ...")
}

func (c *Core) getConfig(msg *pubsub.Message) {
	msg.Reply <- &c.settings.Config
	mlog.Info("Sent config")
}

func (c *Core) importMovies(msg *pubsub.Message) {
	t0 := time.Now()
	// mlog.Info("Begin movie scanning ...")
	lib.Notify(c.bus, "import:begin", "Import process started")

	c.wg.Add(1)

	c.bus.Pub(nil, "/command/movie/scan")
	//	msg.Reply <- &c.settings.Config
	// mlog.Info("Import finished")

	go func() {
		c.wg.Wait()
		// 𝛥t := float64(time.Since(t0)) / 1e9
		lib.Notify(c.bus, "import:end", fmt.Sprintf("Import process finished (%s elapsed)", time.Since(t0).String()))
	}()

}

func (c *Core) addMovie(msg *pubsub.Message) {
	// I'm expecting movie to have the following fields filled
	// Title
	movie := msg.Payload.(*model.Movie)

	// Since it's coming in through this endpoint, we know it's a stub
	movie.File_Title = movie.Title
	movie.Resolution = "n/a"
	movie.FileType = "n/a"
	movie.Location = "n/a"
	movie.Stub = 1

	// 3 operations, rescrape, update and cache
	c.wg.Add(1)

	// rescrape
	scrape := &pubsub.Message{Payload: movie, Reply: make(chan interface{}, 3)}
	c.bus.Pub(scrape, "/command/movie/scrape")

	go func() {
		c.wg.Wait()
		msg.Reply <- movie
	}()
}

func (c *Core) doMovieFound(msg *pubsub.Message) {
	movie := msg.Payload.(*model.Movie)

	check := &pubsub.Message{Payload: movie, Reply: make(chan interface{}, 3)}
	c.bus.Pub(check, "/command/movie/exists")

	reply := <-check.Reply
	exists := reply.(bool)

	if exists {
		mlog.Info("SKIPPED: exists [%s] (%s)", movie.Title, movie.Location)
	} else {
		lib.Notify(c.bus, "import:progress", fmt.Sprintf("NEW: [%s] (%s)", movie.Title, movie.Location))

		c.wg.Add(1)
		c.bus.Pub(msg, "/command/movie/scrape")
	}
}

func (c *Core) fixMovie(msg *pubsub.Message) {
	movie := msg.Payload.(*model.Movie)

	// 3 operations, rescrape, update and cache
	c.wg.Add(1)

	// rescrape
	scrape := &pubsub.Message{Payload: movie, Reply: make(chan interface{}, 3)}
	c.bus.Pub(scrape, "/command/movie/rescrape")

	go func() {
		c.wg.Wait()
		msg.Reply <- movie
	}()
	// go c.waitFixMovie(msg.Reply, movie)
}

func (c *Core) doMovieTmdbNotFound(msg *pubsub.Message) {
	dto := msg.Payload.(*dto.Scrape)

	store := &pubsub.Message{Payload: dto.Movie, Reply: make(chan interface{}, 3)}
	c.bus.Pub(store, "/command/movie/partialstore")
}

func (c *Core) doMovieScraped(msg *pubsub.Message) {
	dto := msg.Payload.(*dto.Scrape)

	mlog.Info("ScrapeDTO: %+v", dto)

	// I treat the following two commands as one, for the sake of the wg
	// now there are two oustanding locks, which will be decreased by each
	// responding service
	c.wg.Add(1)

	store := &pubsub.Message{Payload: dto.Movie, Reply: make(chan interface{}, 3)}
	c.bus.Pub(store, "/command/movie/store")

	cache := &pubsub.Message{Payload: dto, Reply: make(chan interface{}, 3)}
	c.bus.Pub(cache, "/command/movie/cache")

	// mlog.Info("ScrapeDTO: %+v", dto)
}

func (c *Core) doMovieReScraped(msg *pubsub.Message) {
	dto := msg.Payload.(*dto.Scrape)

	mlog.Info("ScrapeDTO: %+v", dto)

	// I treat the following two commands as one, for the sake of the wg
	// now there are two oustanding locks, which will be decreased by each
	// responding service
	c.wg.Add(1)

	store := &pubsub.Message{Payload: dto.Movie, Reply: make(chan interface{}, 3)}
	c.bus.Pub(store, "/command/movie/update")

	cache := &pubsub.Message{Payload: dto, Reply: make(chan interface{}, 3)}
	c.bus.Pub(cache, "/command/movie/cache")

	// mlog.Info("ScrapeDTO: %+v", dto)
}

func (c *Core) pruneMovies(msg *pubsub.Message) {
	t0 := time.Now()

	lib.Notify(c.bus, "prune:begin", "Started Prune Process")

	options := &lib.Options{Offset: 0, Limit: 99999999999999, SortBy: "title", SortOrder: "asc"}
	all := &pubsub.Message{Payload: options, Reply: make(chan interface{}, capacity)}
	c.bus.Pub(all, "/get/movies")

	reply := <-all.Reply
	dto := reply.(*model.MoviesDTO)

	if c.settings.UnraidMode {
		for _, item := range dto.Items {
			// mlog.Info("Item is %s (%s)", item.Title, item.Location)

			index := strings.Index(item.Location, ":")
			if index == -1 {
				// a valid location is wopr:/mnt/user/films/bluray/22 Bullets (2010)/22.Bullets_BLURAY.iso
				// if a ':' isn't found, then this must be a stub
				continue
			}

			host := item.Location[:index]
			location := item.Location[index+1:]

			req := client.NewRequest("io.jbrodriguez.mediagui.agent."+host, "Agent.Exists", &agent.ExistsReq{
				Location: location,
			})

			rsp := &agent.ExistsRsp{}

			if err := client.Call(context.Background(), req, rsp); err != nil {
				// mlog.Warning("Unable to connect to service (%s): %s", "io.jbrodriguez.mediagui.agent."+host, err)
				// lib.Notify(s.bus, "import:progress", "Unable to connect to host "+host)
				// lib.Notify(s.bus, "import:end", "Import process finished")
				// return
				continue
			}

			if !rsp.Exists {
				lib.Notify(c.bus, "prune:selected", fmt.Sprintf("UP FOR DELETION: [%d] %s (%s))", item.Id, item.Title, item.Location))

				movie := &pubsub.Message{Payload: item, Reply: make(chan interface{}, capacity)}
				c.bus.Pub(movie, "/command/movie/delete")
			}
		}
	} else {
		for _, item := range dto.Items {
			matches := c.re.FindStringSubmatch(item.Location)
			if len(matches) == 0 {
				continue
			}

			folder := filepath.Join("/Volumes", matches[1])
			if !c.maps[folder] {
				// mlog.Info("Folder not mapped (%s): %s", folder, item.Location)
				continue
			}

			// mlog.Info("Folder mapped (%s): %s", folder, item.Location)

			if _, err := os.Stat(item.Location); err != nil {
				if os.IsNotExist(err) {
					lib.Notify(c.bus, "prune:selected", fmt.Sprintf("UP FOR DELETION: [%d] %s (%s))", item.Id, item.Title, item.Location))

					movie := &pubsub.Message{Payload: item, Reply: make(chan interface{}, capacity)}
					c.bus.Pub(movie, "/command/movie/delete")
				}
			}
		}
	}

	// for _, folder := range c.settings.MediaFolders {
	// 	mlog.Info("folder: %s", folder)
	// 	if _, err := os.Stat(folder); err != nil {
	// 		if os.IsNotExist(err) {
	// 			lib.Notify(c.bus, "prune:error", fmt.Sprintf("Folder %s is not present. Aborting Prune process.", folder))
	// 			return
	// 		}
	// 	}
	// }

	// lib.Notify(c.bus, "prune:end", "Finished Prune Process")
	lib.Notify(c.bus, "prune:end", fmt.Sprintf("Prune process finished (%s elapsed)", time.Since(t0).String()))

}

func (c *Core) addMediaFolder(msg *pubsub.Message) {
	folder := msg.Payload.(string)
	mlog.Info("addMediaFolder: %s", folder)

	c.settings.MediaFolders = append(c.settings.MediaFolders, folder)
	c.settings.Save()

	cfg := &pubsub.Message{Payload: c.settings}
	c.bus.Pub(cfg, "/event/config/changed")

	msg.Reply <- &c.settings.Config
	// mlog.Info("Sent config")
}

// func (c *Core) waitFixMovie(ch chan interface{}, movie *model.Movie) {
// 	c.wg.Wait()
// 	ch <- movie
// }

// func (c *Core) doMovieUpdated(msg *pubsub.Message) {
// 	c.wg.Done()
// }

// func (c *Core) doMovieCachedForced(msg *pubsub.Message) {
// 	c.wg.Done()
// }

func (c *Core) doWorkUnitDone(msg *pubsub.Message) {
	c.wg.Done()
}
<|endoftext|>


<commit_before>package goxlsx

import (
	"archive/zip"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"log"
	"path"
	"strconv"
)

type Worksheet struct {
	Name        string
	filename    string
	id          string
	rows        map[int]*row
	spreadsheet *Spreadsheet
}

type cell struct {
	Name  string
	Type  string
	Value string
}

type row struct {
	Num   int
	Cells map[int]*cell
}

type Spreadsheet struct {
	filepath        string
	compressedFiles []zip.File
	Worksheets      []*Worksheet
	sharedStrings   []string
}

func readWorkbook(d *xml.Decoder, s *Spreadsheet) []*Worksheet {
	worksheets := make([]*Worksheet, 0, 5)
	var (
		err   error
		token xml.Token
	)

	for {
		token, err = d.Token()
		if err != nil {
			if err != io.EOF {
				log.Fatal(err)
			}
			break
		}
		switch x := token.(type) {
		case xml.StartElement:
			switch x.Name.Local {
			case "sheet":
				ws := new(Worksheet)
				ws.spreadsheet = s
				for _, a := range x.Attr {
					if a.Name.Local == "name" {
						ws.Name = a.Value
					}
					if a.Name.Local == "sheetId" {
						ws.id = a.Value
					}
				}
				worksheets = append(worksheets, ws)
			}
		}
	}
	return worksheets
}

func readStrings(d *xml.Decoder, s *Spreadsheet) {
	var (
		err   error
		data  []byte
		token xml.Token
	)
	for {
		token, err = d.Token()
		if err != nil {
			if err != io.EOF {
				log.Fatal(err)
			}
			break
		}
		switch x := token.(type) {
		case xml.StartElement:
			switch x.Name.Local {
			case "sst":
				// root element
				for i := 0; i < len(x.Attr); i++ {
					if x.Attr[i].Name.Local == "uniqueCount" {
						count, err := strconv.Atoi(x.Attr[i].Value)
						if err != nil {
							log.Fatal(err)
						}
						s.sharedStrings = make([]string, 0, count)
					}
				}
			default:
				// log.Println(x.Name.Local)
			}
		case xml.CharData:
			data = x.Copy()
		case xml.EndElement:
			switch x.Name.Local {
			case "t":
				s.sharedStrings = append(s.sharedStrings, string(data))
			}
		}

	}
}

func OpenFile(path string) (*Spreadsheet, error) {
	xlsx := new(Spreadsheet)
	xlsx.filepath = path

	r, err := zip.OpenReader(path)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	for _, f := range r.File {
		if f.Name == "xl/workbook.xml" {
			rc, err := f.Open()
			if err != nil {
				return nil, err
			}
			xlsx.Worksheets = readWorkbook(xml.NewDecoder(rc), xlsx)
			rc.Close()
		}
		if f.Name == "xl/sharedStrings.xml" {
			rc, err := f.Open()
			if err != nil {
				return nil, err
			}
			readStrings(xml.NewDecoder(rc), xlsx)
			rc.Close()
		}
	}
	return xlsx, nil
}

func readWorksheetXML(dec *xml.Decoder, s *Spreadsheet) (map[int]*row, error) {
	rows := make(map[int]*row)
	var (
		err         error
		token       xml.Token
		rownum      int
		currentCell *cell
		currentRow  *row
	)
	for {
		token, err = dec.Token()
		if err != nil {
			if err != io.EOF {
				return nil, err
			}
			break
		}
		switch x := token.(type) {
		case xml.StartElement:
			switch x.Name.Local {
			case "row":
				currentRow = &row{}
				currentRow.Cells = make(map[int]*cell)
				for _, a := range x.Attr {
					if a.Name.Local == "r" {
						rownum, err = strconv.Atoi(a.Value)
						if err != nil {
							return nil, err
						}
					}
				}
				currentRow.Num = rownum
				rows[rownum] = currentRow
			case "c":
				currentCell = &cell{}
				var cellnumber rune
				for _, a := range x.Attr {
					switch a.Name.Local {
					case "r":
						for _, v := range a.Value {
							if v >= 'A' && v <= 'Z' {
								cellnumber = cellnumber*26 + v - 'A' + 1
							}
						}
					case "t":
						if a.Value == "s" {
							currentCell.Type = "s"
						}
					}

				}
				currentRow.Cells[int(cellnumber)] = currentCell
			}
		case xml.EndElement:
			switch x.Name.Local {
			case "c":
				currentCell = nil
			}
		case xml.CharData:
			if currentCell != nil {
				val := string(x.Copy())
				if currentCell.Type == "s" {
					valInt, _ := strconv.Atoi(val)
					currentCell.Value = s.sharedStrings[valInt]
				} else {
					currentCell.Value = val
				}
			}
		}
	}
	return rows, nil
}

func (ws *Worksheet) readWorksheetZIP() error {
	r, err := zip.OpenReader(ws.spreadsheet.filepath)
	if err != nil {
		return err
	}
	defer r.Close()

	for _, f := range r.File {
		if f.Name == ws.filename {
			rc, err := f.Open()
			if err != nil {
				return err
			}
			defer rc.Close()
			rows, err := readWorksheetXML(xml.NewDecoder(rc), ws.spreadsheet)
			ws.rows = rows
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func (ws *Worksheet) Cell(row, column int) string {
	xrow := ws.rows[row]
	if xrow == nil {
		return ""
	}
	if xrow.Cells[column] == nil {
		return ""
	}
	return xrow.Cells[column].Value
}

func (s *Spreadsheet) GetWorksheet(number int) (*Worksheet, error) {
	if number >= len(s.Worksheets) || number < 0 {
		return nil, errors.New("Index out of range")
	}
	ws := s.Worksheets[number]
	ws.filename = path.Join("xl/worksheets", fmt.Sprintf("sheet%s.xml", ws.id))
	err := ws.readWorksheetZIP()
	if err != nil {
		return nil, err
	}
	return ws, nil
}
<commit_msg>Don't use path package at all<commit_after>package goxlsx

import (
	"archive/zip"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"log"
	"strconv"
)

type Worksheet struct {
	Name        string
	filename    string
	id          string
	rows        map[int]*row
	spreadsheet *Spreadsheet
}

type cell struct {
	Name  string
	Type  string
	Value string
}

type row struct {
	Num   int
	Cells map[int]*cell
}

type Spreadsheet struct {
	filepath        string
	compressedFiles []zip.File
	Worksheets      []*Worksheet
	sharedStrings   []string
}

func readWorkbook(d *xml.Decoder, s *Spreadsheet) []*Worksheet {
	worksheets := make([]*Worksheet, 0, 5)
	var (
		err   error
		token xml.Token
	)

	for {
		token, err = d.Token()
		if err != nil {
			if err != io.EOF {
				log.Fatal(err)
			}
			break
		}
		switch x := token.(type) {
		case xml.StartElement:
			switch x.Name.Local {
			case "sheet":
				ws := new(Worksheet)
				ws.spreadsheet = s
				for _, a := range x.Attr {
					if a.Name.Local == "name" {
						ws.Name = a.Value
					}
					if a.Name.Local == "sheetId" {
						ws.id = a.Value
					}
				}
				worksheets = append(worksheets, ws)
			}
		}
	}
	return worksheets
}

func readStrings(d *xml.Decoder, s *Spreadsheet) {
	var (
		err   error
		data  []byte
		token xml.Token
	)
	for {
		token, err = d.Token()
		if err != nil {
			if err != io.EOF {
				log.Fatal(err)
			}
			break
		}
		switch x := token.(type) {
		case xml.StartElement:
			switch x.Name.Local {
			case "sst":
				// root element
				for i := 0; i < len(x.Attr); i++ {
					if x.Attr[i].Name.Local == "uniqueCount" {
						count, err := strconv.Atoi(x.Attr[i].Value)
						if err != nil {
							log.Fatal(err)
						}
						s.sharedStrings = make([]string, 0, count)
					}
				}
			default:
				// log.Println(x.Name.Local)
			}
		case xml.CharData:
			data = x.Copy()
		case xml.EndElement:
			switch x.Name.Local {
			case "t":
				s.sharedStrings = append(s.sharedStrings, string(data))
			}
		}

	}
}

func OpenFile(path string) (*Spreadsheet, error) {
	xlsx := new(Spreadsheet)
	xlsx.filepath = path

	r, err := zip.OpenReader(path)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	for _, f := range r.File {
		if f.Name == "xl/workbook.xml" {
			rc, err := f.Open()
			if err != nil {
				return nil, err
			}
			xlsx.Worksheets = readWorkbook(xml.NewDecoder(rc), xlsx)
			rc.Close()
		}
		if f.Name == "xl/sharedStrings.xml" {
			rc, err := f.Open()
			if err != nil {
				return nil, err
			}
			readStrings(xml.NewDecoder(rc), xlsx)
			rc.Close()
		}
	}
	return xlsx, nil
}

func readWorksheetXML(dec *xml.Decoder, s *Spreadsheet) (map[int]*row, error) {
	rows := make(map[int]*row)
	var (
		err         error
		token       xml.Token
		rownum      int
		currentCell *cell
		currentRow  *row
	)
	for {
		token, err = dec.Token()
		if err != nil {
			if err != io.EOF {
				return nil, err
			}
			break
		}
		switch x := token.(type) {
		case xml.StartElement:
			switch x.Name.Local {
			case "row":
				currentRow = &row{}
				currentRow.Cells = make(map[int]*cell)
				for _, a := range x.Attr {
					if a.Name.Local == "r" {
						rownum, err = strconv.Atoi(a.Value)
						if err != nil {
							return nil, err
						}
					}
				}
				currentRow.Num = rownum
				rows[rownum] = currentRow
			case "c":
				currentCell = &cell{}
				var cellnumber rune
				for _, a := range x.Attr {
					switch a.Name.Local {
					case "r":
						for _, v := range a.Value {
							if v >= 'A' && v <= 'Z' {
								cellnumber = cellnumber*26 + v - 'A' + 1
							}
						}
					case "t":
						if a.Value == "s" {
							currentCell.Type = "s"
						}
					}

				}
				currentRow.Cells[int(cellnumber)] = currentCell
			}
		case xml.EndElement:
			switch x.Name.Local {
			case "c":
				currentCell = nil
			}
		case xml.CharData:
			if currentCell != nil {
				val := string(x.Copy())
				if currentCell.Type == "s" {
					valInt, _ := strconv.Atoi(val)
					currentCell.Value = s.sharedStrings[valInt]
				} else {
					currentCell.Value = val
				}
			}
		}
	}
	return rows, nil
}

func (ws *Worksheet) readWorksheetZIP() error {
	r, err := zip.OpenReader(ws.spreadsheet.filepath)
	if err != nil {
		return err
	}
	defer r.Close()

	for _, f := range r.File {
		if f.Name == ws.filename {
			rc, err := f.Open()
			if err != nil {
				return err
			}
			defer rc.Close()
			rows, err := readWorksheetXML(xml.NewDecoder(rc), ws.spreadsheet)
			ws.rows = rows
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func (ws *Worksheet) Cell(row, column int) string {
	xrow := ws.rows[row]
	if xrow == nil {
		return ""
	}
	if xrow.Cells[column] == nil {
		return ""
	}
	return xrow.Cells[column].Value
}

func (s *Spreadsheet) GetWorksheet(number int) (*Worksheet, error) {
	if number >= len(s.Worksheets) || number < 0 {
		return nil, errors.New("Index out of range")
	}
	ws := s.Worksheets[number]
	ws.filename = "xl/worksheets" + "/" + fmt.Sprintf("sheet%s.xml", ws.id)
	err := ws.readWorksheetZIP()
	if err != nil {
		return nil, err
	}
	return ws, nil
}
<|endoftext|>