<commit_before>package inputfile

import (
	"bufio"
	"bytes"
	"context"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/go-fsnotify/fsnotify"
	"github.com/tsaikd/KDGoLib/errutil"
	"github.com/tsaikd/gogstash/config"
	"github.com/tsaikd/gogstash/config/logevent"
	"golang.org/x/sync/errgroup"
)

// ModuleName is the name used in config file
const ModuleName = "file"

// InputConfig holds the configuration json fields and internal objects
type InputConfig struct {
	config.InputConfig
	Path                 string `json:"path"`
	StartPos             string `json:"start_position,omitempty"` // one of ["beginning", "end"]
	SinceDBPath          string `json:"sincedb_path,omitempty"`
	SinceDBWriteInterval int    `json:"sincedb_write_interval,omitempty"`

	hostname            string
	SinceDBInfos        map[string]*SinceDBInfo `json:"-"`
	sinceDBLastInfosRaw []byte
	SinceDBLastSaveTime time.Time `json:"-"`
}

// DefaultInputConfig returns an InputConfig struct with default values
func DefaultInputConfig() InputConfig {
	return InputConfig{
		InputConfig: config.InputConfig{
			CommonConfig: config.CommonConfig{
				Type: ModuleName,
			},
		},
		StartPos:             "end",
		SinceDBPath:          ".sincedb.json",
		SinceDBWriteInterval: 15,

		SinceDBInfos: map[string]*SinceDBInfo{},
	}
}

// errors
var (
	ErrorGlobFailed1 = errutil.NewFactory("glob(%q) failed")
)

// InitHandler initialize the input plugin
func InitHandler(ctx context.Context, raw *config.ConfigRaw) (config.TypeInputConfig, error) {
	conf := DefaultInputConfig()
	err := config.ReflectConfig(raw, &conf)
	if err != nil {
		return nil, err
	}

	if conf.hostname, err = os.Hostname(); err != nil {
		return nil, err
	}

	return &conf, nil
}

// Start wraps the actual function starting the plugin
func (t *InputConfig) Start(ctx context.Context, msgChan chan<- logevent.LogEvent) (err error) {
	logger := config.Logger

	if err = t.LoadSinceDBInfos(); err != nil {
		return
	}

	matches, err := filepath.Glob(t.Path)
	if err != nil {
		return ErrorGlobFailed1.New(err, t.Path)
	}

	eg, ctx := errgroup.WithContext(ctx)

	eg.Go(func() error {
		return t.CheckSaveSinceDBInfosLoop(ctx)
	})

	for _, fpath := range matches {
		if fpath, err = filepath.EvalSymlinks(fpath); err != nil {
			logger.Errorf("Get symlinks failed: %q\n%v", fpath, err)
			continue
		}

		var fi os.FileInfo
		if fi, err = os.Stat(fpath); err != nil {
			logger.Errorf("stat(%q) failed\n%s", t.Path, err)
			continue
		}

		if fi.IsDir() {
			logger.Infof("Skipping directory: %q", t.Path)
			continue
		}

		func(fpath string) {
			readEventChan := make(chan fsnotify.Event, 10)
			eg.Go(func() error {
				return t.fileReadLoop(ctx, readEventChan, fpath, logger, msgChan)
			})
			eg.Go(func() error {
				return t.fileWatchLoop(ctx, readEventChan, fpath, fsnotify.Create|fsnotify.Write)
			})
		}(fpath)
	}

	return eg.Wait()
}

func (t *InputConfig) fileReadLoop(
	ctx context.Context,
	readEventChan chan fsnotify.Event,
	fpath string,
	logger *logrus.Logger,
	msgChan chan<- logevent.LogEvent,
) (err error) {
	var (
		since     *SinceDBInfo
		fp        *os.File
		truncated bool
		ok        bool
		whence    int
		reader    *bufio.Reader
		line      string
		size      int

		buffer = &bytes.Buffer{}
	)

	if fpath, err = filepath.EvalSymlinks(fpath); err != nil {
		logger.Errorf("Get symlinks failed: %q\n%v", fpath, err)
		return
	}

	if since, ok = t.SinceDBInfos[fpath]; !ok {
		t.SinceDBInfos[fpath] = &SinceDBInfo{}
		since = t.SinceDBInfos[fpath]
	}

	if since.Offset == 0 {
		if t.StartPos == "end" {
			whence = os.SEEK_END
		} else {
			whence = os.SEEK_SET
		}
	} else {
		whence = os.SEEK_SET
	}

	if fp, reader, err = openfile(fpath, since.Offset, whence); err != nil {
		return
	}
	defer fp.Close()

	if truncated, err = isFileTruncated(fp, since); err != nil {
		return
	}
	if truncated {
		logger.Warnf("File truncated, seeking to beginning: %q", fpath)
		since.Offset = 0
		if _, err = fp.Seek(since.Offset, os.SEEK_SET); err != nil {
			logger.Errorf("seek file failed: %q", fpath)
			return
		}
	}

	for {
		select {
		case <-ctx.Done():
			return
		default:
		}

		if line, size, err = readline(ctx, reader, buffer); err != nil {
			if err == io.EOF {
				watchev := <-readEventChan
				logger.Debug("fileReadLoop recv:", watchev)
				if watchev.Op&fsnotify.Create == fsnotify.Create {
					logger.Warnf("File recreated, seeking to beginning: %q", fpath)
					fp.Close()
					since.Offset = 0
					if fp, reader, err = openfile(fpath, since.Offset, os.SEEK_SET); err != nil {
						return
					}
				}
				if truncated, err = isFileTruncated(fp, since); err != nil {
					return
				}
				if truncated {
					logger.Warnf("File truncated, seeking to beginning: %q", fpath)
					since.Offset = 0
					if _, err = fp.Seek(since.Offset, os.SEEK_SET); err != nil {
						logger.Errorf("seek file failed: %q", fpath)
						return
					}
					continue
				}
				logger.Debugf("watch %q %q %v", watchev.Name, fpath, watchev)
				continue
			} else {
				return
			}
		}

		event := logevent.LogEvent{
			Timestamp: time.Now(),
			Message:   line,
			Extra: map[string]interface{}{
				"host":   t.hostname,
				"path":   fpath,
				"offset": since.Offset,
			},
		}

		since.Offset += int64(size)

		logger.Debugf("%q %v", event.Message, event)
		msgChan <- event
		//self.SaveSinceDBInfos()
		t.CheckSaveSinceDBInfos()
	}
}

func (self *InputConfig) fileWatchLoop(ctx context.Context, readEventChan chan fsnotify.Event, fpath string, op fsnotify.Op) (err error) {
	var (
		event fsnotify.Event
	)
	for {
		select {
		case <-ctx.Done():
			return nil
		default:
		}

		if event, err = waitWatchEvent(ctx, fpath, op); err != nil {
			return
		}
		readEventChan <- event
	}
}

func isFileTruncated(fp *os.File, since *SinceDBInfo) (truncated bool, err error) {
	var (
		fi os.FileInfo
	)
	if fi, err = fp.Stat(); err != nil {
		err = errutil.New("stat file failed: "+fp.Name(), err)
		return
	}
	if fi.Size() < since.Offset {
		truncated = true
	} else {
		truncated = false
	}
	return
}

func openfile(fpath string, offset int64, whence int) (fp *os.File, reader *bufio.Reader, err error) {
	if fp, err = os.Open(fpath); err != nil {
		err = errutil.New("open file failed: "+fpath, err)
		return
	}

	if _, err = fp.Seek(offset, whence); err != nil {
		err = errutil.New("seek file failed: " + fpath)
		return
	}

	reader = bufio.NewReaderSize(fp, 16*1024)
	return
}

func readline(ctx context.Context, reader *bufio.Reader, buffer *bytes.Buffer) (line string, size int, err error) {
	var (
		segment []byte
	)

	for {
		select {
		case <-ctx.Done():
			return
		default:
		}

		if segment, err = reader.ReadBytes('\n'); err != nil {
			if err != io.EOF {
				err = errutil.New("read line failed", err)
			}
			return
		}

		if _, err = buffer.Write(segment); err != nil {
			err = errutil.New("write buffer failed", err)
			return
		}

		if isPartialLine(segment) {
			time.Sleep(1 * time.Second)
		} else {
			size = buffer.Len()
			line = buffer.String()
			buffer.Reset()
			line = strings.TrimRight(line, "\r\n")
			return
		}
	}
}

func isPartialLine(segment []byte) bool {
	if len(segment) < 1 {
		return true
	}
	if segment[len(segment)-1] != '\n' {
		return true
	}
	return false
}

var (
	mapWatcher = map[string]*fsnotify.Watcher{}
)

func waitWatchEvent(ctx context.Context, fpath string, op fsnotify.Op) (event fsnotify.Event, err error) {
	var (
		fdir    string
		watcher *fsnotify.Watcher
		ok      bool
	)

	if fpath, err = filepath.EvalSymlinks(fpath); err != nil {
		err = errutil.New("Get symlinks failed: "+fpath, err)
		return
	}

	fdir = filepath.Dir(fpath)

	if watcher, ok = mapWatcher[fdir]; !ok {
		//		logger.Debugf("create new watcher for %q", fdir)
		if watcher, err = fsnotify.NewWatcher(); err != nil {
			err = errutil.New("create new watcher failed: "+fdir, err)
			return
		}
		mapWatcher[fdir] = watcher
		if err = watcher.Add(fdir); err != nil {
			err = errutil.New("add new watch path failed: "+fdir, err)
			return
		}
	}

	for {
		select {
		case <-ctx.Done():
			return
		case event = <-watcher.Events:
			if event.Name == fpath {
				if op > 0 {
					if event.Op&op > 0 {
						return
					}
				} else {
					return
				}
			}
		case err = <-watcher.Errors:
			err = errutil.New("watcher error", err)
			return
		}
	}
}
<commit_msg>for https://github.com/tsaikd/gogstash/issues/30<commit_after>package inputfile

import (
	"bufio"
	"bytes"
	"context"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/go-fsnotify/fsnotify"
	"github.com/tsaikd/KDGoLib/errutil"
	"github.com/tsaikd/KDGoLib/futil"
	"github.com/tsaikd/gogstash/config"
	"github.com/tsaikd/gogstash/config/logevent"
	"golang.org/x/sync/errgroup"
)

// ModuleName is the name used in config file
const ModuleName = "file"

// InputConfig holds the configuration json fields and internal objects
type InputConfig struct {
	config.InputConfig
	Path                 string `json:"path"`
	StartPos             string `json:"start_position,omitempty"` // one of ["beginning", "end"]
	SinceDBPath          string `json:"sincedb_path,omitempty"`
	SinceDBWriteInterval int    `json:"sincedb_write_interval,omitempty"`

	hostname            string
	SinceDBInfos        map[string]*SinceDBInfo `json:"-"`
	sinceDBLastInfosRaw []byte
	SinceDBLastSaveTime time.Time `json:"-"`
}

// DefaultInputConfig returns an InputConfig struct with default values
func DefaultInputConfig() InputConfig {
	return InputConfig{
		InputConfig: config.InputConfig{
			CommonConfig: config.CommonConfig{
				Type: ModuleName,
			},
		},
		StartPos:             "end",
		SinceDBPath:          ".sincedb.json",
		SinceDBWriteInterval: 15,

		SinceDBInfos: map[string]*SinceDBInfo{},
	}
}

// errors
var (
	ErrorGlobFailed1 = errutil.NewFactory("glob(%q) failed")
)

// InitHandler initialize the input plugin
func InitHandler(ctx context.Context, raw *config.ConfigRaw) (config.TypeInputConfig, error) {
	conf := DefaultInputConfig()
	err := config.ReflectConfig(raw, &conf)
	if err != nil {
		return nil, err
	}

	if conf.hostname, err = os.Hostname(); err != nil {
		return nil, err
	}

	return &conf, nil
}

// Start wraps the actual function starting the plugin
func (t *InputConfig) Start(ctx context.Context, msgChan chan<- logevent.LogEvent) (err error) {
	logger := config.Logger

	if err = t.LoadSinceDBInfos(); err != nil {
		return
	}

	matches, err := filepath.Glob(t.Path)
	if err != nil {
		return ErrorGlobFailed1.New(err, t.Path)
	}

	eg, ctx := errgroup.WithContext(ctx)

	eg.Go(func() error {
		return t.CheckSaveSinceDBInfosLoop(ctx)
	})

	for _, fpath := range matches {
		if fpath, err = evalSymlinks(ctx, fpath); err != nil {
			logger.Errorf("Get symlinks failed: %q\n%v", fpath, err)
			continue
		}

		var fi os.FileInfo
		if fi, err = os.Stat(fpath); err != nil {
			logger.Errorf("stat(%q) failed\n%s", t.Path, err)
			continue
		}

		if fi.IsDir() {
			logger.Infof("Skipping directory: %q", t.Path)
			continue
		}

		func(fpath string) {
			readEventChan := make(chan fsnotify.Event, 10)
			eg.Go(func() error {
				return t.fileReadLoop(ctx, readEventChan, fpath, logger, msgChan)
			})
			eg.Go(func() error {
				return t.fileWatchLoop(ctx, readEventChan, fpath, fsnotify.Create|fsnotify.Write)
			})
		}(fpath)
	}

	return eg.Wait()
}

func (t *InputConfig) fileReadLoop(
	ctx context.Context,
	readEventChan chan fsnotify.Event,
	fpath string,
	logger *logrus.Logger,
	msgChan chan<- logevent.LogEvent,
) (err error) {
	var (
		since     *SinceDBInfo
		fp        *os.File
		truncated bool
		ok        bool
		whence    int
		reader    *bufio.Reader
		line      string
		size      int

		buffer = &bytes.Buffer{}
	)

	if fpath, err = evalSymlinks(ctx, fpath); err != nil {
		logger.Errorf("Get symlinks failed: %q\n%v", fpath, err)
		return
	}

	if since, ok = t.SinceDBInfos[fpath]; !ok {
		t.SinceDBInfos[fpath] = &SinceDBInfo{}
		since = t.SinceDBInfos[fpath]
	}

	if since.Offset == 0 {
		if t.StartPos == "end" {
			whence = os.SEEK_END
		} else {
			whence = os.SEEK_SET
		}
	} else {
		whence = os.SEEK_SET
	}

	if fp, reader, err = openfile(fpath, since.Offset, whence); err != nil {
		return
	}
	defer fp.Close()

	if truncated, err = isFileTruncated(fp, since); err != nil {
		return
	}
	if truncated {
		logger.Warnf("File truncated, seeking to beginning: %q", fpath)
		since.Offset = 0
		if _, err = fp.Seek(since.Offset, os.SEEK_SET); err != nil {
			logger.Errorf("seek file failed: %q", fpath)
			return
		}
	}

	for {
		select {
		case <-ctx.Done():
			return
		default:
		}

		if line, size, err = readline(ctx, reader, buffer); err != nil {
			if err == io.EOF {
				watchev := <-readEventChan
				logger.Debug("fileReadLoop recv:", watchev)
				if watchev.Op&fsnotify.Create == fsnotify.Create {
					logger.Warnf("File recreated, seeking to beginning: %q", fpath)
					fp.Close()
					since.Offset = 0
					if fp, reader, err = openfile(fpath, since.Offset, os.SEEK_SET); err != nil {
						return
					}
				}
				if truncated, err = isFileTruncated(fp, since); err != nil {
					return
				}
				if truncated {
					logger.Warnf("File truncated, seeking to beginning: %q", fpath)
					since.Offset = 0
					if _, err = fp.Seek(since.Offset, os.SEEK_SET); err != nil {
						logger.Errorf("seek file failed: %q", fpath)
						return
					}
					continue
				}
				logger.Debugf("watch %q %q %v", watchev.Name, fpath, watchev)
				continue
			} else {
				return
			}
		}

		event := logevent.LogEvent{
			Timestamp: time.Now(),
			Message:   line,
			Extra: map[string]interface{}{
				"host":   t.hostname,
				"path":   fpath,
				"offset": since.Offset,
			},
		}

		since.Offset += int64(size)

		logger.Debugf("%q %v", event.Message, event)
		msgChan <- event
		//self.SaveSinceDBInfos()
		t.CheckSaveSinceDBInfos()
	}
}

func (self *InputConfig) fileWatchLoop(ctx context.Context, readEventChan chan fsnotify.Event, fpath string, op fsnotify.Op) (err error) {
	var (
		event fsnotify.Event
	)
	for {
		select {
		case <-ctx.Done():
			return nil
		default:
		}

		if event, err = waitWatchEvent(ctx, fpath, op); err != nil {
			return
		}
		readEventChan <- event
	}
}

func isFileTruncated(fp *os.File, since *SinceDBInfo) (truncated bool, err error) {
	var (
		fi os.FileInfo
	)
	if fi, err = fp.Stat(); err != nil {
		err = errutil.New("stat file failed: "+fp.Name(), err)
		return
	}
	if fi.Size() < since.Offset {
		truncated = true
	} else {
		truncated = false
	}
	return
}

func openfile(fpath string, offset int64, whence int) (fp *os.File, reader *bufio.Reader, err error) {
	if fp, err = os.Open(fpath); err != nil {
		err = errutil.New("open file failed: "+fpath, err)
		return
	}

	if _, err = fp.Seek(offset, whence); err != nil {
		err = errutil.New("seek file failed: " + fpath)
		return
	}

	reader = bufio.NewReaderSize(fp, 16*1024)
	return
}

func readline(ctx context.Context, reader *bufio.Reader, buffer *bytes.Buffer) (line string, size int, err error) {
	var (
		segment []byte
	)

	for {
		select {
		case <-ctx.Done():
			return
		default:
		}

		if segment, err = reader.ReadBytes('\n'); err != nil {
			if err != io.EOF {
				err = errutil.New("read line failed", err)
			}
			return
		}

		if _, err = buffer.Write(segment); err != nil {
			err = errutil.New("write buffer failed", err)
			return
		}

		if isPartialLine(segment) {
			time.Sleep(1 * time.Second)
		} else {
			size = buffer.Len()
			line = buffer.String()
			buffer.Reset()
			line = strings.TrimRight(line, "\r\n")
			return
		}
	}
}

func isPartialLine(segment []byte) bool {
	if len(segment) < 1 {
		return true
	}
	if segment[len(segment)-1] != '\n' {
		return true
	}
	return false
}

var (
	mapWatcher = map[string]*fsnotify.Watcher{}
)

func waitWatchEvent(ctx context.Context, fpath string, op fsnotify.Op) (event fsnotify.Event, err error) {
	var (
		fdir    string
		watcher *fsnotify.Watcher
		ok      bool
	)

	if fpath, err = evalSymlinks(ctx, fpath); err != nil {
		err = errutil.New("Get symlinks failed: "+fpath, err)
		return
	}

	fdir = filepath.Dir(fpath)

	if watcher, ok = mapWatcher[fdir]; !ok {
		//		logger.Debugf("create new watcher for %q", fdir)
		if watcher, err = fsnotify.NewWatcher(); err != nil {
			err = errutil.New("create new watcher failed: "+fdir, err)
			return
		}
		mapWatcher[fdir] = watcher
		if err = watcher.Add(fdir); err != nil {
			err = errutil.New("add new watch path failed: "+fdir, err)
			return
		}
	}

	for {
		select {
		case <-ctx.Done():
			return
		case event = <-watcher.Events:
			if event.Name == fpath {
				if op > 0 {
					if event.Op&op > 0 {
						return
					}
				} else {
					return
				}
			}
		case err = <-watcher.Errors:
			err = errutil.New("watcher error", err)
			return
		}
	}
}

func evalSymlinks(ctx context.Context, path string) (string, error) {
	// https://github.com/tsaikd/gogstash/issues/30
	for retry := 5; retry > 0; retry-- {
		if futil.IsExist(path) {
			break
		}

		select {
		case <-ctx.Done():
			return path, nil
		case <-time.After(500 * time.Millisecond):
		}
	}

	if futil.IsNotExist(path) {
		return path, os.ErrNotExist
	}

	return filepath.EvalSymlinks(path)
}
<|endoftext|>


<commit_before>package main

import (
    "flag"
    "fmt"
    "log"
    "os"
    "time"
    "net/http"
    "sort"

    "github.com/coreos/pkg/flagutil"
    "github.com/dghubble/go-twitter/twitter"
    "github.com/dghubble/oauth1"
    "github.com/davecgh/go-spew/spew"
    "github.com/gorilla/feeds"
)

type replaceObject struct {
    from int
    to int
    replacement string
}

type ReplacementList []replaceObject

func (dl ReplacementList) Len() int {
   return len(dl)
}

func (dl ReplacementList) Swap(i, j int) {
    dl[i], dl[j] = dl[j], dl[i]
}

func (dl ReplacementList) Less(i, j int) bool {
    return dl[i].from < dl[j].from
}

// Parse the text, identify twitter shortened URLs and replace them
func parseTweetText(tweet twitter.Tweet) string {
    text := tweet.Text
    var replacements ReplacementList

    // Special case, if it's retweeted then the URL placement might not be correct
    // and the tweet can also contain cut off text.
    // The Twitter timeline also doesn't show the RT message but the retweeted Tweet.
    if tweet.RetweetedStatus != nil {
        text = "RT @" + tweet.RetweetedStatus.User.ScreenName + ":
\n" + parseTweetText(*tweet.RetweetedStatus)
        return text
    }

    // Go through each URL object and replace it with a link and correct text
    for _, url := range tweet.Entities.Urls {
        replacement := "" + url.DisplayURL + ""
        from := url.Indices[0]
        to := url.Indices[1]
        replacements = append(replacements, replaceObject{ from, to, replacement })
    }

    // Go through each Media object and replace it the link in the text with it
    for _, media := range tweet.Entities.Media {
        var mediaUrl, replacement string
        if media.Type != "photo" {
            fmt.Println("media.Type not photo")
            spew.Dump(tweet)
            replacement = "unsupported_mediatype"
        } else if media.Type == "photo" {
            // or maybe we should use media.URLEntity.ExpandedURL?
            if len(media.MediaURLHttps) != 0 {
               mediaUrl = media.MediaURLHttps
            } else {
               mediaUrl = media.MediaURL
            }
            replacement = "

"
        }
        from := media.Indices[0]
        to := media.Indices[1]
        replacements = append(replacements, replaceObject{ from, to, replacement })
    }

    sort.Sort(replacements)

    // replacement is sorted, start from the end, since we change the length of the string
    //fmt.Println(text)
    for i := len(replacements) - 1; i >= 0; i-- {
      // cut text after character and not byte by converting the string to a rune and back to a string
      text = string([]rune(text)[:replacements[i].from]) + replacements[i].replacement + string([]rune(text)[replacements[i].to:])
      //fmt.Println(text)
    }
    //fmt.Println("---\n")

    // Does this tweet quote another tweet?
    if tweet.QuotedStatus != nil {
        quotedTweet := *tweet.QuotedStatus
        url := "https://twitter.com/" + quotedTweet.User.ScreenName + "/status/" + quotedTweet.IDStr
        header := "" + quotedTweet.User.Name + "
\n"
        quotedText := parseTweetText(quotedTweet)
        text +=  "\n
\n" + header + quotedText + "\n
\n"
    }

    return text
}

func getRss() string {
    flags := flag.NewFlagSet("user-auth", flag.ExitOnError)
    consumerKey := flags.String("consumer-key", "", "Twitter Consumer Key")
    consumerSecret := flags.String("consumer-secret", "", "Twitter Consumer Secret")
    accessToken := flags.String("access-token", "", "Twitter Access Token")
    accessSecret := flags.String("access-secret", "", "Twitter Access Secret")
    debug := flags.Bool("debug", false, "Debug")

    flags.Parse(os.Args[1:])
    flagutil.SetFlagsFromEnv(flags, "TWITTER")

    if *consumerKey == "" || *consumerSecret == "" || *accessToken == "" || *accessSecret == "" {
        log.Fatal("Consumer key/secret and Access token/secret required")
    }

    config := oauth1.NewConfig(*consumerKey, *consumerSecret)
    token := oauth1.NewToken(*accessToken, *accessSecret)
    // OAuth1 http.Client will automatically authorize Requests
    httpClient := config.Client(oauth1.NoContext, token)

    // Twitter Client
    client := twitter.NewClient(httpClient)

    now := time.Now()

    // rss feed
    feed := &feeds.Feed{
      Title:       "Twitter Home Timeline",
      Description: "Twitter Home Timeline RSS Feed",
      Author:      &feeds.Author{"Twitter2RSS", "lists.volker@gmail.com"},
      Link:        &feeds.Link{Href: "http://github.com:volker-fr/twitter2rss/"},
      Created:     now,
    }
    feed.Items = []*feeds.Item{}

    // Get timeline
    homeTimelineParams := &twitter.HomeTimelineParams{Count: 50}
    tweets, _, _ := client.Timelines.HomeTimeline(homeTimelineParams)

    /* // debugging & testing
    var tweetId int64 = 1234
    tweet, _, _ := client.Statuses.Show(tweetId, &twitter.StatusShowParams{})
    fmt.Println("https://twitter.com/" + tweet.User.ScreenName + "/status/" + tweet.IDStr)
    println(parseTweetText(*tweet))
    return "" */

    for _,tweet := range tweets {
        if *debug { spew.Dump(tweet) }

        t, _ := time.Parse(time.RubyDate, tweet.CreatedAt)

        url := "https://twitter.com/" + tweet.User.ScreenName + "/status/" + tweet.IDStr
        if *debug { fmt.Println(url) }

        item := &feeds.Item{
          Title:       fmt.Sprintf("%s: %s...", tweet.User.Name, tweet.Text[:10] ),
          Link:        &feeds.Link{Href: url},
          Description: "" + tweet.User.Name + ":
\n" + parseTweetText(tweet),
          Author:      &feeds.Author{tweet.User.Name, tweet.User.ScreenName},
          Created:     t,
          Id:          tweet.IDStr,
        }
        feed.Add(item)
    }

    // Create feed
    atom, err := feed.ToAtom()
    if err != nil {
        log.Fatal(err)
    }

    if *debug { fmt.Println(atom, "\n") }

    return atom
}

func handler(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w, "%s", getRss())
}

func main() {
    //_ = getRss()

    http.HandleFunc("/", handler)
    http.ListenAndServe("127.0.0.1:8080", nil)
}
<commit_msg>go fmt for the win && use oEmbed API to see if the output is nicer<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"sort"
	"time"

	"github.com/coreos/pkg/flagutil"
	"github.com/davecgh/go-spew/spew"
	"github.com/dghubble/go-twitter/twitter"
	"github.com/dghubble/oauth1"
	"github.com/gorilla/feeds"
)

type replaceObject struct {
	from        int
	to          int
	replacement string
}

type ReplacementList []replaceObject

func (dl ReplacementList) Len() int {
	return len(dl)
}

func (dl ReplacementList) Swap(i, j int) {
	dl[i], dl[j] = dl[j], dl[i]
}

func (dl ReplacementList) Less(i, j int) bool {
	return dl[i].from < dl[j].from
}

func getTweetUrl(tweet twitter.Tweet) string {
	return "https://twitter.com/" + tweet.User.ScreenName + "/status/" + tweet.IDStr
}

func getTweetText(id int64, client *twitter.Client) string {
	tweet, _, _ := client.Statuses.OEmbed(id, &twitter.StatusOEmbedParams{})

	return tweet.HTML
}

// Parse the text, identify twitter shortened URLs and replace them
func parseTweetText(tweet twitter.Tweet) string {
	text := tweet.Text
	var replacements ReplacementList

	// Special case, if it's retweeted then the URL placement might not be correct
	// and the tweet can also contain cut off text.
	// The Twitter timeline also doesn't show the RT message but the retweeted Tweet.
	if tweet.RetweetedStatus != nil {
		text = parseTweetText(*tweet.RetweetedStatus)
		return "" + tweet.User.Name + ": RT @" + tweet.RetweetedStatus.User.ScreenName + ":
\n" + text
	}

	// Go through each URL object and replace it with a link and correct text
	for _, url := range tweet.Entities.Urls {
		replacement := "" + url.DisplayURL + ""
		from := url.Indices[0]
		to := url.Indices[1]
		replacements = append(replacements, replaceObject{from, to, replacement})
	}

	// Go through each Media object and replace it the link in the text with it
	for _, media := range tweet.Entities.Media {
		var mediaUrl, replacement string
		if media.Type != "photo" {
			fmt.Println("media.Type not photo")
			spew.Dump(tweet)
			replacement = "unsupported_mediatype"
		} else if media.Type == "photo" {
			// or maybe we should use media.URLEntity.ExpandedURL?
			if len(media.MediaURLHttps) != 0 {
				mediaUrl = media.MediaURLHttps
			} else {
				mediaUrl = media.MediaURL
			}
			replacement = "

"
		}
		from := media.Indices[0]
		to := media.Indices[1]
		replacements = append(replacements, replaceObject{from, to, replacement})
	}

	sort.Sort(replacements)

	// replacement is sorted, start from the end, since we change the length of the string
	//fmt.Println(text)
	for i := len(replacements) - 1; i >= 0; i-- {
		// cut text after character and not byte by converting the string to a rune and back to a string
		text = string([]rune(text)[:replacements[i].from]) + replacements[i].replacement + string([]rune(text)[replacements[i].to:])
		//fmt.Println(text)
	}
	//fmt.Println("---\n")

	// Does this tweet quote another tweet?
	if tweet.QuotedStatus != nil {
		quotedTweet := *tweet.QuotedStatus
		header := "" + quotedTweet.User.Name + "
\n"
		quotedText := parseTweetText(quotedTweet)
		text += "\n
\n" + header + quotedText + "\n
\n"
	}

	return "" + tweet.User.Name + ":
\n" + text
}

func getRss() string {
	flags := flag.NewFlagSet("user-auth", flag.ExitOnError)
	consumerKey := flags.String("consumer-key", "", "Twitter Consumer Key")
	consumerSecret := flags.String("consumer-secret", "", "Twitter Consumer Secret")
	accessToken := flags.String("access-token", "", "Twitter Access Token")
	accessSecret := flags.String("access-secret", "", "Twitter Access Secret")
	debug := flags.Bool("debug", false, "Debug")

	flags.Parse(os.Args[1:])
	flagutil.SetFlagsFromEnv(flags, "TWITTER")

	if *consumerKey == "" || *consumerSecret == "" || *accessToken == "" || *accessSecret == "" {
		log.Fatal("Consumer key/secret and Access token/secret required")
	}

	config := oauth1.NewConfig(*consumerKey, *consumerSecret)
	token := oauth1.NewToken(*accessToken, *accessSecret)
	// OAuth1 http.Client will automatically authorize Requests
	httpClient := config.Client(oauth1.NoContext, token)

	// Twitter Client
	client := twitter.NewClient(httpClient)

	now := time.Now()

	// rss feed
	feed := &feeds.Feed{
		Title:       "Twitter Home Timeline",
		Description: "Twitter Home Timeline RSS Feed",
		Author:      &feeds.Author{"Twitter2RSS", "lists.volker@gmail.com"},
		Link:        &feeds.Link{Href: "http://github.com:volker-fr/twitter2rss/"},
		Created:     now,
	}
	feed.Items = []*feeds.Item{}

	// Get timeline
	homeTimelineParams := &twitter.HomeTimelineParams{Count: 50}
	tweets, _, _ := client.Timelines.HomeTimeline(homeTimelineParams)

	/* // debugging & testing
		var tweetId int64 = 1234
		tweet, _, _ := client.Statuses.OEmbed(tweetId, &twitter.StatusOEmbedParams{})
		spew.Dump(tweet)
	    //tweet, _, _ := client.Statuses.Show(tweetId, &twitter.StatusShowParams{})
		//fmt.Println("https://twitter.com/" + tweet.User.ScreenName + "/status/" + tweet.IDStr)
	    //println(parseTweetText(*tweet))
		return "" */

	for _, tweet := range tweets {
		if *debug {
			spew.Dump(tweet)
		}

		t, _ := time.Parse(time.RubyDate, tweet.CreatedAt)

		item := &feeds.Item{
			Title: fmt.Sprintf("%s: %s...", tweet.User.Name, tweet.Text[:10]),
			Link:  &feeds.Link{Href: getTweetUrl(tweet)},
			//Description: parseTweetText(tweet),
			Description: getTweetText(tweet.ID, client),
			Author:      &feeds.Author{tweet.User.Name, tweet.User.ScreenName},
			Created:     t,
			Id:          tweet.IDStr,
		}
		feed.Add(item)
	}

	// Create feed
	atom, err := feed.ToAtom()
	if err != nil {
		log.Fatal(err)
	}

	if *debug {
		fmt.Println(atom, "\n")
	}

	return atom
}

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "%s", getRss())
}

func main() {
	//_ = getRss()

	http.HandleFunc("/", handler)
	http.ListenAndServe("127.0.0.1:8080", nil)
}
<|endoftext|>


<commit_before>package integration

import (
	"encoding/json"
	"fmt"
	"github.com/astaxie/beego"
	"github.com/astaxie/beego/orm"
	"github.com/robxu9/kahinah/models"
	"github.com/robxu9/kahinah/util"
	"io/ioutil"
	"log"
	"menteslibres.net/gosexy/dig"
	"menteslibres.net/gosexy/to"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"
)

const (
	ABF_URL = "https://abf.io/api/v1"
)

var (
	user        = beego.AppConfig.String("abf::abf_user")
	pass        = beego.AppConfig.String("abf::abf_pass")
	platforms   *util.Set
	platformids *util.Set
	client      = &http.Client{}
)

func init() {
	platforms = util.NewSet()
	platformids = util.NewSet()
	for _, v := range strings.Split(beego.AppConfig.String("abf::abf_platforms"), ";") {
		platform := strings.Split(v, ":")
		platforms.Add(platform[0])
		platformids.Add(platform[1])
	}
}

type ABF byte

func (a ABF) Ping() error {

	for v := range platformids.Iterator() {
		go a.pingBuildCompleted(v)
		go a.pingTestingBuilds(v)
	}

	// FUTURE TODO: ping published & rejected builds to ensure consistency

	//if err != nil && err2 != nil {
	//	return fmt.Errorf("abf: two errors: %s | %s", err, err2)
	//} else if err != nil {
	//	return err
	//} else if err2 != nil {
	//	return err2
	//}

	return nil
}

func (a ABF) pingBuildCompleted(platformId string) error {
	// regular usage: use 0 (Build has been completed)
	// below: use 12000 ([testing] build has been published)
	// FIXME - remove hardcoded openmandriva2013.0 filter (but find alternative, we need said filter)
	req, err := http.NewRequest("GET", ABF_URL+"/build_lists.json?per_page=100&filter[status]=0&filter[ownership]=index&filter[platform_id]="+platformId, nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	var result map[string]interface{}

	err = json.Unmarshal(body, &result)
	if err != nil {
		return err
	}

	var lists []interface{}

	err = dig.Get(&result, &lists, "build_lists")
	if err != nil {
		return err
	}

	o := orm.NewOrm()

	for _, v := range lists {
		asserted := v.(map[string]interface{})
		id := dig.Uint64(&asserted, "id")

		num, err := o.QueryTable(new(models.BuildList)).Filter("HandleId", id).Count()
		if num <= 0 || err != nil {
			list, err := a.getBuildList(id)
			if err != nil {
				log.Printf("abf: Error retrieving build list %s: %s\n", id, err)
			}

			if !platforms.Contains(list.Platform) {
				// ignore
				continue
			}

			// Now send it to testing
			go a.sendToTesting(id)

			_, err = o.Insert(list)
			if err != nil {
				log.Printf("abf: Error saving build list %s: %s\n", id, err)
			}

			for _, listpkg := range list.Packages {
				listpkg.List = list
				o.Insert(listpkg)
			}
		}
	}

	return nil
}

func (a ABF) pingTestingBuilds(platformId string) error {
	// FIXME - remove hardcoded openmandriva2013.0 filter (but find alternative, we need said filter)
	req, err := http.NewRequest("GET", ABF_URL+"/build_lists.json?per_page=100&filter[status]=12000&filter[ownership]=index&filter[platform_id]="+platformId, nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	var result map[string]interface{}

	err = json.Unmarshal(body, &result)
	if err != nil {
		return err
	}

	var lists []interface{}

	err = dig.Get(&result, &lists, "build_lists")
	if err != nil {
		return err
	}

	o := orm.NewOrm()

	for _, v := range lists {
		asserted := v.(map[string]interface{})
		id := dig.Uint64(&asserted, "id")

		num, err := o.QueryTable(new(models.BuildList)).Filter("HandleId", id).Count()
		if num <= 0 || err != nil {
			list, err := a.getBuildList(id)
			if err != nil {
				log.Printf("abf: Error retrieving build list %s: %s\n", id, err)
			}

			if !platforms.Contains(list.Platform) {
				// ignore
				continue
			}

			_, err = o.Insert(list)
			if err != nil {
				log.Printf("abf: Error saving build list %s: %s\n", id, err)
			}

			for _, listpkg := range list.Packages {
				listpkg.List = list
				o.Insert(listpkg)
			}
		}
	}

	return nil
}

func (a ABF) PingParams(m map[string]string) error {
	return fmt.Errorf("abf: PingParams not supported yet.")
}

func (a ABF) Commits(m *models.BuildList) string {
	return "https://abf.io/" + m.HandleProject + "/commits/" + m.Platform
}

func (a ABF) Url(m *models.BuildList) string {
	return "https://abf.io/build_lists/" + m.HandleId
}

func (a ABF) Publish(m *models.BuildList) error {
	id := to.Uint64(m.HandleId)
	req, err := http.NewRequest("PUT", ABF_URL+"/build_lists/"+to.String(id)+"/publish.json", nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)
	req.Header.Add("Content-Length", "0")

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	return nil
}

func (a ABF) Reject(m *models.BuildList) error {
	id := to.Uint64(m.HandleId)
	req, err := http.NewRequest("PUT", ABF_URL+"/build_lists/"+to.String(id)+"/reject_publish.json", nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)
	req.Header.Add("Content-Length", "0")

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	return nil
}

func (a ABF) sendToTesting(id uint64) error {
	req, err := http.NewRequest("PUT", ABF_URL+"/build_lists/"+to.String(id)+"/publish_into_testing.json", nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)
	req.Header.Add("Content-Length", "0")

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	return nil
}

func (a ABF) getBuildList(id uint64) (*models.BuildList, error) {
	req, err := http.NewRequest("GET", ABF_URL+"/build_lists/"+to.String(id)+".json", nil)
	if err != nil {
		return nil, err
	}

	req.SetBasicAuth(user, pass)

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var result map[string]interface{}

	err = json.Unmarshal(body, &result)
	if err != nil {
		return nil, err
	}

	var list map[string]interface{}

	dig.Get(&result, &list, "build_list")

	pkgs := make([]interface{}, 0)
	dig.Get(&list, &pkgs, "packages")

	pkg := make([]*models.BuildListPkg, 0)

	pkgrep := func(m map[string]interface{}) *models.BuildListPkg {
		pkgType := dig.String(&m, "type")
		if strings.HasSuffix(dig.String(&m, "name"), "-debuginfo") {
			pkgType = "debuginfo"
		}

		return &models.BuildListPkg{
			Name:    dig.String(&m, "name"),
			Type:    pkgType,
			Epoch:   dig.Int64(&m, "epoch"),
			Version: dig.String(&m, "version"),
			Release: dig.String(&m, "release"),
			Url:     dig.String(&m, "url"),
		}
	}

	for _, v := range pkgs {
		asserted := v.(map[string]interface{})
		pkg = append(pkg, pkgrep(asserted))
	}

	changelog := ""

	logs := make([]interface{}, 0)
	dig.Get(&list, &logs, "logs")

	for _, v := range logs {
		asserted := v.(map[string]interface{})
		if dig.String(&asserted, "file_name") == "changelog.log" {
			changelog = dig.String(&asserted, "url")
			break
		}
	}

	user := a.getUser(dig.Uint64(&list, "user", "id"))

	bl := models.BuildList{
		HandleId:      to.String(dig.Uint64(&list, "id")),
		HandleProject: dig.String(&list, "project", "fullname"),
		Diff:          a.getDiff(dig.String(&list, "project", "git_url"), dig.String(&list, "last_published_commit_hash"), dig.String(&list, "commit_hash")),

		//Platform:     dig.String(&list, "build_for_platform", "name"),
		Platform:     dig.String(&list, "save_to_repository", "platform", "name"),
		Repo:         dig.String(&list, "save_to_repository", "name"),
		Architecture: dig.String(&list, "arch", "name"),
		Name:         dig.String(&list, "project", "name"),
		Submitter:    user,
		Type:         dig.String(&list, "update_type"),
		Status:       models.STATUS_TESTING,
		Changelog:    changelog, // url
		Packages:     pkg,
		BuildDate:    time.Unix(dig.Int64(&list, "updated_at"), 0),
	}

	return &bl, nil

}

func (a ABF) getDiff(gitUrl, fromHash, toHash string) string {
	// ugh, looks like we'll have to do this the sadly hard way
	tmpdir, err := ioutil.TempDir("", "kahinah_")
	if err != nil {
		return "Error creating directory for diff creation: " + err.Error()
	}

	defer os.RemoveAll(tmpdir)

	gitresult := exec.Command("git", "clone", gitUrl, tmpdir).Run()
	if gitresult != nil { // git better exit with status zero
		return "Repository could not be cloned for diff: " + gitresult.Error()
	}

	if fromHash == "" {
		gitdiffcmd := exec.Command("git", "show", "--format=fuller", "--patch-with-stat", "--summary", toHash)
		gitdiffcmd.Dir = tmpdir

		gitdiff, err := gitdiffcmd.Output()
		if err != nil {
			return "No diff available: " + err.Error()
		}

		return fmt.Sprintf("$ git show --format=fuller --patch-with-stat --summary %s\n\n%s", toHash, string(gitdiff))
	} else {
		gitdiffcmd := exec.Command("git", "diff", "--patch-with-stat", "--summary", fromHash+".."+toHash)
		gitdiffcmd.Dir = tmpdir

		gitdiff, err := gitdiffcmd.Output()
		if err != nil {
			return "No diff available: " + err.Error()
		}

		return fmt.Sprintf("$ git diff --patch-with-stat --summary %s\n\n%s", fromHash+".."+toHash, string(gitdiff))
	}
}

func (a ABF) getUser(id uint64) *models.User {

	o := orm.NewOrm()

	var userModel *models.User

	var userIntegration models.User
	err := o.QueryTable(new(models.User)).Filter("Integration", id).One(&userIntegration)
	if err == orm.ErrNoRows {

		req, err := http.NewRequest("GET", ABF_URL+"/users/"+to.String(id)+".json", nil)
		if err != nil {
			return nil
		}

		req.SetBasicAuth(user, pass)

		resp, err := client.Do(req)
		if err != nil {
			return nil
		}

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil
		}

		var result map[string]interface{}

		err = json.Unmarshal(body, &result)
		if err != nil {
			return nil
		}

		email := dig.String(&result, "user", "email")

		userModel = models.FindUser(email)
		userModel.Integration = to.String(id)
		userModel.Save()

	} else if err != nil {
		panic(err)
	} else {
		userModel = &userIntegration
	}

	return userModel
}
<commit_msg>git 1.7.1 asks for password(?!?!) so remove any user we don't want your password prompts<commit_after>package integration

import (
	"encoding/json"
	"fmt"
	"github.com/astaxie/beego"
	"github.com/astaxie/beego/orm"
	"github.com/robxu9/kahinah/models"
	"github.com/robxu9/kahinah/util"
	"io/ioutil"
	"log"
	"menteslibres.net/gosexy/dig"
	"menteslibres.net/gosexy/to"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"
)

const (
	ABF_URL = "https://abf.io/api/v1"
)

var (
	user        = beego.AppConfig.String("abf::abf_user")
	pass        = beego.AppConfig.String("abf::abf_pass")
	platforms   *util.Set
	platformids *util.Set
	client      = &http.Client{}
)

func init() {
	platforms = util.NewSet()
	platformids = util.NewSet()
	for _, v := range strings.Split(beego.AppConfig.String("abf::abf_platforms"), ";") {
		platform := strings.Split(v, ":")
		platforms.Add(platform[0])
		platformids.Add(platform[1])
	}
}

type ABF byte

func (a ABF) Ping() error {

	for v := range platformids.Iterator() {
		go a.pingBuildCompleted(v)
		go a.pingTestingBuilds(v)
	}

	// FUTURE TODO: ping published & rejected builds to ensure consistency

	//if err != nil && err2 != nil {
	//	return fmt.Errorf("abf: two errors: %s | %s", err, err2)
	//} else if err != nil {
	//	return err
	//} else if err2 != nil {
	//	return err2
	//}

	return nil
}

func (a ABF) pingBuildCompleted(platformId string) error {
	// regular usage: use 0 (Build has been completed)
	// below: use 12000 ([testing] build has been published)
	// FIXME - remove hardcoded openmandriva2013.0 filter (but find alternative, we need said filter)
	req, err := http.NewRequest("GET", ABF_URL+"/build_lists.json?per_page=100&filter[status]=0&filter[ownership]=index&filter[platform_id]="+platformId, nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	var result map[string]interface{}

	err = json.Unmarshal(body, &result)
	if err != nil {
		return err
	}

	var lists []interface{}

	err = dig.Get(&result, &lists, "build_lists")
	if err != nil {
		return err
	}

	o := orm.NewOrm()

	for _, v := range lists {
		asserted := v.(map[string]interface{})
		id := dig.Uint64(&asserted, "id")

		num, err := o.QueryTable(new(models.BuildList)).Filter("HandleId", id).Count()
		if num <= 0 || err != nil {
			list, err := a.getBuildList(id)
			if err != nil {
				log.Printf("abf: Error retrieving build list %s: %s\n", id, err)
			}

			if !platforms.Contains(list.Platform) {
				// ignore
				continue
			}

			// Now send it to testing
			go a.sendToTesting(id)

			_, err = o.Insert(list)
			if err != nil {
				log.Printf("abf: Error saving build list %s: %s\n", id, err)
			}

			for _, listpkg := range list.Packages {
				listpkg.List = list
				o.Insert(listpkg)
			}
		}
	}

	return nil
}

func (a ABF) pingTestingBuilds(platformId string) error {
	// FIXME - remove hardcoded openmandriva2013.0 filter (but find alternative, we need said filter)
	req, err := http.NewRequest("GET", ABF_URL+"/build_lists.json?per_page=100&filter[status]=12000&filter[ownership]=index&filter[platform_id]="+platformId, nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	var result map[string]interface{}

	err = json.Unmarshal(body, &result)
	if err != nil {
		return err
	}

	var lists []interface{}

	err = dig.Get(&result, &lists, "build_lists")
	if err != nil {
		return err
	}

	o := orm.NewOrm()

	for _, v := range lists {
		asserted := v.(map[string]interface{})
		id := dig.Uint64(&asserted, "id")

		num, err := o.QueryTable(new(models.BuildList)).Filter("HandleId", id).Count()
		if num <= 0 || err != nil {
			list, err := a.getBuildList(id)
			if err != nil {
				log.Printf("abf: Error retrieving build list %s: %s\n", id, err)
			}

			if !platforms.Contains(list.Platform) {
				// ignore
				continue
			}

			_, err = o.Insert(list)
			if err != nil {
				log.Printf("abf: Error saving build list %s: %s\n", id, err)
			}

			for _, listpkg := range list.Packages {
				listpkg.List = list
				o.Insert(listpkg)
			}
		}
	}

	return nil
}

func (a ABF) PingParams(m map[string]string) error {
	return fmt.Errorf("abf: PingParams not supported yet.")
}

func (a ABF) Commits(m *models.BuildList) string {
	return "https://abf.io/" + m.HandleProject + "/commits/" + m.Platform
}

func (a ABF) Url(m *models.BuildList) string {
	return "https://abf.io/build_lists/" + m.HandleId
}

func (a ABF) Publish(m *models.BuildList) error {
	id := to.Uint64(m.HandleId)
	req, err := http.NewRequest("PUT", ABF_URL+"/build_lists/"+to.String(id)+"/publish.json", nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)
	req.Header.Add("Content-Length", "0")

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	return nil
}

func (a ABF) Reject(m *models.BuildList) error {
	id := to.Uint64(m.HandleId)
	req, err := http.NewRequest("PUT", ABF_URL+"/build_lists/"+to.String(id)+"/reject_publish.json", nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)
	req.Header.Add("Content-Length", "0")

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	return nil
}

func (a ABF) sendToTesting(id uint64) error {
	req, err := http.NewRequest("PUT", ABF_URL+"/build_lists/"+to.String(id)+"/publish_into_testing.json", nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)
	req.Header.Add("Content-Length", "0")

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	return nil
}

func (a ABF) getBuildList(id uint64) (*models.BuildList, error) {
	req, err := http.NewRequest("GET", ABF_URL+"/build_lists/"+to.String(id)+".json", nil)
	if err != nil {
		return nil, err
	}

	req.SetBasicAuth(user, pass)

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var result map[string]interface{}

	err = json.Unmarshal(body, &result)
	if err != nil {
		return nil, err
	}

	var list map[string]interface{}

	dig.Get(&result, &list, "build_list")

	pkgs := make([]interface{}, 0)
	dig.Get(&list, &pkgs, "packages")

	pkg := make([]*models.BuildListPkg, 0)

	pkgrep := func(m map[string]interface{}) *models.BuildListPkg {
		pkgType := dig.String(&m, "type")
		if strings.HasSuffix(dig.String(&m, "name"), "-debuginfo") {
			pkgType = "debuginfo"
		}

		return &models.BuildListPkg{
			Name:    dig.String(&m, "name"),
			Type:    pkgType,
			Epoch:   dig.Int64(&m, "epoch"),
			Version: dig.String(&m, "version"),
			Release: dig.String(&m, "release"),
			Url:     dig.String(&m, "url"),
		}
	}

	for _, v := range pkgs {
		asserted := v.(map[string]interface{})
		pkg = append(pkg, pkgrep(asserted))
	}

	changelog := ""

	logs := make([]interface{}, 0)
	dig.Get(&list, &logs, "logs")

	for _, v := range logs {
		asserted := v.(map[string]interface{})
		if dig.String(&asserted, "file_name") == "changelog.log" {
			changelog = dig.String(&asserted, "url")
			break
		}
	}

	user := a.getUser(dig.Uint64(&list, "user", "id"))

	bl := models.BuildList{
		HandleId:      to.String(dig.Uint64(&list, "id")),
		HandleProject: dig.String(&list, "project", "fullname"),
		Diff:          a.getDiff(dig.String(&list, "project", "git_url"), dig.String(&list, "last_published_commit_hash"), dig.String(&list, "commit_hash")),

		//Platform:     dig.String(&list, "build_for_platform", "name"),
		Platform:     dig.String(&list, "save_to_repository", "platform", "name"),
		Repo:         dig.String(&list, "save_to_repository", "name"),
		Architecture: dig.String(&list, "arch", "name"),
		Name:         dig.String(&list, "project", "name"),
		Submitter:    user,
		Type:         dig.String(&list, "update_type"),
		Status:       models.STATUS_TESTING,
		Changelog:    changelog, // url
		Packages:     pkg,
		BuildDate:    time.Unix(dig.Int64(&list, "updated_at"), 0),
	}

	return &bl, nil

}

func (a ABF) getDiff(gitUrl, fromHash, toHash string) string {
	// ugh, looks like we'll have to do this the sadly hard way
	tmpdir, err := ioutil.TempDir("", "kahinah_")
	if err != nil {
		return "Error creating directory for diff creation: " + err.Error()
	}

	defer os.RemoveAll(tmpdir)

	if strings.Contains(gitUrl, "@") {
		gitUrl = gitUrl[:strings.Index(gitUrl, "//")+2] + gitUrl[strings.Index(gitUrl, "@")+1:]
	}

	gitresult := exec.Command("git", "clone", gitUrl, tmpdir).Run()
	if gitresult != nil { // git better exit with status zero
		return "Repository could not be cloned for diff: " + gitresult.Error()
	}

	if fromHash == "" {
		gitdiffcmd := exec.Command("git", "show", "--format=fuller", "--patch-with-stat", "--summary", toHash)
		gitdiffcmd.Dir = tmpdir

		gitdiff, err := gitdiffcmd.Output()
		if err != nil {
			return "No diff available: " + err.Error()
		}

		return fmt.Sprintf("$ git show --format=fuller --patch-with-stat --summary %s\n\n%s", toHash, string(gitdiff))
	} else {
		gitdiffcmd := exec.Command("git", "diff", "--patch-with-stat", "--summary", fromHash+".."+toHash)
		gitdiffcmd.Dir = tmpdir

		gitdiff, err := gitdiffcmd.Output()
		if err != nil {
			return "No diff available: " + err.Error()
		}

		return fmt.Sprintf("$ git diff --patch-with-stat --summary %s\n\n%s", fromHash+".."+toHash, string(gitdiff))
	}
}

func (a ABF) getUser(id uint64) *models.User {

	o := orm.NewOrm()

	var userModel *models.User

	var userIntegration models.User
	err := o.QueryTable(new(models.User)).Filter("Integration", id).One(&userIntegration)
	if err == orm.ErrNoRows {

		req, err := http.NewRequest("GET", ABF_URL+"/users/"+to.String(id)+".json", nil)
		if err != nil {
			return nil
		}

		req.SetBasicAuth(user, pass)

		resp, err := client.Do(req)
		if err != nil {
			return nil
		}

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil
		}

		var result map[string]interface{}

		err = json.Unmarshal(body, &result)
		if err != nil {
			return nil
		}

		email := dig.String(&result, "user", "email")

		userModel = models.FindUser(email)
		userModel.Integration = to.String(id)
		userModel.Save()

	} else if err != nil {
		panic(err)
	} else {
		userModel = &userIntegration
	}

	return userModel
}
<|endoftext|>


<commit_before>// Copyright 2013-2014 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import (
	"errors"
	"strings"
)

// Aerospike error implements error interface for aerospike specific errors
type AerospikeError struct {
	error

	resultCode ResultCode
}

func (ase *AerospikeError) ResultCode() ResultCode {
	return ase.resultCode
}

// Error Types
type ErrInvalidNode struct{ AerospikeError }
type ErrTimeout struct{ AerospikeError }
type ErrSerialization struct{ AerospikeError }
type ErrParse struct{ AerospikeError }
type ErrConnection struct{ AerospikeError }
type ErrQueryTerminated struct{ AerospikeError }
type ErrScanTerminated struct{ AerospikeError }
type ErrCommandRejected struct{ AerospikeError }
type ErrTypeNotSupported struct{ AerospikeError }

////////////////////////////////////////////////////////////////////////////////
// Generator for Aerospike errors.
// If no message is provided, the result code will be translated into the default
// error message
func NewAerospikeError(code ResultCode, messages ...string) (err error) {
	if len(messages) == 0 {
		messages = []string{ResultCodeToString(code)}
	}

	errMsg := errors.New(strings.Join(messages, " "))

	switch ResultCode(code) {
	case TYPE_NOT_SUPPORTED:
		err = ErrTypeNotSupported{AerospikeError{error: errMsg, resultCode: code}}

	case COMMAND_REJECTED:
		err = ErrCommandRejected{AerospikeError{error: errMsg, resultCode: code}}

	case QUERY_TERMINATED:
		err = ErrQueryTerminated{AerospikeError{error: errMsg, resultCode: code}}

	case SCAN_TERMINATED:
		err = ErrScanTerminated{AerospikeError{error: errMsg, resultCode: code}}

	case INVALID_NODE_ERROR:
		err = ErrInvalidNode{AerospikeError{error: errMsg, resultCode: code}}

	case PARSE_ERROR:
		err = ErrParse{AerospikeError{error: errMsg, resultCode: code}}

	case SERIALIZE_ERROR:
		err = ErrSerialization{AerospikeError{error: errMsg, resultCode: code}}

	case OK:
		err = AerospikeError{error: errMsg, resultCode: code}

	case SERVER_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case KEY_NOT_FOUND_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case GENERATION_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case PARAMETER_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case KEY_EXISTS_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case BIN_EXISTS_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case CLUSTER_KEY_MISMATCH:
		err = AerospikeError{error: errMsg, resultCode: code}

	case SERVER_MEM_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case TIMEOUT:
		err = ErrTimeout{AerospikeError{error: errMsg, resultCode: code}}

	case NO_XDS:
		err = AerospikeError{error: errMsg, resultCode: code}

	case SERVER_NOT_AVAILABLE:
		err = ErrConnection{AerospikeError{error: errMsg, resultCode: code}}

	case BIN_TYPE_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case RECORD_TOO_BIG:
		err = AerospikeError{error: errMsg, resultCode: code}

	case KEY_BUSY:
		err = AerospikeError{error: errMsg, resultCode: code}

	case SCAN_ABORT:
		err = AerospikeError{error: errMsg, resultCode: code}

	case UNSUPPORTED_FEATURE:
		err = AerospikeError{error: errMsg, resultCode: code}

	case BIN_NOT_FOUND:
		err = AerospikeError{error: errMsg, resultCode: code}

	case DEVICE_OVERLOAD:
		err = AerospikeError{error: errMsg, resultCode: code}

	case KEY_MISMATCH:
		err = AerospikeError{error: errMsg, resultCode: code}

	case UDF_BAD_RESPONSE:
		err = AerospikeError{error: errMsg, resultCode: code}

	case INDEX_FOUND:
		err = AerospikeError{error: errMsg, resultCode: code}

	case INDEX_NOTFOUND:
		err = AerospikeError{error: errMsg, resultCode: code}

	case INDEX_OOM:
		err = AerospikeError{error: errMsg, resultCode: code}

	case INDEX_NOTREADABLE:
		err = AerospikeError{error: errMsg, resultCode: code}

	case INDEX_GENERIC:
		err = AerospikeError{error: errMsg, resultCode: code}

	case INDEX_NAME_MAXLEN:
		err = AerospikeError{error: errMsg, resultCode: code}

	case INDEX_MAXCOUNT:
		err = AerospikeError{error: errMsg, resultCode: code}

	case QUERY_ABORTED:
		err = AerospikeError{error: errMsg, resultCode: code}

	case QUERY_QUEUEFULL:
		err = AerospikeError{error: errMsg, resultCode: code}

	case QUERY_TIMEOUT:
		err = AerospikeError{error: errMsg, resultCode: code}

	case QUERY_GENERIC:
		err = AerospikeError{error: errMsg, resultCode: code}

	default:
		err = AerospikeError{error: errMsg, resultCode: code}
	}

	return err
}
<commit_msg>remove specialized error types<commit_after>// Copyright 2013-2014 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import (
	"errors"
	"strings"
)

// Aerospike error implements error interface for aerospike specific errors
type AerospikeError struct {
	error

	resultCode ResultCode
}

// ResultCode returns the ResultCode from AerospikeError object
func (ase AerospikeError) ResultCode() ResultCode {
	return ase.resultCode
}

////////////////////////////////////////////////////////////////////////////////
// Generator for Aerospike errors.
// If no message is provided, the result code will be translated into the default
// error message
func NewAerospikeError(code ResultCode, messages ...string) (err error) {
	if len(messages) == 0 {
		messages = []string{ResultCodeToString(code)}
	}

	errMsg := errors.New(strings.Join(messages, " "))

	switch ResultCode(code) {
	case TYPE_NOT_SUPPORTED:
		err = AerospikeError{error: errMsg, resultCode: code}

	case COMMAND_REJECTED:
		err = AerospikeError{error: errMsg, resultCode: code}

	case QUERY_TERMINATED:
		err = AerospikeError{error: errMsg, resultCode: code}

	case SCAN_TERMINATED:
		err = AerospikeError{error: errMsg, resultCode: code}

	case INVALID_NODE_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case PARSE_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case SERIALIZE_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case OK:
		err = AerospikeError{error: errMsg, resultCode: code}

	case SERVER_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case KEY_NOT_FOUND_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case GENERATION_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case PARAMETER_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case KEY_EXISTS_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case BIN_EXISTS_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case CLUSTER_KEY_MISMATCH:
		err = AerospikeError{error: errMsg, resultCode: code}

	case SERVER_MEM_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case TIMEOUT:
		err = AerospikeError{error: errMsg, resultCode: code}

	case NO_XDS:
		err = AerospikeError{error: errMsg, resultCode: code}

	case SERVER_NOT_AVAILABLE:
		err = AerospikeError{error: errMsg, resultCode: code}

	case BIN_TYPE_ERROR:
		err = AerospikeError{error: errMsg, resultCode: code}

	case RECORD_TOO_BIG:
		err = AerospikeError{error: errMsg, resultCode: code}

	case KEY_BUSY:
		err = AerospikeError{error: errMsg, resultCode: code}

	case SCAN_ABORT:
		err = AerospikeError{error: errMsg, resultCode: code}

	case UNSUPPORTED_FEATURE:
		err = AerospikeError{error: errMsg, resultCode: code}

	case BIN_NOT_FOUND:
		err = AerospikeError{error: errMsg, resultCode: code}

	case DEVICE_OVERLOAD:
		err = AerospikeError{error: errMsg, resultCode: code}

	case KEY_MISMATCH:
		err = AerospikeError{error: errMsg, resultCode: code}

	case UDF_BAD_RESPONSE:
		err = AerospikeError{error: errMsg, resultCode: code}

	case INDEX_FOUND:
		err = AerospikeError{error: errMsg, resultCode: code}

	case INDEX_NOTFOUND:
		err = AerospikeError{error: errMsg, resultCode: code}

	case INDEX_OOM:
		err = AerospikeError{error: errMsg, resultCode: code}

	case INDEX_NOTREADABLE:
		err = AerospikeError{error: errMsg, resultCode: code}

	case INDEX_GENERIC:
		err = AerospikeError{error: errMsg, resultCode: code}

	case INDEX_NAME_MAXLEN:
		err = AerospikeError{error: errMsg, resultCode: code}

	case INDEX_MAXCOUNT:
		err = AerospikeError{error: errMsg, resultCode: code}

	case QUERY_ABORTED:
		err = AerospikeError{error: errMsg, resultCode: code}

	case QUERY_QUEUEFULL:
		err = AerospikeError{error: errMsg, resultCode: code}

	case QUERY_TIMEOUT:
		err = AerospikeError{error: errMsg, resultCode: code}

	case QUERY_GENERIC:
		err = AerospikeError{error: errMsg, resultCode: code}

	default:
		err = AerospikeError{error: errMsg, resultCode: code}
	}

	return err
}
<|endoftext|>


<commit_before>package types

import (
	"bytes"
	"compress/gzip"
	"database/sql/driver"
	"encoding/json"
	"errors"

	"io/ioutil"
)

// GzippedText is a []byte which transparently gzips data being submitted to
// a database and ungzips data being Scanned from a database.
type GzippedText []byte

// Value implements the driver.Valuer interface, gzipping the raw value of
// this GzippedText.
func (g GzippedText) Value() (driver.Value, error) {
	b := make([]byte, 0, len(g))
	buf := bytes.NewBuffer(b)
	w := gzip.NewWriter(buf)
	w.Write(g)
	w.Close()
	return buf.Bytes(), nil

}

// Scan implements the sql.Scanner interface, ungzipping the value coming off
// the wire and storing the raw result in the GzippedText.
func (g *GzippedText) Scan(src interface{}) error {
	var source []byte
	switch src.(type) {
	case string:
		source = []byte(src.(string))
	case []byte:
		source = src.([]byte)
	default:
		return errors.New("Incompatible type for GzippedText")
	}
	reader, err := gzip.NewReader(bytes.NewReader(source))
	if err != nil {
		return err
	}
	defer reader.Close()
	b, err := ioutil.ReadAll(reader)
	if err != nil {
		return err
	}
	*g = GzippedText(b)
	return nil
}

// JSONText is a json.RawMessage, which is a []byte underneath.
// Value() validates the json format in the source, and returns an error if
// the json is not valid.  Scan does no validation.  JSONText additionally
// implements `Unmarshal`, which unmarshals the json within to an interface{}
type JSONText json.RawMessage

var _EMPTY_JSON = JSONText("{}")

// MarshalJSON returns the *j as the JSON encoding of j.
func (j *JSONText) MarshalJSON() ([]byte, error) {
	if len(*j) == 0 {
		*j = _EMPTY_JSON
	}
	return *j, nil
}

// UnmarshalJSON sets *j to a copy of data
func (j *JSONText) UnmarshalJSON(data []byte) error {
	if j == nil {
		return errors.New("JSONText: UnmarshalJSON on nil pointer")
	}
	*j = append((*j)[0:0], data...)
	return nil

}

// Value returns j as a value.  This does a validating unmarshal into another
// RawMessage.  If j is invalid json, it returns an error.
func (j JSONText) Value() (driver.Value, error) {
	var m json.RawMessage
	var err = j.Unmarshal(&m)
	if err != nil {
		return []byte{}, err
	}
	return []byte(j), nil
}

// Scan stores the src in *j.  No validation is done.
func (j *JSONText) Scan(src interface{}) error {
	var source []byte
	switch t := src.(type) {
	case string:
		source = []byte(t)
	case []byte:
		if len(t) == 0 {
			source = _EMPTY_JSON
		} else {
			source = t
		}
	case nil:
		*j = _EMPTY_JSON
	default:
		return errors.New("Incompatible type for JSONText")
	}
	*j = JSONText(append((*j)[0:0], source...))
	return nil
}

// Unmarshal unmarshal's the json in j to v, as in json.Unmarshal.
func (j *JSONText) Unmarshal(v interface{}) error {
	if len(*j) == 0 {
		*j = _EMPTY_JSON
	}
	return json.Unmarshal([]byte(*j), v)
}

// Pretty printing for JSONText types
func (j JSONText) String() string {
	return string(j)
}

// BitBool is an implementation of a bool for the MySQL type BIT(1).
// This type allows you to avoid wasting an entire byte for MySQL's boolean type TINYINT.
type BitBool bool

// Value implements the driver.Valuer interface,
// and turns the BitBool into a bitfield (BIT(1)) for MySQL storage.
func (b BitBool) Value() (driver.Value, error) {
	if b {
		return []byte{1}, nil
	}
	return []byte{0}, nil
}

// Scan implements the sql.Scanner interface,
// and turns the bitfield incoming from MySQL into a BitBool
func (b *BitBool) Scan(src interface{}) error {
	v, ok := src.([]byte)
	if !ok {
		return errors.New("bad []byte type assertion")
	}
	*b = v[0] == 1
	return nil
}
<commit_msg>Pass golint<commit_after>package types

import (
	"bytes"
	"compress/gzip"
	"database/sql/driver"
	"encoding/json"
	"errors"

	"io/ioutil"
)

// GzippedText is a []byte which transparently gzips data being submitted to
// a database and ungzips data being Scanned from a database.
type GzippedText []byte

// Value implements the driver.Valuer interface, gzipping the raw value of
// this GzippedText.
func (g GzippedText) Value() (driver.Value, error) {
	b := make([]byte, 0, len(g))
	buf := bytes.NewBuffer(b)
	w := gzip.NewWriter(buf)
	w.Write(g)
	w.Close()
	return buf.Bytes(), nil

}

// Scan implements the sql.Scanner interface, ungzipping the value coming off
// the wire and storing the raw result in the GzippedText.
func (g *GzippedText) Scan(src interface{}) error {
	var source []byte
	switch src.(type) {
	case string:
		source = []byte(src.(string))
	case []byte:
		source = src.([]byte)
	default:
		return errors.New("Incompatible type for GzippedText")
	}
	reader, err := gzip.NewReader(bytes.NewReader(source))
	if err != nil {
		return err
	}
	defer reader.Close()
	b, err := ioutil.ReadAll(reader)
	if err != nil {
		return err
	}
	*g = GzippedText(b)
	return nil
}

// JSONText is a json.RawMessage, which is a []byte underneath.
// Value() validates the json format in the source, and returns an error if
// the json is not valid.  Scan does no validation.  JSONText additionally
// implements `Unmarshal`, which unmarshals the json within to an interface{}
type JSONText json.RawMessage

var _EMPTY_JSON = JSONText("{}")

// MarshalJSON returns the *j as the JSON encoding of j.
func (j *JSONText) MarshalJSON() ([]byte, error) {
	if len(*j) == 0 {
		*j = _EMPTY_JSON
	}
	return *j, nil
}

// UnmarshalJSON sets *j to a copy of data
func (j *JSONText) UnmarshalJSON(data []byte) error {
	if j == nil {
		return errors.New("JSONText: UnmarshalJSON on nil pointer")
	}
	*j = append((*j)[0:0], data...)
	return nil
}

// Value returns j as a value.  This does a validating unmarshal into another
// RawMessage.  If j is invalid json, it returns an error.
func (j JSONText) Value() (driver.Value, error) {
	var m json.RawMessage
	var err = j.Unmarshal(&m)
	if err != nil {
		return []byte{}, err
	}
	return []byte(j), nil
}

// Scan stores the src in *j.  No validation is done.
func (j *JSONText) Scan(src interface{}) error {
	var source []byte
	switch t := src.(type) {
	case string:
		source = []byte(t)
	case []byte:
		if len(t) == 0 {
			source = _EMPTY_JSON
		} else {
			source = t
		}
	case nil:
		*j = _EMPTY_JSON
	default:
		return errors.New("Incompatible type for JSONText")
	}
	*j = JSONText(append((*j)[0:0], source...))
	return nil
}

// Unmarshal unmarshal's the json in j to v, as in json.Unmarshal.
func (j *JSONText) Unmarshal(v interface{}) error {
	if len(*j) == 0 {
		*j = _EMPTY_JSON
	}
	return json.Unmarshal([]byte(*j), v)
}

// String supports pretty printing for JSONText types.
func (j JSONText) String() string {
	return string(j)
}

// BitBool is an implementation of a bool for the MySQL type BIT(1).
// This type allows you to avoid wasting an entire byte for MySQL's boolean type TINYINT.
type BitBool bool

// Value implements the driver.Valuer interface,
// and turns the BitBool into a bitfield (BIT(1)) for MySQL storage.
func (b BitBool) Value() (driver.Value, error) {
	if b {
		return []byte{1}, nil
	}
	return []byte{0}, nil
}

// Scan implements the sql.Scanner interface,
// and turns the bitfield incoming from MySQL into a BitBool
func (b *BitBool) Scan(src interface{}) error {
	v, ok := src.([]byte)
	if !ok {
		return errors.New("bad []byte type assertion")
	}
	*b = v[0] == 1
	return nil
}
<|endoftext|>


<commit_before>package types

import (
	"fmt"
)

// Type describes all type structs
type Type interface {
	Equals(other Type) bool
	IsError() bool
	String() string
	isType()
}

// Error signals that whatever expression was supposed to produce this type
// had a semantic error that made proper evaluation impossible
type Error struct{}

// Equals returns false for every other type
func (t Error) Equals(other Type) bool {
	if _, ok := other.(Error); ok {
		return true
	}

	return false
}

// IsError returns true because TypeError is an error
func (t Error) IsError() bool  { return true }
func (t Error) String() string { return "ERROR" }
func (t Error) isType()        {}

// Any can represent all types
type Any struct{}

// Equals returns true for every other type
func (t Any) Equals(other Type) bool { return true }

// IsError returns false because this is a properly resolved type
func (t Any) IsError() bool  { return false }
func (t Any) String() string { return "Any" }
func (t Any) isType()        {}

// Void descirbes the return type of a function that returns no value
type Void struct{}

// Equals returns true if another type has an identical structure and identical names
func (t Void) Equals(other Type) bool {
	if _, ok := other.(Void); ok {
		return true
	}

	return false
}

// IsError returns false because this is a properly resolved type
func (t Void) IsError() bool  { return false }
func (t Void) String() string { return "Void" }
func (t Void) isType()        {}

// Function describes mappings of 0+ parameter types to a return type
type Function struct {
	Params Tuple
	Ret    Type
}

// Equals returns true if another type has an identical structure and identical names
func (t Function) Equals(other Type) bool {
	if t2, ok := other.(Function); ok {
		return t.Params.Equals(t2.Params) && t.Ret.Equals(t2.Ret)
	}

	return false
}

// IsError returns false because this is a properly resolved type
func (t Function) IsError() bool  { return false }
func (t Function) String() string { return fmt.Sprintf("%s => %s", t.Params, t.Ret) }
func (t Function) isType()        {}

// Tuple describes a group of types
type Tuple struct {
	Children []Type
}

// Equals returns true if another type has an identical structure and identical names
func (t Tuple) Equals(other Type) bool {
	if t2, ok := other.(Tuple); ok {
		if len(t.Children) != len(t2.Children) {
			return false
		}

		for i, child := range t.Children {
			child2 := t2.Children[i]
			if child.Equals(child2) == false {
				return false
			}
		}

		return true
	}

	return false
}

// IsError returns false because this is a properly resolved type
func (t Tuple) IsError() bool  { return false }
func (t Tuple) String() string { return fmt.Sprintf("(%s)", concatTypes(t.Children)) }
func (t Tuple) isType()        {}

// List describes an array of a common type
type List struct {
	Child Type
}

// Equals returns true if another type has an identical structure and identical names
func (t List) Equals(other Type) bool {
	if t2, ok := other.(List); ok {
		return t.Child.Equals(t2.Child)
	}

	return false
}

// IsError returns false because this is a properly resolved type
func (t List) IsError() bool  { return false }
func (t List) String() string { return fmt.Sprintf("[%s]", t.Child) }
func (t List) isType()        {}

// Optional describes a type that may resolve to a value or nothing
type Optional struct {
	Child Type
}

// Equals returns true if another type has an identical structure and identical names
func (t Optional) Equals(other Type) bool {
	if t2, ok := other.(Optional); ok {
		return t.Child.Equals(t2.Child)
	}

	return false
}

// IsError returns false because this is a properly resolved type
func (t Optional) IsError() bool  { return false }
func (t Optional) String() string { return fmt.Sprintf("%s?", t.Child) }
func (t Optional) isType()        {}

// Ident describes a type aliased to an identifier
type Ident struct {
	Name string
}

// Equals returns true if another type has an identical structure and identical names
func (t Ident) Equals(other Type) bool {
	if t2, ok := other.(Ident); ok {
		return t.Name == t2.Name
	}

	return false
}

// IsError returns false because this is a properly resolved type
func (t Ident) IsError() bool  { return false }
func (t Ident) String() string { return t.Name }
func (t Ident) isType()        {}

func concatTypes(types []Type) string {
	out := ""
	for i, typ := range types {
		if i > 0 {
			out += " "
		}
		out += typ.String()
	}
	return out
}
<commit_msg>Any should not be equal to Error<commit_after>package types

import (
	"fmt"
)

// Type describes all type structs
type Type interface {
	Equals(other Type) bool
	IsError() bool
	String() string
	isType()
}

// Error signals that whatever expression was supposed to produce this type
// had a semantic error that made proper evaluation impossible
type Error struct{}

// Equals returns false for every other type
func (t Error) Equals(other Type) bool {
	if _, ok := other.(Error); ok {
		return true
	}

	return false
}

// IsError returns true because TypeError is an error
func (t Error) IsError() bool  { return true }
func (t Error) String() string { return "ERROR" }
func (t Error) isType()        {}

// Any can represent all types
type Any struct{}

// Equals returns true for every other type
func (t Any) Equals(other Type) bool {
	if other.IsError() {
		return false
	}

	return true
}

// IsError returns false because this is a properly resolved type
func (t Any) IsError() bool  { return false }
func (t Any) String() string { return "Any" }
func (t Any) isType()        {}

// Void descirbes the return type of a function that returns no value
type Void struct{}

// Equals returns true if another type has an identical structure and identical names
func (t Void) Equals(other Type) bool {
	if _, ok := other.(Void); ok {
		return true
	}

	return false
}

// IsError returns false because this is a properly resolved type
func (t Void) IsError() bool  { return false }
func (t Void) String() string { return "Void" }
func (t Void) isType()        {}

// Function describes mappings of 0+ parameter types to a return type
type Function struct {
	Params Tuple
	Ret    Type
}

// Equals returns true if another type has an identical structure and identical names
func (t Function) Equals(other Type) bool {
	if t2, ok := other.(Function); ok {
		return t.Params.Equals(t2.Params) && t.Ret.Equals(t2.Ret)
	}

	return false
}

// IsError returns false because this is a properly resolved type
func (t Function) IsError() bool  { return false }
func (t Function) String() string { return fmt.Sprintf("%s => %s", t.Params, t.Ret) }
func (t Function) isType()        {}

// Tuple describes a group of types
type Tuple struct {
	Children []Type
}

// Equals returns true if another type has an identical structure and identical names
func (t Tuple) Equals(other Type) bool {
	if t2, ok := other.(Tuple); ok {
		if len(t.Children) != len(t2.Children) {
			return false
		}

		for i, child := range t.Children {
			child2 := t2.Children[i]
			if child.Equals(child2) == false {
				return false
			}
		}

		return true
	}

	return false
}

// IsError returns false because this is a properly resolved type
func (t Tuple) IsError() bool  { return false }
func (t Tuple) String() string { return fmt.Sprintf("(%s)", concatTypes(t.Children)) }
func (t Tuple) isType()        {}

// List describes an array of a common type
type List struct {
	Child Type
}

// Equals returns true if another type has an identical structure and identical names
func (t List) Equals(other Type) bool {
	if t2, ok := other.(List); ok {
		return t.Child.Equals(t2.Child)
	}

	return false
}

// IsError returns false because this is a properly resolved type
func (t List) IsError() bool  { return false }
func (t List) String() string { return fmt.Sprintf("[%s]", t.Child) }
func (t List) isType()        {}

// Optional describes a type that may resolve to a value or nothing
type Optional struct {
	Child Type
}

// Equals returns true if another type has an identical structure and identical names
func (t Optional) Equals(other Type) bool {
	if t2, ok := other.(Optional); ok {
		return t.Child.Equals(t2.Child)
	}

	return false
}

// IsError returns false because this is a properly resolved type
func (t Optional) IsError() bool  { return false }
func (t Optional) String() string { return fmt.Sprintf("%s?", t.Child) }
func (t Optional) isType()        {}

// Ident describes a type aliased to an identifier
type Ident struct {
	Name string
}

// Equals returns true if another type has an identical structure and identical names
func (t Ident) Equals(other Type) bool {
	if t2, ok := other.(Ident); ok {
		return t.Name == t2.Name
	}

	return false
}

// IsError returns false because this is a properly resolved type
func (t Ident) IsError() bool  { return false }
func (t Ident) String() string { return t.Name }
func (t Ident) isType()        {}

func concatTypes(types []Type) string {
	out := ""
	for i, typ := range types {
		if i > 0 {
			out += " "
		}
		out += typ.String()
	}
	return out
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/Financial-Times/up-neoutil-go"
	"github.com/gorilla/mux"
	"github.com/jmcvetta/neoism"
	"log"
	"net/http"
	"os"
	"os/signal"
)

func main() {

	neoURL := os.Getenv("NEO_URL")
	if neoURL == "" {
		log.Println("no $NEO_URL set, defaulting to local")
		neoURL = "http://localhost:7474/db/data"
	}
	log.Printf("connecting to %s\n", neoURL)

	var err error
	db, err = neoism.Connect(neoURL)
	if err != nil {
		panic(err)
	}

	neoutil.EnsureIndexes(db, map[string]string{
		"Organisation": "uuid",
		"Concept":      "uuid",
		"Industry":     "uuid",
	})

	port := 8080

	m := mux.NewRouter()
	http.Handle("/", m)

	m.HandleFunc("/organisations/{uuid}", writeHandler).Methods("PUT")

	go func() {
		log.Printf("listening on %d", port)
		if err := http.ListenAndServe(fmt.Sprintf(":%d", port), nil); err != nil {
			log.Printf("web stuff failed: %v\n", err)
		}
	}()

	sw = neoutil.NewSafeWriter(db, 1024)

	// wait for ctrl-c
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	<-c

	sw.Close()

	log.Println("exiting")
}

var db *neoism.Database

var sw neoutil.CypherWriter

func writeHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	uuid := vars["uuid"]

	var o organisation
	dec := json.NewDecoder(r.Body)
	err := dec.Decode(&o)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if o.UUID != uuid {
		fmt.Printf("%v\n", o)
		http.Error(w, fmt.Sprintf("id does not match: %v %v", o.UUID, uuid), http.StatusBadRequest)
		return
	}

	err = sw.WriteCypher(toQueries(o))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func toQueries(o organisation) []*neoism.CypherQuery {
	p := map[string]interface{}{
		"uuid": o.UUID,
	}

	if o.Extinct == true {
		p["extinct"] = true
	}
	if o.FormerNames != nil && len(o.FormerNames) != 0 {
		p["formerNames"] = o.FormerNames
	}
	if o.HiddenLabel != "" {
		p["hiddenLabel"] = o.HiddenLabel
	}
	if o.LegalName != "" {
		p["legalName"] = o.LegalName
	}
	if o.LocalNames != nil && len(o.LocalNames) != 0 {
		p["localNames"] = o.LocalNames
	}
	if o.ProperName != "" {
		p["properName"] = o.ProperName
		p["prefLabel"] = o.ProperName
	}
	if o.ShortName != "" {
		p["shortName"] = o.ShortName
	}
	if o.TradeNames != nil && len(o.TradeNames) != 0 {
		p["tradeNames"] = o.TradeNames
	}
	for _, identifier := range o.Identifiers {
		if identifier.Authority == fsAuthority {
			p["factsetIdentifier"] = identifier.IdentifierValue
		}
		if identifier.Authority == leiAuthority {
			p["leiIdentifier"] = identifier.IdentifierValue
		}
	}
	p["uuid"] = o.UUID

	parms := map[string]interface{}{
		"uuid":     o.UUID,
		"allProps": neoism.Props(p),
		"puuid":    o.ParentOrganisation,
		"icuuid":   o.IndustryClassification,
	}

	statement := `
			MERGE (n:Concept {uuid: {uuid}})
			SET n = {allProps}
			SET n :Organisation
		`

	if o.Type != "Organisation" && o.Type != "" {
		statement += fmt.Sprintf("SET n :%s\n", o.Type)
	}

	if o.ParentOrganisation != "" {
		statement += `
			MERGE (p:Concept {uuid: {puuid}})
			MERGE (n)-[:SUB_ORG_OF]->(p)
			SET p :Organisation
		`
	}

	if o.IndustryClassification != "" {
		statement += `
			MERGE (ic:Concept {uuid: {icuuid}})
			MERGE (n)-[:IN_INDUSTRY]->(ic)
			SET ic :Industry
		`
	}

	return []*neoism.CypherQuery{
		&neoism.CypherQuery{Statement: statement, Parameters: parms},
	}
}

const (
	fsAuthority  = "http://api.ft.com/system/FACTSET-EDM"
	leiAuthority = "http://api.ft.com/system/LEI"
)
<commit_msg>use new "safe" writer<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/Financial-Times/up-neoutil-go"
	"github.com/gorilla/mux"
	"github.com/jmcvetta/neoism"
	"log"
	"net/http"
	"os"
	"os/signal"
)

func main() {

	neoURL := os.Getenv("NEO_URL")
	if neoURL == "" {
		log.Println("no $NEO_URL set, defaulting to local")
		neoURL = "http://localhost:7474/db/data"
	}
	log.Printf("connecting to %s\n", neoURL)

	var err error
	db, err = neoism.Connect(neoURL)
	if err != nil {
		panic(err)
	}

	neoutil.EnsureIndexes(db, map[string]string{
		"Organisation": "uuid",
		"Concept":      "uuid",
		"Industry":     "uuid",
	})

	port := 8080

	m := mux.NewRouter()
	http.Handle("/", m)

	m.HandleFunc("/organisations/{uuid}", writeHandler).Methods("PUT")

	go func() {
		log.Printf("listening on %d", port)
		if err := http.ListenAndServe(fmt.Sprintf(":%d", port), nil); err != nil {
			log.Printf("web stuff failed: %v\n", err)
		}
	}()

	cw = neoutil.NewSafeWriter(db, 1024)

	// wait for ctrl-c
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	<-c

	cw.Close()

	log.Println("exiting")
}

var db *neoism.Database

var cw neoutil.CypherWriter

func writeHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	uuid := vars["uuid"]

	var o organisation
	dec := json.NewDecoder(r.Body)
	err := dec.Decode(&o)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if o.UUID != uuid {
		fmt.Printf("%v\n", o)
		http.Error(w, fmt.Sprintf("id does not match: %v %v", o.UUID, uuid), http.StatusBadRequest)
		return
	}

	err = cw.WriteCypher(toQueries(o))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func toQueries(o organisation) []*neoism.CypherQuery {
	p := map[string]interface{}{
		"uuid": o.UUID,
	}

	if o.Extinct == true {
		p["extinct"] = true
	}
	if o.FormerNames != nil && len(o.FormerNames) != 0 {
		p["formerNames"] = o.FormerNames
	}
	if o.HiddenLabel != "" {
		p["hiddenLabel"] = o.HiddenLabel
	}
	if o.LegalName != "" {
		p["legalName"] = o.LegalName
	}
	if o.LocalNames != nil && len(o.LocalNames) != 0 {
		p["localNames"] = o.LocalNames
	}
	if o.ProperName != "" {
		p["properName"] = o.ProperName
		p["prefLabel"] = o.ProperName
	}
	if o.ShortName != "" {
		p["shortName"] = o.ShortName
	}
	if o.TradeNames != nil && len(o.TradeNames) != 0 {
		p["tradeNames"] = o.TradeNames
	}
	for _, identifier := range o.Identifiers {
		if identifier.Authority == fsAuthority {
			p["factsetIdentifier"] = identifier.IdentifierValue
		}
		if identifier.Authority == leiAuthority {
			p["leiIdentifier"] = identifier.IdentifierValue
		}
	}
	p["uuid"] = o.UUID

	parms := map[string]interface{}{
		"uuid":     o.UUID,
		"allProps": neoism.Props(p),
		"puuid":    o.ParentOrganisation,
		"icuuid":   o.IndustryClassification,
	}

	statement := `
			MERGE (n:Concept {uuid: {uuid}})
			SET n = {allProps}
			SET n :Organisation
		`

	if o.Type != "Organisation" && o.Type != "" {
		statement += fmt.Sprintf("SET n :%s\n", o.Type)
	}

	if o.ParentOrganisation != "" {
		statement += `
			MERGE (p:Concept {uuid: {puuid}})
			MERGE (n)-[:SUB_ORG_OF]->(p)
			SET p :Organisation
		`
	}

	if o.IndustryClassification != "" {
		statement += `
			MERGE (ic:Concept {uuid: {icuuid}})
			MERGE (n)-[:IN_INDUSTRY]->(ic)
			SET ic :Industry
		`
	}

	return []*neoism.CypherQuery{
		&neoism.CypherQuery{Statement: statement, Parameters: parms},
	}
}

const (
	fsAuthority  = "http://api.ft.com/system/FACTSET-EDM"
	leiAuthority = "http://api.ft.com/system/LEI"
)
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package graph

import (
	"errors"
	"fmt"
	"sync"

	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

// A visitor in a directed graph whose nodes are identified by strings.
type Visitor interface {
	// Process the supplied node and return a list of direct successors.
	Visit(ctx context.Context, node string) (adjacent []string, err error)
}

// Invoke v.Visit on each node reachable from the supplied search roots,
// including the roots themselves. Use the supplied degree of parallelism.
//
// It is guaranteed that if a node N is fed to v.Visit, then either:
//
//  *  N is an element of roots, or
//  *  There exists a direct predecessor N' of N such that v.Visit(N') was
//     called and returned successfully.
//
// In particular, if the graph is a rooted tree and searching starts at the
// root, then parents will be successfully visited before children are visited.
// However note that in arbitrary DAGs it is *not* guaranteed that all of a
// node's predecessors have been visited before it is.
func Traverse(
	ctx context.Context,
	parallelism int,
	roots []string,
	v Visitor) (err error) {
	b := syncutil.NewBundle(ctx)
	defer func() { err = b.Join() }()

	// Set up initial state.
	ts := &traverseState{
		admitted: make(map[string]struct{}),
	}

	ts.mu = syncutil.NewInvariantMutex(ts.checkInvariants)
	ts.cond.L = &ts.mu

	for _, r := range roots {
		ts.admitted[r] = struct{}{}
		ts.toVisit = append(ts.toVisit, r)
	}

	// Ensure that ts.cancelled is set when the context is cancelled (or when we
	// return from this function, if the context will never be cancelled).
	done := ctx.Done()
	if done == nil {
		doneChan := make(chan struct{})
		defer close(doneChan)

		done = doneChan
	}

	go watchForCancel(done, ts)

	// Run the appropriate number of workers.
	for i := 0; i < parallelism; i++ {
		b.Add(func(ctx context.Context) (err error) {
			err = traverse(ctx, ts, v)
			return
		})
	}

	return
}

////////////////////////////////////////////////////////////////////////
// traverseState
////////////////////////////////////////////////////////////////////////

// State shared by each traverse worker.
type traverseState struct {
	mu syncutil.InvariantMutex

	// All nodes that have ever been seen. If a node is in this map, it will
	// eventually be visted (barring errors returned by the visitor).
	//
	// GUARDED_BY(mu)
	admitted map[string]struct{}

	// Admitted nodes that have yet to be visted.
	//
	// INVARIANT: For each n in toVisit, n is a key of admitted.
	//
	// GUARDED_BY(mu)
	toVisit []string

	// Set to true if the context has been cancelled. All workers should return
	// when this happens.
	//
	// GUARDED_BY(mu)
	cancelled bool

	// The number of workers that are doing something besides waiting on a node
	// to visit. If this hits zero with toVisit empty, it means that there is
	// nothing further to do.
	//
	// GUARDED_BY(mu)
	busyWorkers int

	// Signalled with mu held when any of the following state changes:
	//
	//  *  toVisit
	//  *  cancelled
	//  *  busyWorkers
	//
	// GUARDED_BY(mu)
	cond sync.Cond
}

// LOCKS_REQUIRED(ts.mu)
func (ts *traverseState) checkInvariants() {
	// INVARIANT: For each n in toVisit, n is a key of admitted.
	for _, n := range ts.toVisit {
		if _, ok := ts.admitted[n]; !ok {
			panic(fmt.Sprintf("Expected %q to be in admitted map", n))
		}
	}
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// A single traverse worker.
func traverse(
	ctx context.Context,
	ts *traverseState,
	v Visitor) (err error) {
	err = errors.New("TODO")
	return
}

// Bridge context cancellation with traverseState.cancelled.
func watchForCancel(
	done <-chan struct{},
	ts *traverseState) {
	<-done

	ts.mu.Lock()
	defer ts.mu.Unlock()
	ts.cancelled = true
}
<commit_msg>traverseState.shouldWake<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package graph

import (
	"errors"
	"fmt"
	"sync"

	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

// A visitor in a directed graph whose nodes are identified by strings.
type Visitor interface {
	// Process the supplied node and return a list of direct successors.
	Visit(ctx context.Context, node string) (adjacent []string, err error)
}

// Invoke v.Visit on each node reachable from the supplied search roots,
// including the roots themselves. Use the supplied degree of parallelism.
//
// It is guaranteed that if a node N is fed to v.Visit, then either:
//
//  *  N is an element of roots, or
//  *  There exists a direct predecessor N' of N such that v.Visit(N') was
//     called and returned successfully.
//
// In particular, if the graph is a rooted tree and searching starts at the
// root, then parents will be successfully visited before children are visited.
// However note that in arbitrary DAGs it is *not* guaranteed that all of a
// node's predecessors have been visited before it is.
func Traverse(
	ctx context.Context,
	parallelism int,
	roots []string,
	v Visitor) (err error) {
	b := syncutil.NewBundle(ctx)
	defer func() { err = b.Join() }()

	// Set up initial state.
	ts := &traverseState{
		admitted: make(map[string]struct{}),
	}

	ts.mu = syncutil.NewInvariantMutex(ts.checkInvariants)
	ts.cond.L = &ts.mu

	for _, r := range roots {
		ts.admitted[r] = struct{}{}
		ts.toVisit = append(ts.toVisit, r)
	}

	// Ensure that ts.cancelled is set when the context is cancelled (or when we
	// return from this function, if the context will never be cancelled).
	done := ctx.Done()
	if done == nil {
		doneChan := make(chan struct{})
		defer close(doneChan)

		done = doneChan
	}

	go watchForCancel(done, ts)

	// Run the appropriate number of workers.
	for i := 0; i < parallelism; i++ {
		b.Add(func(ctx context.Context) (err error) {
			err = traverse(ctx, ts, v)
			return
		})
	}

	return
}

////////////////////////////////////////////////////////////////////////
// traverseState
////////////////////////////////////////////////////////////////////////

// State shared by each traverse worker.
type traverseState struct {
	mu syncutil.InvariantMutex

	// All nodes that have ever been seen. If a node is in this map, it will
	// eventually be visted (barring errors returned by the visitor).
	//
	// GUARDED_BY(mu)
	admitted map[string]struct{}

	// Admitted nodes that have yet to be visted.
	//
	// INVARIANT: For each n in toVisit, n is a key of admitted.
	//
	// GUARDED_BY(mu)
	toVisit []string

	// Set to true if the context has been cancelled. All workers should return
	// when this happens.
	//
	// GUARDED_BY(mu)
	cancelled bool

	// The number of workers that are doing something besides waiting on a node
	// to visit. If this hits zero with toVisit empty, it means that there is
	// nothing further to do.
	//
	// GUARDED_BY(mu)
	busyWorkers int

	// Signalled with mu held when any of the following state changes:
	//
	//  *  toVisit
	//  *  cancelled
	//  *  busyWorkers
	//
	// GUARDED_BY(mu)
	cond sync.Cond
}

// LOCKS_REQUIRED(ts.mu)
func (ts *traverseState) checkInvariants() {
	// INVARIANT: For each n in toVisit, n is a key of admitted.
	for _, n := range ts.toVisit {
		if _, ok := ts.admitted[n]; !ok {
			panic(fmt.Sprintf("Expected %q to be in admitted map", n))
		}
	}
}

// Is there anything that needs a worker's attention?
//
// LOCKS_REQUIRED(ts.mu)
func (ts *traverseState) shouldWake() {
	return len(ts.toVisit) != nil || ts.cancelled || ts.busyWorkers == 0
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// A single traverse worker.
func traverse(
	ctx context.Context,
	ts *traverseState,
	v Visitor) (err error) {
	err = errors.New("TODO")
	return
}

// Bridge context cancellation with traverseState.cancelled.
func watchForCancel(
	done <-chan struct{},
	ts *traverseState) {
	<-done

	ts.mu.Lock()
	defer ts.mu.Unlock()
	ts.cancelled = true
}
<|endoftext|>