<commit_before>package drive

import (
    "time"
    "fmt"
    "os"
    "path/filepath"
    "github.com/soniakeys/graph"
    "github.com/sabhiram/go-git-ignore"
    "golang.org/x/net/context"
    "google.golang.org/api/drive/v3"
    "google.golang.org/api/googleapi"
)

const DefaultIgnoreFile = ".gdriveignore"

func (self *Drive) prepareSyncFiles(localPath string, root *drive.File, cmp FileComparer) (*syncFiles, error) {
    localCh := make(chan struct{files []*LocalFile; err error})
    remoteCh := make(chan struct{files []*RemoteFile; err error})

    go func() {
        files, err := prepareLocalFiles(localPath)
        localCh <- struct{files []*LocalFile; err error}{files, err}
    }()

    go func() {
        files, err := self.prepareRemoteFiles(root)
        remoteCh <- struct{files []*RemoteFile; err error}{files, err}
    }()

    local := <-localCh
    if local.err != nil {
        return nil, local.err
    }

    remote := <-remoteCh
    if remote.err != nil {
        return nil, remote.err
    }

    return &syncFiles{
        root: &RemoteFile{file: root},
        local: local.files,
        remote: remote.files,
        compare: cmp,
    }, nil
}

func prepareLocalFiles(root string) ([]*LocalFile, error) {
    var files []*LocalFile

    // Get absolute root path
    absRootPath, err := filepath.Abs(root)
    if err != nil {
        return nil, err
    }

    // Prepare ignorer
    shouldIgnore, err := prepareIgnorer(filepath.Join(absRootPath, DefaultIgnoreFile))
    if err != nil {
        return nil, err
    }

    err = filepath.Walk(absRootPath, func(absPath string, info os.FileInfo, err error) error {
        if err != nil {
            return err
        }

        // Skip root directory
        if absPath == absRootPath {
            return nil
        }

        // Skip files that are not a directory or regular file
        if !info.IsDir() && !info.Mode().IsRegular() {
            return nil
        }

        // Get relative path from root
        relPath, err := filepath.Rel(absRootPath, absPath)
        if err != nil {
            return err
        }

        // Skip file if it is ignored by ignore file
        if shouldIgnore(relPath) {
            return nil
        }

        files = append(files, &LocalFile{
            absPath: absPath,
            relPath: relPath,
            info: info,
        })

        return nil
    })

    if err != nil {
        return nil, fmt.Errorf("Failed to prepare local files: %s", err)
    }

    return files, err
}

func (self *Drive) listAllFiles(q string, fields []googleapi.Field) ([]*drive.File, error) {
    var files []*drive.File

    err := self.service.Files.List().Q(q).Fields(fields...).PageSize(1000).Pages(context.TODO(), func(fl *drive.FileList) error {
        files = append(files, fl.Files...)
        return nil
    })

    return files, err
}

func (self *Drive) prepareRemoteFiles(rootDir *drive.File) ([]*RemoteFile, error) {
    // Find all files which has rootDir as root
    query := fmt.Sprintf("appProperties has {key='syncRootId' and value='%s'}", rootDir.Id)
    fields := []googleapi.Field{"nextPageToken", "files(id,name,parents,md5Checksum,mimeType)"}
    files, err := self.listAllFiles(query, fields)
    if err != nil {
        return nil, fmt.Errorf("Failed listing files: %s", err)
    }

    if err := checkFiles(files); err != nil {
        return nil, err
    }

    relPaths, err := prepareRemoteRelPaths(rootDir, files)
    if err != nil {
        return nil, err
    }

    var remoteFiles []*RemoteFile
    for _, f := range files {
        relPath, ok := relPaths[f.Id]
        if !ok {
            return nil, fmt.Errorf("File %s does not have a valid parent", f.Id)
        }
        remoteFiles = append(remoteFiles, &RemoteFile{
            relPath: relPath,
            file: f,
        })
    }

    return remoteFiles, nil
}

func prepareRemoteRelPaths(root *drive.File, files []*drive.File) (map[string]string, error) {
    // The graph will only hold integer values so we use
    // maps to lookup file by index and index by file id
    indexLookup := map[string]graph.NI{}
    fileLookup := map[graph.NI]*drive.File{}

    // All files includes root dir
    allFiles := append([]*drive.File{root}, files...)

    // Prepare lookup maps
    for i, f := range allFiles {
        indexLookup[f.Id] = graph.NI(i)
        fileLookup[graph.NI(i)] = f
    }

    // Graph will hold relationship between parent and file
    g := &graph.AdjacencyList{}

    // Add relationship between parent and file for all files to graph
    for i, f := range allFiles {
        if f == root {
            continue
        }

        // Lookup index of parent
        parentIdx, found := indexLookup[f.Parents[0]]
        if !found {
            return nil, fmt.Errorf("Could not find parent of %s (%s)", f.Id, f.Name)
        }

        g.AddEdge(graph.NI(parentIdx), graph.NI(i))
    }

    // This will hold a map of file id => relative path
    paths := map[string]string{}

    // Find relative path from root for all files
    for _, f := range allFiles {
        if f == root {
            continue
        }

        // Find nodes between root and file
        nodes := g.BreadthFirstPath(0, indexLookup[f.Id])

        // This will hold the name of all paths between root and
        // file (exluding root and including file itself)
        pathNames := []string{}

        // Lookup file for each node and grab name
        for _, n := range nodes {
            file := fileLookup[n]
            if file == root {
                continue
            }
            pathNames = append(pathNames, file.Name)
        }

        // Join path names to form relative path and add to map
        paths[f.Id] = filepath.Join(pathNames...)
    }

    return paths, nil
}

func checkFiles(files []*drive.File) error {
    uniq := map[string]string{}

    for _, f := range files {
        // Ensure all files have exactly one parent
        if len(f.Parents) != 1 {
            return fmt.Errorf("File %s does not have exacly one parent", f.Id)
        }

        // Ensure that there are no duplicate files
        uniqKey := f.Name + f.Parents[0]
        if dupeId, isDupe := uniq[uniqKey]; isDupe {
            return fmt.Errorf("Found name collision between %s and %s", f.Id, dupeId)
        }
        uniq[uniqKey] = f.Id
    }

    return nil
}

type LocalFile struct {
    absPath string
    relPath string
    info os.FileInfo
}

type RemoteFile struct {
    relPath string
    file *drive.File
}

type changedFile struct {
    local *LocalFile
    remote *RemoteFile
}

type syncFiles struct {
    root *RemoteFile
    local []*LocalFile
    remote []*RemoteFile
    compare FileComparer
}

type FileComparer interface {
    Changed(*LocalFile, *RemoteFile) bool
}

func (self LocalFile) AbsPath() string {
    return self.absPath
}

func (self LocalFile) Size() int64 {
    return self.info.Size()
}

func (self LocalFile) Modified() time.Time {
    return self.info.ModTime()
}

func (self RemoteFile) Md5() string {
    return self.file.Md5Checksum
}

func (self RemoteFile) Size() int64 {
    return self.file.Size
}

func (self RemoteFile) Modified() time.Time {
    t, _ := time.Parse(time.RFC3339, self.file.ModifiedTime)
    return t
}

func (self *syncFiles) filterMissingRemoteDirs() []*LocalFile {
    var files []*LocalFile

    for _, lf := range self.local {
        if lf.info.IsDir() && !self.existsRemote(lf) {
            files = append(files, lf)
        }
    }

    return files
}

func (self *syncFiles) filterMissingLocalDirs() []*RemoteFile {
    var files []*RemoteFile

    for _, rf := range self.remote {
        if isDir(rf.file) && !self.existsLocal(rf) {
            files = append(files, rf)
        }
    }

    return files
}

func (self *syncFiles) filterMissingRemoteFiles() []*LocalFile {
    var files []*LocalFile

    for _, lf := range self.local {
        if !lf.info.IsDir() && !self.existsRemote(lf) {
            files = append(files, lf)
        }
    }

    return files
}

func (self *syncFiles) filterMissingLocalFiles() []*RemoteFile {
    var files []*RemoteFile

    for _, rf := range self.remote {
        if !isDir(rf.file) && !self.existsLocal(rf) {
            files = append(files, rf)
        }
    }

    return files
}

func (self *syncFiles) filterChangedLocalFiles() []*changedFile {
    var files []*changedFile

    for _, lf := range self.local {
        // Skip directories
        if lf.info.IsDir() {
            continue
        }

        // Skip files that don't exist on drive
        rf, found := self.findRemoteByPath(lf.relPath)
        if !found {
            continue
        }

        // Check if file has changed
        if self.compare.Changed(lf, rf) {
            files = append(files, &changedFile{
                local: lf,
                remote: rf,
            })
        }
    }

    return files
}

func (self *syncFiles) filterChangedRemoteFiles() []*changedFile {
    var files []*changedFile

    for _, rf := range self.remote {
        // Skip directories
        if isDir(rf.file) {
            continue
        }

        // Skip local files that don't exist
        lf, found := self.findLocalByPath(rf.relPath)
        if !found {
            continue
        }

        // Check if file has changed
        if self.compare.Changed(lf, rf) {
            files = append(files, &changedFile{
                local: lf,
                remote: rf,
            })
        }
    }

    return files
}

func (self *syncFiles) filterExtraneousRemoteFiles() []*RemoteFile {
    var files []*RemoteFile

    for _, rf := range self.remote {
        if !self.existsLocal(rf) {
            files = append(files, rf)
        }
    }

    return files
}

func (self *syncFiles) filterExtraneousLocalFiles() []*LocalFile {
    var files []*LocalFile

    for _, lf := range self.local {
        if !self.existsRemote(lf) {
            files = append(files, lf)
        }
    }

    return files
}

func (self *syncFiles) existsRemote(lf *LocalFile) bool {
    _, found := self.findRemoteByPath(lf.relPath)
    return found
}

func (self *syncFiles) existsLocal(rf *RemoteFile) bool {
    _, found := self.findLocalByPath(rf.relPath)
    return found
}

func (self *syncFiles) findRemoteByPath(relPath string) (*RemoteFile, bool) {
    if relPath == "." {
        return self.root, true
    }

    for _, rf := range self.remote {
        if relPath == rf.relPath {
            return rf, true
        }
    }

    return nil, false
}

func (self *syncFiles) findLocalByPath(relPath string) (*LocalFile, bool) {
    for _, lf := range self.local {
        if relPath == lf.relPath {
            return lf, true
        }
    }

    return nil, false
}

type byLocalPathLength []*LocalFile

func (self byLocalPathLength) Len() int {
    return len(self)
}

func (self byLocalPathLength) Swap(i, j int) {
    self[i], self[j] = self[j], self[i]
}

func (self byLocalPathLength) Less(i, j int) bool {
    return pathLength(self[i].relPath) < pathLength(self[j].relPath)
}

type byRemotePathLength []*RemoteFile

func (self byRemotePathLength) Len() int {
    return len(self)
}

func (self byRemotePathLength) Swap(i, j int) {
    self[i], self[j] = self[j], self[i]
}

func (self byRemotePathLength) Less(i, j int) bool {
    return pathLength(self[i].relPath) < pathLength(self[j].relPath)
}

type ignoreFunc func(string) bool

func prepareIgnorer(path string) (ignoreFunc, error) {
    acceptAll := func(string) bool {
        return false
    }

    if !fileExists(path) {
        return acceptAll, nil
    }

    ignorer, err := ignore.CompileIgnoreFile(path)
    if err != nil {
        return acceptAll, fmt.Errorf("Failed to prepare ignorer: %s", err)
    }

    return ignorer.MatchesPath, nil
}
<commit_msg>Use parent pointer tree<commit_after>package drive

import (
    "time"
    "fmt"
    "os"
    "path/filepath"
    "github.com/soniakeys/graph"
    "github.com/sabhiram/go-git-ignore"
    "golang.org/x/net/context"
    "google.golang.org/api/drive/v3"
    "google.golang.org/api/googleapi"
)

const DefaultIgnoreFile = ".gdriveignore"

func (self *Drive) prepareSyncFiles(localPath string, root *drive.File, cmp FileComparer) (*syncFiles, error) {
    localCh := make(chan struct{files []*LocalFile; err error})
    remoteCh := make(chan struct{files []*RemoteFile; err error})

    go func() {
        files, err := prepareLocalFiles(localPath)
        localCh <- struct{files []*LocalFile; err error}{files, err}
    }()

    go func() {
        files, err := self.prepareRemoteFiles(root)
        remoteCh <- struct{files []*RemoteFile; err error}{files, err}
    }()

    local := <-localCh
    if local.err != nil {
        return nil, local.err
    }

    remote := <-remoteCh
    if remote.err != nil {
        return nil, remote.err
    }

    return &syncFiles{
        root: &RemoteFile{file: root},
        local: local.files,
        remote: remote.files,
        compare: cmp,
    }, nil
}

func prepareLocalFiles(root string) ([]*LocalFile, error) {
    var files []*LocalFile

    // Get absolute root path
    absRootPath, err := filepath.Abs(root)
    if err != nil {
        return nil, err
    }

    // Prepare ignorer
    shouldIgnore, err := prepareIgnorer(filepath.Join(absRootPath, DefaultIgnoreFile))
    if err != nil {
        return nil, err
    }

    err = filepath.Walk(absRootPath, func(absPath string, info os.FileInfo, err error) error {
        if err != nil {
            return err
        }

        // Skip root directory
        if absPath == absRootPath {
            return nil
        }

        // Skip files that are not a directory or regular file
        if !info.IsDir() && !info.Mode().IsRegular() {
            return nil
        }

        // Get relative path from root
        relPath, err := filepath.Rel(absRootPath, absPath)
        if err != nil {
            return err
        }

        // Skip file if it is ignored by ignore file
        if shouldIgnore(relPath) {
            return nil
        }

        files = append(files, &LocalFile{
            absPath: absPath,
            relPath: relPath,
            info: info,
        })

        return nil
    })

    if err != nil {
        return nil, fmt.Errorf("Failed to prepare local files: %s", err)
    }

    return files, err
}

func (self *Drive) listAllFiles(q string, fields []googleapi.Field) ([]*drive.File, error) {
    var files []*drive.File

    err := self.service.Files.List().Q(q).Fields(fields...).PageSize(1000).Pages(context.TODO(), func(fl *drive.FileList) error {
        files = append(files, fl.Files...)
        return nil
    })

    return files, err
}

func (self *Drive) prepareRemoteFiles(rootDir *drive.File) ([]*RemoteFile, error) {
    // Find all files which has rootDir as root
    query := fmt.Sprintf("appProperties has {key='syncRootId' and value='%s'}", rootDir.Id)
    fields := []googleapi.Field{"nextPageToken", "files(id,name,parents,md5Checksum,mimeType)"}
    files, err := self.listAllFiles(query, fields)
    if err != nil {
        return nil, fmt.Errorf("Failed listing files: %s", err)
    }

    if err := checkFiles(files); err != nil {
        return nil, err
    }

    relPaths, err := prepareRemoteRelPaths(rootDir, files)
    if err != nil {
        return nil, err
    }

    var remoteFiles []*RemoteFile
    for _, f := range files {
        relPath, ok := relPaths[f.Id]
        if !ok {
            return nil, fmt.Errorf("File %s does not have a valid parent", f.Id)
        }
        remoteFiles = append(remoteFiles, &RemoteFile{
            relPath: relPath,
            file: f,
        })
    }

    return remoteFiles, nil
}

func prepareRemoteRelPaths(root *drive.File, files []*drive.File) (map[string]string, error) {
    // The tree only holds integer values so we use
    // maps to lookup file by index and index by file id
    indexLookup := map[string]graph.NI{}
    fileLookup := map[graph.NI]*drive.File{}

    // All files includes root dir
    allFiles := append([]*drive.File{root}, files...)

    // Prepare lookup maps
    for i, f := range allFiles {
        indexLookup[f.Id] = graph.NI(i)
        fileLookup[graph.NI(i)] = f
    }

    // This will hold 'parent index' -> 'file index' relationships
    pathEnds := make([]graph.PathEnd, len(allFiles))

    // Prepare parent -> file relationships
    for i, f := range allFiles {
        if f == root {
            pathEnds[i] = graph.PathEnd{From: -1}
            continue
        }

        // Lookup index of parent
        parentIdx, found := indexLookup[f.Parents[0]]
        if !found {
            return nil, fmt.Errorf("Could not find parent of %s (%s)", f.Id, f.Name)
        }
        pathEnds[i] = graph.PathEnd{From: parentIdx}
    }

    // Create parent pointer tree and calculate path lengths
    tree := &graph.FromList{Paths: pathEnds}
    tree.RecalcLeaves()
    tree.RecalcLen()

    // This will hold a map of file id => relative path
    paths := map[string]string{}

    // Find relative path from root for all files
    for _, f := range allFiles {
        if f == root {
            continue
        }

        // Find nodes between root and file
        nodes := tree.PathTo(indexLookup[f.Id], nil)

        // This will hold the name of all paths between root and
        // file (exluding root and including file itself)
        pathNames := []string{}

        // Lookup file for each node and grab name
        for _, n := range nodes {
            file := fileLookup[n]
            if file == root {
                continue
            }
            pathNames = append(pathNames, file.Name)
        }

        // Join path names to form relative path and add to map
        paths[f.Id] = filepath.Join(pathNames...)
    }

    return paths, nil
}

func checkFiles(files []*drive.File) error {
    uniq := map[string]string{}

    for _, f := range files {
        // Ensure all files have exactly one parent
        if len(f.Parents) != 1 {
            return fmt.Errorf("File %s does not have exacly one parent", f.Id)
        }

        // Ensure that there are no duplicate files
        uniqKey := f.Name + f.Parents[0]
        if dupeId, isDupe := uniq[uniqKey]; isDupe {
            return fmt.Errorf("Found name collision between %s and %s", f.Id, dupeId)
        }
        uniq[uniqKey] = f.Id
    }

    return nil
}

type LocalFile struct {
    absPath string
    relPath string
    info os.FileInfo
}

type RemoteFile struct {
    relPath string
    file *drive.File
}

type changedFile struct {
    local *LocalFile
    remote *RemoteFile
}

type syncFiles struct {
    root *RemoteFile
    local []*LocalFile
    remote []*RemoteFile
    compare FileComparer
}

type FileComparer interface {
    Changed(*LocalFile, *RemoteFile) bool
}

func (self LocalFile) AbsPath() string {
    return self.absPath
}

func (self LocalFile) Size() int64 {
    return self.info.Size()
}

func (self LocalFile) Modified() time.Time {
    return self.info.ModTime()
}

func (self RemoteFile) Md5() string {
    return self.file.Md5Checksum
}

func (self RemoteFile) Size() int64 {
    return self.file.Size
}

func (self RemoteFile) Modified() time.Time {
    t, _ := time.Parse(time.RFC3339, self.file.ModifiedTime)
    return t
}

func (self *syncFiles) filterMissingRemoteDirs() []*LocalFile {
    var files []*LocalFile

    for _, lf := range self.local {
        if lf.info.IsDir() && !self.existsRemote(lf) {
            files = append(files, lf)
        }
    }

    return files
}

func (self *syncFiles) filterMissingLocalDirs() []*RemoteFile {
    var files []*RemoteFile

    for _, rf := range self.remote {
        if isDir(rf.file) && !self.existsLocal(rf) {
            files = append(files, rf)
        }
    }

    return files
}

func (self *syncFiles) filterMissingRemoteFiles() []*LocalFile {
    var files []*LocalFile

    for _, lf := range self.local {
        if !lf.info.IsDir() && !self.existsRemote(lf) {
            files = append(files, lf)
        }
    }

    return files
}

func (self *syncFiles) filterMissingLocalFiles() []*RemoteFile {
    var files []*RemoteFile

    for _, rf := range self.remote {
        if !isDir(rf.file) && !self.existsLocal(rf) {
            files = append(files, rf)
        }
    }

    return files
}

func (self *syncFiles) filterChangedLocalFiles() []*changedFile {
    var files []*changedFile

    for _, lf := range self.local {
        // Skip directories
        if lf.info.IsDir() {
            continue
        }

        // Skip files that don't exist on drive
        rf, found := self.findRemoteByPath(lf.relPath)
        if !found {
            continue
        }

        // Check if file has changed
        if self.compare.Changed(lf, rf) {
            files = append(files, &changedFile{
                local: lf,
                remote: rf,
            })
        }
    }

    return files
}

func (self *syncFiles) filterChangedRemoteFiles() []*changedFile {
    var files []*changedFile

    for _, rf := range self.remote {
        // Skip directories
        if isDir(rf.file) {
            continue
        }

        // Skip local files that don't exist
        lf, found := self.findLocalByPath(rf.relPath)
        if !found {
            continue
        }

        // Check if file has changed
        if self.compare.Changed(lf, rf) {
            files = append(files, &changedFile{
                local: lf,
                remote: rf,
            })
        }
    }

    return files
}

func (self *syncFiles) filterExtraneousRemoteFiles() []*RemoteFile {
    var files []*RemoteFile

    for _, rf := range self.remote {
        if !self.existsLocal(rf) {
            files = append(files, rf)
        }
    }

    return files
}

func (self *syncFiles) filterExtraneousLocalFiles() []*LocalFile {
    var files []*LocalFile

    for _, lf := range self.local {
        if !self.existsRemote(lf) {
            files = append(files, lf)
        }
    }

    return files
}

func (self *syncFiles) existsRemote(lf *LocalFile) bool {
    _, found := self.findRemoteByPath(lf.relPath)
    return found
}

func (self *syncFiles) existsLocal(rf *RemoteFile) bool {
    _, found := self.findLocalByPath(rf.relPath)
    return found
}

func (self *syncFiles) findRemoteByPath(relPath string) (*RemoteFile, bool) {
    if relPath == "." {
        return self.root, true
    }

    for _, rf := range self.remote {
        if relPath == rf.relPath {
            return rf, true
        }
    }

    return nil, false
}

func (self *syncFiles) findLocalByPath(relPath string) (*LocalFile, bool) {
    for _, lf := range self.local {
        if relPath == lf.relPath {
            return lf, true
        }
    }

    return nil, false
}

type byLocalPathLength []*LocalFile

func (self byLocalPathLength) Len() int {
    return len(self)
}

func (self byLocalPathLength) Swap(i, j int) {
    self[i], self[j] = self[j], self[i]
}

func (self byLocalPathLength) Less(i, j int) bool {
    return pathLength(self[i].relPath) < pathLength(self[j].relPath)
}

type byRemotePathLength []*RemoteFile

func (self byRemotePathLength) Len() int {
    return len(self)
}

func (self byRemotePathLength) Swap(i, j int) {
    self[i], self[j] = self[j], self[i]
}

func (self byRemotePathLength) Less(i, j int) bool {
    return pathLength(self[i].relPath) < pathLength(self[j].relPath)
}

type ignoreFunc func(string) bool

func prepareIgnorer(path string) (ignoreFunc, error) {
    acceptAll := func(string) bool {
        return false
    }

    if !fileExists(path) {
        return acceptAll, nil
    }

    ignorer, err := ignore.CompileIgnoreFile(path)
    if err != nil {
        return acceptAll, fmt.Errorf("Failed to prepare ignorer: %s", err)
    }

    return ignorer.MatchesPath, nil
}
<|endoftext|>


<commit_before>package onedrive

import (
	"fmt"
	"reflect"
	"testing"
)

var expectedDefaultDrive = &Drive{
	ID:        "0123456789abc",
	DriveType: "personal",
	Owner: &IdentitySet{
		User: &Identity{
			DisplayName: "Gordan Grasarevic",
			ID:          "0123456789abc",
		},
	},
	Quota: &Quota{
		Deleted:   0,
		Remaining: 16095471537,
		State:     "normal",
		Total:     16106127360,
		Used:      10655823,
	},
}

func validFixtureFromDriveID(driveID string) string {
	switch driveID {
	case "", "root":
		return "fixtures/drive.valid.default.json"
	default:
		return fmt.Sprintf("fixtures/drive.valid.%s.json", driveID)
	}
}

func TestGet(t *testing.T) {
	setup()
	defer teardown()

	tt := []struct {
		driveID        string
		expectedPath   string
		expectedStatus int
		expectedOut    *Drive
	}{
		{"", "/drive", 200, expectedDefaultDrive},
		{"root", "/drive/root", 200, expectedDefaultDrive},
		{"test-id", "/drives/test-id", 200, &Drive{
			ID:        "test-id",
			DriveType: "consumer",
			Owner: &IdentitySet{
				Device: &Identity{
					DisplayName: "test app",
					ID:          "test-id",
				},
			},
			Quota: &Quota{
				Deleted: 1,
			},
		},
		},
	}
	for i, tst := range tt {
		if got, want := drivePathFromID(tst.driveID), tst.expectedPath; got != want {
			t.Errorf("Got %q Expected %q", got, want)
		}

		mux.HandleFunc(tst.expectedPath, fileWrapperHandler(validFixtureFromDriveID(tst.driveID)))
		drive, _, err := oneDrive.Drives.Get(tst.driveID)
		if err != nil {
			t.Fatalf("Problem fetching the default drive: %s", err.Error())
		}

		if !reflect.DeepEqual(drive, tst.expectedOut) {
			t.Errorf("[%d] Got %v Expected %v", i, drive, tst.expectedOut)
		}
	}
}

func TestGetDefaultDrive(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/drive", fileWrapperHandler("fixtures/drive.valid.default.json"))
	defaultDrive, resp, err := oneDrive.Drives.GetDefaultDrive()
	if err != nil {
		t.Fatalf("Problem fetching the default drive: %s", err.Error())
	}

	if resp.StatusCode != 200 {
		t.Fatalf("Expected response to be 200 got %d", resp.StatusCode)
	}

	if !reflect.DeepEqual(defaultDrive, expectedDefaultDrive) {
		t.Errorf("Got %v Expected %v", defaultDrive, expectedDefaultDrive)
	}
}

func TestGetRootDrive(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/drive/root", fileWrapperHandler("fixtures/drive.valid.default.json"))
	rootDrive, resp, err := oneDrive.Drives.GetRootDrive()
	if err != nil {
		t.Fatalf("Problem fetching the root drive: %s", err.Error())
	}

	if resp.StatusCode != 200 {
		t.Fatalf("Expected response to be 200 got %d", resp.StatusCode)
	}

	if !reflect.DeepEqual(rootDrive, expectedDefaultDrive) {
		t.Errorf("Got %v Expected %v", rootDrive, expectedDefaultDrive)
	}
}
<commit_msg>Add test for attempting to get a missing drive.<commit_after>package onedrive

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"reflect"
	"testing"
)

var expectedDefaultDrive = &Drive{
	ID:        "0123456789abc",
	DriveType: "personal",
	Owner: &IdentitySet{
		User: &Identity{
			DisplayName: "Gordan Grasarevic",
			ID:          "0123456789abc",
		},
	},
	Quota: &Quota{
		Deleted:   0,
		Remaining: 16095471537,
		State:     "normal",
		Total:     16106127360,
		Used:      10655823,
	},
}

func validFixtureFromDriveID(driveID string) string {
	switch driveID {
	case "", "root":
		return "fixtures/drive.valid.default.json"
	default:
		return fmt.Sprintf("fixtures/drive.valid.%s.json", driveID)
	}
}

func TestGet(t *testing.T) {
	setup()
	defer teardown()

	tt := []struct {
		driveID        string
		expectedPath   string
		expectedStatus int
		expectedOut    *Drive
	}{
		{"", "/drive", 200, expectedDefaultDrive},
		{"root", "/drive/root", 200, expectedDefaultDrive},
		{"test-id", "/drives/test-id", 200, &Drive{
			ID:        "test-id",
			DriveType: "consumer",
			Owner: &IdentitySet{
				Device: &Identity{
					DisplayName: "test app",
					ID:          "test-id",
				},
			},
			Quota: &Quota{
				Deleted: 1,
			},
		},
		},
	}
	for i, tst := range tt {
		if got, want := drivePathFromID(tst.driveID), tst.expectedPath; got != want {
			t.Errorf("Got %q Expected %q", got, want)
		}

		mux.HandleFunc(tst.expectedPath, fileWrapperHandler(validFixtureFromDriveID(tst.driveID)))
		drive, _, err := oneDrive.Drives.Get(tst.driveID)
		if err != nil {
			t.Fatalf("Problem fetching the default drive: %s", err.Error())
		}

		if !reflect.DeepEqual(drive, tst.expectedOut) {
			t.Errorf("[%d] Got %v Expected %v", i, drive, tst.expectedOut)
		}
	}
}

func TestGetMissing(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/drives/missing-drive", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(404)
		b, err := ioutil.ReadFile("fixtures/drive.invalid.missing.json")
		if err != nil {
			panic(err)
		}
		w.Write(b)
	})

	missingDrive, resp, err := oneDrive.Drives.Get("missing-drive")
	if missingDrive != nil {
		t.Fatalf("A drive was returned when an error was expected: %v", resp)
	}

	if resp.StatusCode != 404 {
		t.Fatalf("Expected response to be 404 got %d", resp.StatusCode)
	}

	expectedErr := &Error{
		InnerError{
			Code:    "itemNotFound",
			Message: "Item Does Not Exist",
			InnerError: &InnerError{
				Code: "itemDoesNotExist",
				InnerError: &InnerError{
					Code: "folderDoesNotExist",
				},
			},
		},
	}

	if !reflect.DeepEqual(expectedErr, err) {
		t.Errorf("Got %v Expected %v", err, expectedErr)
	}
}

func TestGetDefaultDrive(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/drive", fileWrapperHandler("fixtures/drive.valid.default.json"))
	defaultDrive, resp, err := oneDrive.Drives.GetDefaultDrive()
	if err != nil {
		t.Fatalf("Problem fetching the default drive: %s", err.Error())
	}

	if resp.StatusCode != 200 {
		t.Fatalf("Expected response to be 200 got %d", resp.StatusCode)
	}

	if !reflect.DeepEqual(defaultDrive, expectedDefaultDrive) {
		t.Errorf("Got %v Expected %v", defaultDrive, expectedDefaultDrive)
	}
}

func TestGetRootDrive(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/drive/root", fileWrapperHandler("fixtures/drive.valid.default.json"))
	rootDrive, resp, err := oneDrive.Drives.GetRootDrive()
	if err != nil {
		t.Fatalf("Problem fetching the root drive: %s", err.Error())
	}

	if resp.StatusCode != 200 {
		t.Fatalf("Expected response to be 200 got %d", resp.StatusCode)
	}

	if !reflect.DeepEqual(rootDrive, expectedDefaultDrive) {
		t.Errorf("Got %v Expected %v", rootDrive, expectedDefaultDrive)
	}
}
<|endoftext|>


<commit_before>package file

/*
 * Copyright 2016 Georg Großberger
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"encoding/xml"

	"github.com/garfieldius/t3ll/log"
	"github.com/kr/pretty"
)

type T3Root struct {
	XMLName string  `xml:"T3locallang"`
	Meta    *T3Meta `xml:"meta,omitempty"`
	Data    *T3Data `xml:"data"`
	Src     string  `xml:"-"`
}

type T3Meta struct {
	Typ  string `xml:"type,attr,omitempty"`
	For  string `xml:"type,omitempty"`
	Desc string `xml:"description,omitempty"`
}

type T3Data struct {
	Typ   string    `xml:"type,attr,omitempty"`
	Langs []*T3Lang `xml:"languageKey,omitempty"`
}

type T3Lang struct {
	Typ    string     `xml:"type,attr,omitempty"`
	Lang   string     `xml:"index,attr,omitempty"`
	Labels []*T3Label `xml:"label,omitempty"`
}

type T3Label struct {
	Key string `xml:"index,attr"`
	Cnt string `xml:",innerxml"`
}

func (l *T3Label) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	start.Attr = []xml.Attr{{Name: xml.Name{Local: "index"}, Value: l.Key}}
	e.EncodeToken(start)
	e.EncodeToken(xml.CharData(l.Cnt))
	e.EncodeToken(xml.EndElement{Name: start.Name})
	return nil
}

func (t T3Root) Labels() *Labels {
	data := &Labels{
		Type:     XmlLegacy,
		FromFile: t.Src,
		Langs:    make([]string, 0),
		Data:     make([]*Label, 0),
	}

	for _, lang := range t.Data.Langs {
		langKey := lang.Lang

		if langKey == "default" {
			langKey = "en"
		}

		data.Langs = append(data.Langs, langKey)

		for _, label := range lang.Labels {
			found := false
			trans := &Translation{Lng: langKey, Content: label.Cnt}

			for _, l := range data.Data {
				if l.Id == label.Key {
					l.Trans = append(l.Trans, trans)
					found = true
					break
				}
			}

			if !found {
				newTrans := &Label{Id: label.Key, Trans: []*Translation{trans}}
				data.Data = append(data.Data, newTrans)
			}
		}
	}

	log.Msg("Converted ll tree into %# v", pretty.Formatter(data))

	return data
}
<commit_msg>No empty <label> tags in .xml files<commit_after>package file

/*
 * Copyright 2016 Georg Großberger
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"encoding/xml"

	"github.com/garfieldius/t3ll/log"
	"github.com/kr/pretty"
)

type T3Root struct {
	XMLName string  `xml:"T3locallang"`
	Meta    *T3Meta `xml:"meta,omitempty"`
	Data    *T3Data `xml:"data"`
	Src     string  `xml:"-"`
}

type T3Meta struct {
	Typ  string `xml:"type,attr,omitempty"`
	For  string `xml:"type,omitempty"`
	Desc string `xml:"description,omitempty"`
}

type T3Data struct {
	Typ   string    `xml:"type,attr,omitempty"`
	Langs []*T3Lang `xml:"languageKey,omitempty"`
}

type T3Lang struct {
	Typ    string     `xml:"type,attr,omitempty"`
	Lang   string     `xml:"index,attr,omitempty"`
	Labels []*T3Label `xml:"label,omitempty"`
}

type T3Label struct {
	Key string `xml:"index,attr"`
	Cnt string `xml:",innerxml"`
}

func (l *T3Label) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	if l.Cnt != "" {
		start.Attr = []xml.Attr{{Name: xml.Name{Local: "index"}, Value: l.Key}}
		e.EncodeToken(start)
		e.EncodeToken(xml.CharData(l.Cnt))
		e.EncodeToken(xml.EndElement{Name: start.Name})
	}
	return nil
}

func (t T3Root) Labels() *Labels {
	data := &Labels{
		Type:     XmlLegacy,
		FromFile: t.Src,
		Langs:    make([]string, 0),
		Data:     make([]*Label, 0),
	}

	for _, lang := range t.Data.Langs {
		langKey := lang.Lang

		if langKey == "default" {
			langKey = "en"
		}

		data.Langs = append(data.Langs, langKey)

		for _, label := range lang.Labels {
			found := false
			trans := &Translation{Lng: langKey, Content: label.Cnt}

			for _, l := range data.Data {
				if l.Id == label.Key {
					l.Trans = append(l.Trans, trans)
					found = true
					break
				}
			}

			if !found {
				newTrans := &Label{Id: label.Key, Trans: []*Translation{trans}}
				data.Data = append(data.Data, newTrans)
			}
		}
	}

	log.Msg("Converted ll tree into %# v", pretty.Formatter(data))

	return data
}
<|endoftext|>


<commit_before>package flags

import (
	"bytes"
	"fmt"
	"testing"
	"time"
)

func TestDelayStr(t *testing.T) {
	for _, v := range []struct {
		dstr string
		cmp  string
		json []byte
	}{
		{"1h", "1h", []byte("\"1h\"")},
	} {
		td, err := time.ParseDuration(v.dstr)
		if err != nil {
			t.Error(err)
		}
		d := Delay{Duration: td}
		cmp := d.String()
		if cmp != v.cmp {
			t.Errorf("Mismatch: %q vs %q", cmp, v.cmp)
		}
		j, err := d.MarshalJSON()
		if err != nil {
			t.Error(err)
		}
		if !bytes.Equal(j, v.json) {
			t.Errorf("Mismatch: %q vs %q", j, v.json)
		}
	}
}

func TestDelaySet(t *testing.T) {
	for _, v := range []struct {
		dstr  string
		above string
		str   string
		err   error
	}{
		{"abc", "", "", fmt.Errorf("time: invalid duration abc")},
		{"0.5s", "", "", fmt.Errorf("Less than a second: 500ms")},
		{"1.5s", "", "", fmt.Errorf("Not a multiple of a second: 1.5s")},
		{"1s1s", "", "2s", nil},
		{"3s", "4s", "", fmt.Errorf("Should be above 4s: 3s")},
	} {
		td, err := time.ParseDuration(v.dstr)
		if err != nil && err.Error() != v.err.Error() {
			t.Error(err)
		}
		d := Delay{Duration: td}
		if v.above != "" {
			if ad, err := time.ParseDuration(v.above); err != nil {
				t.Error(err)
			} else {
				d.Above = &ad
			}
		}
		err = d.Set(v.dstr)
		if err != nil {
			if err.Error() == v.err.Error() {
				continue
			}
			t.Error(err)
		}
		if d.String() != v.str {
			t.Errorf("Mismatch: %q vs %q", d.String(), v.str)
		}
	}
}

func TestBindSet(t *testing.T) {
	for _, v := range []struct {
		a    string
		cmp  string
		errs []error
	}{
		{"a:b:", "", []error{fmt.Errorf("too many colons in address " + "a:b:")}},
		{"localhost:nonexistent", "localhost:nonexistent", []error{
			fmt.Errorf("lookup tcp/nonexistent: Servname not supported for ai_socktype"),
			fmt.Errorf("lookup tcp/nonexistent: nodename nor servname provided, or not known"),
		}},
		{"localhost", "localhost:9050", nil},
		{"", ":9050", nil},
		{":8001", ":8001", nil},
		{"*:8001", "*:8001", nil},
		{"127", "127:9050", nil},
		{"127.1", "127.1:9050", nil},
		{"127.0.0.1", "127.0.0.1:9050", nil},
		{"127.1:8001", "127.1:8001", nil},
		{"127.0.0.1:8001", "127.0.0.1:8001", nil},
	} {
		b := NewBind(9050)
		if err := b.Set(v.a); err != nil {
			unknownerr := true
			for _, x := range v.errs {
				if err.Error() == x.Error() {
					unknownerr = false
					break
				}
			}
			if !unknownerr {
				continue
			}
			t.Errorf("Error: %q\nExpected errors: %+v\n", err, v.errs)
		}
		if b.string != v.cmp {
			t.Errorf("Mismatch: Bind %v == %v != %v\n", v.a, v.cmp, b.string)
		}
	}
}
<commit_msg>bind flag test: expect error on invalid port input<commit_after>package flags

import (
	"bytes"
	"fmt"
	"testing"
	"time"
)

func TestDelayStr(t *testing.T) {
	for _, v := range []struct {
		dstr string
		cmp  string
		json []byte
	}{
		{"1h", "1h", []byte("\"1h\"")},
	} {
		td, err := time.ParseDuration(v.dstr)
		if err != nil {
			t.Error(err)
		}
		d := Delay{Duration: td}
		cmp := d.String()
		if cmp != v.cmp {
			t.Errorf("Mismatch: %q vs %q", cmp, v.cmp)
		}
		j, err := d.MarshalJSON()
		if err != nil {
			t.Error(err)
		}
		if !bytes.Equal(j, v.json) {
			t.Errorf("Mismatch: %q vs %q", j, v.json)
		}
	}
}

func TestDelaySet(t *testing.T) {
	for _, v := range []struct {
		dstr  string
		above string
		str   string
		err   error
	}{
		{"abc", "", "", fmt.Errorf("time: invalid duration abc")},
		{"0.5s", "", "", fmt.Errorf("Less than a second: 500ms")},
		{"1.5s", "", "", fmt.Errorf("Not a multiple of a second: 1.5s")},
		{"1s1s", "", "2s", nil},
		{"3s", "4s", "", fmt.Errorf("Should be above 4s: 3s")},
	} {
		td, err := time.ParseDuration(v.dstr)
		if err != nil && err.Error() != v.err.Error() {
			t.Error(err)
		}
		d := Delay{Duration: td}
		if v.above != "" {
			if ad, err := time.ParseDuration(v.above); err != nil {
				t.Error(err)
			} else {
				d.Above = &ad
			}
		}
		err = d.Set(v.dstr)
		if err != nil {
			if err.Error() == v.err.Error() {
				continue
			}
			t.Error(err)
		}
		if d.String() != v.str {
			t.Errorf("Mismatch: %q vs %q", d.String(), v.str)
		}
	}
}

func TestBindSet(t *testing.T) {
	for _, v := range []struct {
		a    string
		cmp  string
		errs []error
	}{
		{"a:b:", "", []error{fmt.Errorf("too many colons in address " + "a:b:")}},
		{"localhost:nonexistent", "localhost:nonexistent", []error{
			fmt.Errorf("unknown port tcp/nonexistent"),
			fmt.Errorf("lookup tcp/nonexistent: Servname not supported for ai_socktype"),
			fmt.Errorf("lookup tcp/nonexistent: nodename nor servname provided, or not known"),
		}},
		{"localhost", "localhost:9050", nil},
		{"", ":9050", nil},
		{":8001", ":8001", nil},
		{"*:8001", "*:8001", nil},
		{"127", "127:9050", nil},
		{"127.1", "127.1:9050", nil},
		{"127.0.0.1", "127.0.0.1:9050", nil},
		{"127.1:8001", "127.1:8001", nil},
		{"127.0.0.1:8001", "127.0.0.1:8001", nil},
	} {
		b := NewBind(9050)
		if err := b.Set(v.a); err != nil {
			unknownerr := true
			for _, x := range v.errs {
				if err.Error() == x.Error() {
					unknownerr = false
					break
				}
			}
			if !unknownerr {
				continue
			}
			t.Errorf("Error: %q\nExpected errors: %+v\n", err, v.errs)
		}
		if b.string != v.cmp {
			t.Errorf("Mismatch: Bind %v == %v != %v\n", v.a, v.cmp, b.string)
		}
	}
}
<|endoftext|>


<commit_before>package engine

import (
	"html/template"
	"net/http"

	"path/filepath"

	log "github.com/sirupsen/logrus"
	"github.com/xyproto/algernon/lua/codelib"
	"github.com/xyproto/algernon/lua/convert"
	"github.com/xyproto/algernon/lua/datastruct"
	"github.com/xyproto/algernon/lua/httpclient"
	"github.com/xyproto/algernon/lua/jnode"
	"github.com/xyproto/algernon/lua/mssql"
	"github.com/xyproto/algernon/lua/onthefly"
	"github.com/xyproto/algernon/lua/pquery"
	"github.com/xyproto/algernon/lua/pure"
	"github.com/xyproto/algernon/lua/upload"
	"github.com/xyproto/algernon/lua/users"
	"github.com/xyproto/algernon/utils"
	"github.com/xyproto/gluamapper"
	"github.com/xyproto/gopher-lua"
)

// LoadCommonFunctions adds most of the available Lua functions in algernon to
// the given Lua state struct
func (ac *Config) LoadCommonFunctions(w http.ResponseWriter, req *http.Request, filename string, L *lua.LState, flushFunc func(), httpStatus *FutureStatus) {

	// Make basic functions, like print, available to the Lua script.
	// Only exports functions that can relate to HTTP responses or requests.
	ac.LoadBasicWeb(w, req, L, filename, flushFunc, httpStatus)

	// Make other basic functions available
	ac.LoadBasicSystemFunctions(L)

	// Functions for rendering markdown or amber
	ac.LoadRenderFunctions(w, req, L)

	// If there is a database backend
	if ac.perm != nil {

		// Retrieve the userstate
		userstate := ac.perm.UserState()

		// Set the cookie secret, if set
		if ac.cookieSecret != "" {
			userstate.SetCookieSecret(ac.cookieSecret)
		}

		// Functions for serving files in the same directory as a script
		ac.LoadServeFile(w, req, L, filename)

		// Functions mainly for adding admin prefixes and configuring permissions
		ac.LoadServerConfigFunctions(L, filename)

		// Make the functions related to userstate available to the Lua script
		users.Load(w, req, L, userstate)

		creator := userstate.Creator()

		// Simpleredis data structures
		datastruct.LoadList(L, creator)
		datastruct.LoadSet(L, creator)
		datastruct.LoadHash(L, creator)
		datastruct.LoadKeyValue(L, creator)

		// For saving and loading Lua functions
		codelib.Load(L, creator)

		// For executing PostgreSQL queries
		pquery.Load(L, ac.perm)

		// For executing MSSQL queries
		mssql.Load(L, ac.perm)

	}

	// For handling JSON data
	jnode.LoadJSONFunctions(L)
	ac.LoadJFile(L, filepath.Dir(filename))
	jnode.Load(L)

	// Extras
	pure.Load(L)

	// pprint
	//exportREPL(L)

	// Plugins
	ac.LoadPluginFunctions(L, nil)

	// Cache
	ac.LoadCacheFunctions(L)

	// Pages and Tags
	onthefly.Load(L)

	// File uploads
	upload.Load(L, w, req, filepath.Dir(filename))

	// HTTP Client
	httpclient.Load(L, ac.serverHeaderName)
}

// RunLua uses a Lua file as the HTTP handler. Also has access to the userstate
// and permissions. Returns an error if there was a problem with running the lua
// script, otherwise nil.
func (ac *Config) RunLua(w http.ResponseWriter, req *http.Request, filename string, flushFunc func(), fust *FutureStatus) error {

	// Retrieve a Lua state
	L := ac.luapool.Get()
	defer ac.luapool.Put(L)

	// Warn if the connection is closed before the script has finished.
	// Requires that the requestWriter has CloseNotify.
	if ac.verboseMode {

		done := make(chan bool)

		// Stop the background goroutine when this function returns
		// There must be a receiver for the done channel,
		// or else this will hang everything!
		defer func() {
			_, ok := w.(http.CloseNotifier)
			if ok {
				// Only do this is the select case below is sure to be running!
				done <- true
			}
		}()

		// Set up a background notifier
		go func() {
			wCloseNotify, ok := w.(http.CloseNotifier)
			if !ok {
				//log.Error("ResponseWriter has no CloseNotify()!")
				return
			}
			for {
				select {
				case <-wCloseNotify.CloseNotify():
					// Client is done
					log.Warn("Connection to client closed")
				case <-done:
					// We are done
					return
				}
			}
		}() // Call the goroutine
	}

	// Export functions to the Lua state
	// Flush can be an uninitialized channel, it is handled in the function.
	ac.LoadCommonFunctions(w, req, filename, L, flushFunc, fust)

	// Run the script and return the error value.
	// Logging and/or HTTP response is handled elsewhere.
	return L.DoFile(filename)
}

// RunConfiguration runs a Lua file as a configuration script. Also has access
// to the userstate and permissions. Returns an error if there was a problem
// with running the lua script, otherwise nil. perm can be nil, but then several
// Lua functions will not be exposed.
//
// The idea is not to change the Lua struct or the luapool, but to set the
// configuration variables with the given Lua configuration script.
//
// luaHandler is a flag that lets Lua functions like "handle" and "servedir" be available or not.
func (ac *Config) RunConfiguration(filename string, mux *http.ServeMux, withHandlerFunctions bool) error {

	// Retrieve a Lua state
	L := ac.luapool.Get()

	// Basic system functions, like log()
	ac.LoadBasicSystemFunctions(L)

	// If there is a database backend
	if ac.perm != nil {

		// Retrieve the userstate
		userstate := ac.perm.UserState()

		// Server configuration functions
		ac.LoadServerConfigFunctions(L, filename)

		creator := userstate.Creator()

		// Simpleredis data structures (could be used for storing server stats)
		datastruct.LoadList(L, creator)
		datastruct.LoadSet(L, creator)
		datastruct.LoadHash(L, creator)
		datastruct.LoadKeyValue(L, creator)

		// For saving and loading Lua functions
		codelib.Load(L, creator)

		// For executing PostgreSQL queries
		pquery.Load(L, ac.perm)

		// For executing MSSQL queries
		mssql.Load(L, ac.perm)
	}

	// For handling JSON data
	jnode.LoadJSONFunctions(L)
	ac.LoadJFile(L, filepath.Dir(filename))
	jnode.Load(L)

	// Extras
	pure.Load(L)

	// Plugins
	ac.LoadPluginFunctions(L, nil)

	// Cache
	ac.LoadCacheFunctions(L)

	// Pages and Tags
	onthefly.Load(L)

	// HTTP Client
	httpclient.Load(L, ac.serverHeaderName)

	if withHandlerFunctions {
		// Lua HTTP handlers
		ac.LoadLuaHandlerFunctions(L, filename, mux, false, nil, ac.defaultTheme)
	}

	// Run the script
	if err := L.DoFile(filename); err != nil {
		// Close the Lua state
		L.Close()

		// Logging and/or HTTP response is handled elsewhere
		return err
	}

	// Only put the Lua state back if there were no errors
	ac.luapool.Put(L)

	return nil
}

/*LuaFunctionMap returns the functions available in the given Lua code as
 * functions in a map that can be used by templates.
 *
 * Note that the lua functions must only accept and return strings
 * and that only the first returned value will be accessible.
 * The Lua functions may take an optional number of arguments.
 */
func (ac *Config) LuaFunctionMap(w http.ResponseWriter, req *http.Request, luadata []byte, filename string) (template.FuncMap, error) {
	ac.pongomutex.Lock()
	defer ac.pongomutex.Unlock()

	// Retrieve a Lua state
	L := ac.luapool.Get()
	defer ac.luapool.Put(L)

	// Prepare an empty map of functions (and variables)
	funcs := make(template.FuncMap)

	// Give no filename (an empty string will be handled correctly by the function).
	ac.LoadCommonFunctions(w, req, filename, L, nil, nil)

	// Run the script
	if err := L.DoString(string(luadata)); err != nil {
		// Close the Lua state
		L.Close()

		// Logging and/or HTTP response is handled elsewhere
		return funcs, err
	}

	// Extract the available functions from the Lua state
	globalTable := L.G.Global
	globalTable.ForEach(func(key, value lua.LValue) {

		// Check if the current value is a string variable
		if luaString, ok := value.(lua.LString); ok {

			// Store the variable in the same map as the functions (string -> interface)
			// for ease of use together with templates.
			funcs[key.String()] = luaString.String()

		} else if luaTable, ok := value.(*lua.LTable); ok {

			// Convert the table to a map and save it.
			// Ignore values of a different type.
			mapinterface, _ := convert.Table2map(luaTable, false)
			switch m := mapinterface.(type) {
			case map[string]string:
				funcs[key.String()] = map[string]string(m)
			case map[string]int:
				funcs[key.String()] = map[string]int(m)
			case map[int]string:
				funcs[key.String()] = map[int]string(m)
			case map[int]int:
				funcs[key.String()] = map[int]int(m)
			}

			// Check if the current value is a function
		} else if luaFunc, ok := value.(*lua.LFunction); ok {

			// Only export the functions defined in the given Lua code,
			// not all the global functions. IsG is true if the function is global.
			if !luaFunc.IsG {

				functionName := key.String()

				// Register the function, with a variable number of string arguments
				// Functions returning (string, error) are supported by html.template
				funcs[functionName] = func(args ...string) (interface{}, error) {

					// Create a brand new Lua state
					L2 := ac.luapool.New()
					defer L2.Close()

					// Set up a new Lua state with the current http.ResponseWriter and *http.Request
					ac.LoadCommonFunctions(w, req, filename, L2, nil, nil)

					// Push the Lua function to run
					L2.Push(luaFunc)

					// Push the given arguments
					for _, arg := range args {
						L2.Push(lua.LString(arg))
					}

					// Run the Lua function
					err := L2.PCall(len(args), lua.MultRet, nil)
					if err != nil {
						// If calling the function did not work out, return the infostring and error
						return utils.Infostring(functionName, args), err
					}

					// Empty return value if no values were returned
					var retval interface{}

					// Return the first of the returned arguments, as a string
					if L2.GetTop() >= 1 {
						lv := L2.Get(-1)
						tbl, isTable := lv.(*lua.LTable)
						switch {
						case isTable:
							// lv was a Lua Table
							retval = gluamapper.ToGoValue(tbl, gluamapper.Option{
								NameFunc: func(s string) string {
									return s
								},
							})
							if ac.debugMode && ac.verboseMode {
								log.Info(utils.Infostring(functionName, args) + " -> (map)")
							}
						case lv.Type() == lua.LTString:
							// lv is a Lua String
							retstr := L2.ToString(1)
							retval = retstr
							if ac.debugMode && ac.verboseMode {
								log.Info(utils.Infostring(functionName, args) + " -> \"" + retstr + "\"")
							}
						default:
							retval = ""
							log.Warn("The return type of " + utils.Infostring(functionName, args) + " can't be converted")
						}
					}

					// No return value, return an empty string and nil
					return retval, nil
				}
			}
		}
	})

	// Return the map of functions
	return funcs, nil
}
<commit_msg>Use req.Context instead of deprecated CloseNotifier<commit_after>package engine

import (
	"html/template"
	"net/http"

	"path/filepath"

	log "github.com/sirupsen/logrus"
	"github.com/xyproto/algernon/lua/codelib"
	"github.com/xyproto/algernon/lua/convert"
	"github.com/xyproto/algernon/lua/datastruct"
	"github.com/xyproto/algernon/lua/httpclient"
	"github.com/xyproto/algernon/lua/jnode"
	"github.com/xyproto/algernon/lua/mssql"
	"github.com/xyproto/algernon/lua/onthefly"
	"github.com/xyproto/algernon/lua/pquery"
	"github.com/xyproto/algernon/lua/pure"
	"github.com/xyproto/algernon/lua/upload"
	"github.com/xyproto/algernon/lua/users"
	"github.com/xyproto/algernon/utils"
	"github.com/xyproto/gluamapper"
	lua "github.com/xyproto/gopher-lua"
)

// LoadCommonFunctions adds most of the available Lua functions in algernon to
// the given Lua state struct
func (ac *Config) LoadCommonFunctions(w http.ResponseWriter, req *http.Request, filename string, L *lua.LState, flushFunc func(), httpStatus *FutureStatus) {

	// Make basic functions, like print, available to the Lua script.
	// Only exports functions that can relate to HTTP responses or requests.
	ac.LoadBasicWeb(w, req, L, filename, flushFunc, httpStatus)

	// Make other basic functions available
	ac.LoadBasicSystemFunctions(L)

	// Functions for rendering markdown or amber
	ac.LoadRenderFunctions(w, req, L)

	// If there is a database backend
	if ac.perm != nil {

		// Retrieve the userstate
		userstate := ac.perm.UserState()

		// Set the cookie secret, if set
		if ac.cookieSecret != "" {
			userstate.SetCookieSecret(ac.cookieSecret)
		}

		// Functions for serving files in the same directory as a script
		ac.LoadServeFile(w, req, L, filename)

		// Functions mainly for adding admin prefixes and configuring permissions
		ac.LoadServerConfigFunctions(L, filename)

		// Make the functions related to userstate available to the Lua script
		users.Load(w, req, L, userstate)

		creator := userstate.Creator()

		// Simpleredis data structures
		datastruct.LoadList(L, creator)
		datastruct.LoadSet(L, creator)
		datastruct.LoadHash(L, creator)
		datastruct.LoadKeyValue(L, creator)

		// For saving and loading Lua functions
		codelib.Load(L, creator)

		// For executing PostgreSQL queries
		pquery.Load(L, ac.perm)

		// For executing MSSQL queries
		mssql.Load(L, ac.perm)

	}

	// For handling JSON data
	jnode.LoadJSONFunctions(L)
	ac.LoadJFile(L, filepath.Dir(filename))
	jnode.Load(L)

	// Extras
	pure.Load(L)

	// pprint
	//exportREPL(L)

	// Plugins
	ac.LoadPluginFunctions(L, nil)

	// Cache
	ac.LoadCacheFunctions(L)

	// Pages and Tags
	onthefly.Load(L)

	// File uploads
	upload.Load(L, w, req, filepath.Dir(filename))

	// HTTP Client
	httpclient.Load(L, ac.serverHeaderName)
}

// RunLua uses a Lua file as the HTTP handler. Also has access to the userstate
// and permissions. Returns an error if there was a problem with running the lua
// script, otherwise nil.
func (ac *Config) RunLua(w http.ResponseWriter, req *http.Request, filename string, flushFunc func(), fust *FutureStatus) error {

	// Retrieve a Lua state
	L := ac.luapool.Get()
	defer ac.luapool.Put(L)

	// Warn if the connection is closed before the script has finished.
	// Requires that the requestWriter has CloseNotify.
	if ac.verboseMode {

		done := make(chan bool)

		// Stop the background goroutine when this function returns
		// There must be a receiver for the done channel,
		// or else this will hang everything!
		defer func() {
			done <- true
		}()

		// Set up a background notifier
		go func() {
			ctx := req.Context()
			for {
				select {
				case <-ctx.Done():
					// Client is done
					log.Warn("Connection to client closed")
				case <-done:
					// We are done
					return
				}
			}
		}() // Call the goroutine
	}

	// Export functions to the Lua state
	// Flush can be an uninitialized channel, it is handled in the function.
	ac.LoadCommonFunctions(w, req, filename, L, flushFunc, fust)

	// Run the script and return the error value.
	// Logging and/or HTTP response is handled elsewhere.
	return L.DoFile(filename)
}

// RunConfiguration runs a Lua file as a configuration script. Also has access
// to the userstate and permissions. Returns an error if there was a problem
// with running the lua script, otherwise nil. perm can be nil, but then several
// Lua functions will not be exposed.
//
// The idea is not to change the Lua struct or the luapool, but to set the
// configuration variables with the given Lua configuration script.
//
// luaHandler is a flag that lets Lua functions like "handle" and "servedir" be available or not.
func (ac *Config) RunConfiguration(filename string, mux *http.ServeMux, withHandlerFunctions bool) error {

	// Retrieve a Lua state
	L := ac.luapool.Get()

	// Basic system functions, like log()
	ac.LoadBasicSystemFunctions(L)

	// If there is a database backend
	if ac.perm != nil {

		// Retrieve the userstate
		userstate := ac.perm.UserState()

		// Server configuration functions
		ac.LoadServerConfigFunctions(L, filename)

		creator := userstate.Creator()

		// Simpleredis data structures (could be used for storing server stats)
		datastruct.LoadList(L, creator)
		datastruct.LoadSet(L, creator)
		datastruct.LoadHash(L, creator)
		datastruct.LoadKeyValue(L, creator)

		// For saving and loading Lua functions
		codelib.Load(L, creator)

		// For executing PostgreSQL queries
		pquery.Load(L, ac.perm)

		// For executing MSSQL queries
		mssql.Load(L, ac.perm)
	}

	// For handling JSON data
	jnode.LoadJSONFunctions(L)
	ac.LoadJFile(L, filepath.Dir(filename))
	jnode.Load(L)

	// Extras
	pure.Load(L)

	// Plugins
	ac.LoadPluginFunctions(L, nil)

	// Cache
	ac.LoadCacheFunctions(L)

	// Pages and Tags
	onthefly.Load(L)

	// HTTP Client
	httpclient.Load(L, ac.serverHeaderName)

	if withHandlerFunctions {
		// Lua HTTP handlers
		ac.LoadLuaHandlerFunctions(L, filename, mux, false, nil, ac.defaultTheme)
	}

	// Run the script
	if err := L.DoFile(filename); err != nil {
		// Close the Lua state
		L.Close()

		// Logging and/or HTTP response is handled elsewhere
		return err
	}

	// Only put the Lua state back if there were no errors
	ac.luapool.Put(L)

	return nil
}

/*LuaFunctionMap returns the functions available in the given Lua code as
 * functions in a map that can be used by templates.
 *
 * Note that the lua functions must only accept and return strings
 * and that only the first returned value will be accessible.
 * The Lua functions may take an optional number of arguments.
 */
func (ac *Config) LuaFunctionMap(w http.ResponseWriter, req *http.Request, luadata []byte, filename string) (template.FuncMap, error) {
	ac.pongomutex.Lock()
	defer ac.pongomutex.Unlock()

	// Retrieve a Lua state
	L := ac.luapool.Get()
	defer ac.luapool.Put(L)

	// Prepare an empty map of functions (and variables)
	funcs := make(template.FuncMap)

	// Give no filename (an empty string will be handled correctly by the function).
	ac.LoadCommonFunctions(w, req, filename, L, nil, nil)

	// Run the script
	if err := L.DoString(string(luadata)); err != nil {
		// Close the Lua state
		L.Close()

		// Logging and/or HTTP response is handled elsewhere
		return funcs, err
	}

	// Extract the available functions from the Lua state
	globalTable := L.G.Global
	globalTable.ForEach(func(key, value lua.LValue) {

		// Check if the current value is a string variable
		if luaString, ok := value.(lua.LString); ok {

			// Store the variable in the same map as the functions (string -> interface)
			// for ease of use together with templates.
			funcs[key.String()] = luaString.String()

		} else if luaTable, ok := value.(*lua.LTable); ok {

			// Convert the table to a map and save it.
			// Ignore values of a different type.
			mapinterface, _ := convert.Table2map(luaTable, false)
			switch m := mapinterface.(type) {
			case map[string]string:
				funcs[key.String()] = map[string]string(m)
			case map[string]int:
				funcs[key.String()] = map[string]int(m)
			case map[int]string:
				funcs[key.String()] = map[int]string(m)
			case map[int]int:
				funcs[key.String()] = map[int]int(m)
			}

			// Check if the current value is a function
		} else if luaFunc, ok := value.(*lua.LFunction); ok {

			// Only export the functions defined in the given Lua code,
			// not all the global functions. IsG is true if the function is global.
			if !luaFunc.IsG {

				functionName := key.String()

				// Register the function, with a variable number of string arguments
				// Functions returning (string, error) are supported by html.template
				funcs[functionName] = func(args ...string) (interface{}, error) {

					// Create a brand new Lua state
					L2 := ac.luapool.New()
					defer L2.Close()

					// Set up a new Lua state with the current http.ResponseWriter and *http.Request
					ac.LoadCommonFunctions(w, req, filename, L2, nil, nil)

					// Push the Lua function to run
					L2.Push(luaFunc)

					// Push the given arguments
					for _, arg := range args {
						L2.Push(lua.LString(arg))
					}

					// Run the Lua function
					err := L2.PCall(len(args), lua.MultRet, nil)
					if err != nil {
						// If calling the function did not work out, return the infostring and error
						return utils.Infostring(functionName, args), err
					}

					// Empty return value if no values were returned
					var retval interface{}

					// Return the first of the returned arguments, as a string
					if L2.GetTop() >= 1 {
						lv := L2.Get(-1)
						tbl, isTable := lv.(*lua.LTable)
						switch {
						case isTable:
							// lv was a Lua Table
							retval = gluamapper.ToGoValue(tbl, gluamapper.Option{
								NameFunc: func(s string) string {
									return s
								},
							})
							if ac.debugMode && ac.verboseMode {
								log.Info(utils.Infostring(functionName, args) + " -> (map)")
							}
						case lv.Type() == lua.LTString:
							// lv is a Lua String
							retstr := L2.ToString(1)
							retval = retstr
							if ac.debugMode && ac.verboseMode {
								log.Info(utils.Infostring(functionName, args) + " -> \"" + retstr + "\"")
							}
						default:
							retval = ""
							log.Warn("The return type of " + utils.Infostring(functionName, args) + " can't be converted")
						}
					}

					// No return value, return an empty string and nil
					return retval, nil
				}
			}
		}
	})

	// Return the map of functions
	return funcs, nil
}
<|endoftext|>


<commit_before>package stringsutil

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
)

// Unshift adds an element at the first position
// of the slice.
func Unshift(a []string, x string) []string {
	return append([]string{x}, a...)
}

// SliceCondenseSpace trims space from lines and removes
// empty lines.
func SliceCondenseSpace(lines []string) []string {
	return SliceLinesTrimSpace(lines, true)
}

// SliceLinesTrimSpace removes leading and trailing spaces per
// string and optionally removes empty strings.
func SliceLinesTrimSpace(lines []string, condense bool) []string {
	return SliceLinesTrim(lines, " ", condense)
}

// SliceLinesTrim trims each line in a slice of lines using a
// provided cut string.
func SliceLinesTrim(lines []string, cutstr string, condense bool) []string {
	for i, line := range lines {
		line = strings.Trim(line, cutstr)
		if condense && len(line) == 0 {
			continue
		}
		lines[i] = line
	}
	return lines
}

// SliceIndexOrEmpty returns the element at the index
// provided or an empty string.
func SliceIndexOrEmpty(s []string, index uint64) string {
	if int(index) >= len(s) {
		return ""
	}
	return s[index]
}

// JoinAny takes an array of interface{} and converts
// each value to a string using fmt.Sprintf("%v")
func JoinAny(a []interface{}, sep string) string {
	strs := []string{}
	for _, item := range a {
		strs = append(strs, fmt.Sprintf("%v", item))
	}
	return strings.Join(strs, sep)
}

func JoinInt(a []int, sep string) string {
	strs := []string{}
	for _, i := range a {
		strs = append(strs, strconv.Itoa(i))
	}
	return strings.Join(strs, sep)
}

func JoinCondenseTrimSpace(slice []string, sep string) string {
	return strings.Join(SliceLinesTrimSpace(slice, true), sep)
}

func SliceCondenseRegexps(texts []string, regexps []*regexp.Regexp, replacement string) []string {
	parts := []string{}
	for _, part := range texts {
		for _, rx := range regexps {
			part = rx.ReplaceAllString(part, replacement)
		}
		part = strings.TrimSpace(part)
		if len(part) > 0 {
			parts = append(parts, part)
		}
	}
	return parts
}

func SliceCondensePunctuation(texts []string) []string {
	parts := []string{}
	for _, part := range texts {
		part = regexp.MustCompile(`[^a-zA-Z0-9]+`).ReplaceAllString(part, " ")
		part = regexp.MustCompile(`\s+`).ReplaceAllString(part, " ")
		part = strings.TrimSpace(part)
		if len(part) > 0 {
			parts = append(parts, part)
		}
	}
	return parts
}

func SliceCondenseAndQuoteSpace(items []string, quoteLeft, quoteRight string) []string {
	return SliceCondenseAndQuote(items, " ", " ", quoteLeft, quoteRight)
}

func SliceCondenseAndQuote(items []string, trimLeft, trimRight, quoteLeft, quoteRight string) []string {
	newItems := []string{}
	for _, item := range items {
		item = strings.TrimLeft(item, trimLeft)
		item = strings.TrimRight(item, trimRight)
		if len(item) > 0 {
			item = quoteLeft + item + quoteRight
			newItems = append(newItems, item)
		}
	}
	return newItems
}

// SplitTrimSpace splits a string and trims spaces on
// remaining elements.
func SplitTrimSpace(s, sep string) []string {
	split := strings.Split(s, sep)
	strs := []string{}
	for _, str := range split {
		strs = append(strs, strings.TrimSpace(str))
	}
	return strs
}

// SliceToSingleIntOrNeg converts a single element slice
// with a string to an integer or `-1`
func SliceToSingleIntOrNeg(vals []string) int {
	if len(vals) != 1 {
		return -1
	}
	num, err := strconv.Atoi(vals[0])
	if err != nil {
		return -1
	}
	return num
}

// Dedupe returns a string slice with duplicate values
// removed. First observance is kept.
func Dedupe(vals []string) []string {
	deduped := []string{}
	seen := map[string]int{}

	for _, val := range vals {
		if _, ok := seen[val]; ok {
			continue
		}
		seen[val] = 1
		deduped = append(deduped, val)
	}
	return deduped
}

// SliceIndexOf returns the index of an element in a
// string slice. Returns -1 if not found.
func SliceIndexOf(needle string, haystack []string) int {
	for k, v := range haystack {
		if v == needle {
			return k
		}
	}
	return -1 //not found.
}

// SliceIndexOfLcTrimSpace returns the index of an element in a
// string slice. Returns -1 if not found.
func SliceIndexOfLcTrimSpace(needle string, haystack []string) int {
	needle = strings.ToLower(strings.TrimSpace(needle))
	for k, v := range haystack {
		v = strings.ToLower(strings.TrimSpace(v))
		if v == needle {
			return k
		}
	}
	return -1 //not found.
}

func SliceChooseOnePreferredLowerTrimSpace(options, preferenceOrder []string) string {
	if len(options) == 0 {
		return ""
	} else if len(preferenceOrder) == 0 {
		return strings.ToLower(strings.TrimSpace(options[0]))
	}
	optMap := map[string]int{}
	for _, opt := range options {
		opt = strings.ToLower(strings.TrimSpace(opt))
		if len(opt) > 0 {
			optMap[opt] = 1
		}
	}
	for _, pref := range preferenceOrder {
		pref = strings.ToLower(strings.TrimSpace(pref))
		if _, ok := optMap[pref]; ok {
			return pref
		}
	}
	return strings.ToLower(strings.TrimSpace(options[0]))
}

func SliceJoinQuotedMaxLength(slice []string, begQuote, endQuote, sep string, maxLength int) []string {
	words := []string{}
	curWords := []string{}
	curLength := 0
	for _, word := range slice {
		if curLength+len(begQuote+word+endQuote+sep) > maxLength {
			words = append(words, strings.Join(curWords, sep))
			curWords = []string{}
			curLength = 0
		} else {
			curWords = append(curWords, begQuote+word+endQuote)
			curLength += len(begQuote + word + endQuote + sep)
		}
	}
	if len(curWords) > 0 {
		words = append(words, strings.Join(curWords, sep))
	}
	return words
}

type JoinCustomConfig struct {
	QuoteBegin string
	QuoteEnd   string
	Separator  string
	MaxLength  int
	TrimSpace  bool
	SkipEmpty  bool
}

func JoinCustom(slice []string, cfg JoinCustomConfig) []string {
	lines := []string{}
	curWords := []string{}
	curLength := 0
	for _, word := range slice {
		if cfg.TrimSpace {
			word = strings.TrimSpace(word)
		}
		if cfg.SkipEmpty && len(word) == 0 {
			continue
		}
		if cfg.MaxLength > 0 {
			if curLength+len(cfg.QuoteBegin+word+cfg.QuoteEnd+cfg.Separator) > cfg.MaxLength {
				lines = append(lines, strings.Join(curWords, cfg.Separator))
				curWords = []string{}
				curLength = 0
			} else {
				curWords = append(curWords, cfg.QuoteBegin+word+cfg.QuoteEnd)
				curLength += len(cfg.QuoteBegin + word + cfg.QuoteEnd + cfg.Separator)
			}
		} else {
			curWords = append(curWords, cfg.QuoteBegin+word+cfg.QuoteEnd)
		}
	}
	if len(curWords) > 0 {
		lines = append(lines, strings.Join(curWords, cfg.Separator))
	}
	return lines
}

func SliceJoinQuotedMaxLengthTrimSpaceSkipEmpty(slice []string, begQuote, endQuote, sep string, maxLength int) []string {
	words := []string{}
	curWords := []string{}
	curLength := 0
	for _, word := range slice {
		if curLength+len(begQuote+word+endQuote+sep) > maxLength {
			words = append(words, strings.Join(curWords, sep))
			curWords = []string{}
			curLength = 0
		} else {
			curWords = append(curWords, begQuote+word+endQuote)
			curLength += len(begQuote + word + endQuote + sep)
		}
	}
	if len(curWords) > 0 {
		words = append(words, strings.Join(curWords, sep))
	}
	return words
}

func SliceJoinQuoted(slice []string, begQuote, endQuote, sep string) string {
	words := []string{}
	for _, word := range slice {
		words = append(words, begQuote+word+endQuote)
	}
	return strings.Join(words, sep)
}
<commit_msg>add stringsutil.Reemove() and update stringsutil.SliceCondenseSpace<commit_after>package stringsutil

import (
	"fmt"
	"regexp"
	"sort"
	"strconv"
	"strings"
)

// Unshift adds an element at the first position
// of the slice.
func Unshift(a []string, x string) []string {
	return append([]string{x}, a...)
}

// SliceCondenseSpace trims space from lines and removes
// empty lines. `unique` dedupes lines and `sort` preforms
// a sort on the results.
func SliceCondenseSpace(lines []string, dedupeResults, sortResults bool) []string {
	results := SliceLinesTrimSpace(lines, true)
	if dedupeResults {
		results = Dedupe(results)
	}
	if sortResults {
		sort.Strings(results)
	}
	return results
}

// SliceLinesTrimSpace removes leading and trailing spaces per
// string and optionally removes empty strings.
func SliceLinesTrimSpace(lines []string, condense bool) []string {
	return SliceLinesTrim(lines, " ", condense)
}

// SliceLinesTrim trims each line in a slice of lines using a
// provided cut string.
func SliceLinesTrim(lines []string, cutstr string, condense bool) []string {
	for i, line := range lines {
		line = strings.Trim(line, cutstr)
		if condense && len(line) == 0 {
			continue
		}
		lines[i] = line
	}
	return lines
}

// SliceIndexOrEmpty returns the element at the index
// provided or an empty string.
func SliceIndexOrEmpty(s []string, index uint64) string {
	if int(index) >= len(s) {
		return ""
	}
	return s[index]
}

// JoinAny takes an array of interface{} and converts
// each value to a string using fmt.Sprintf("%v")
func JoinAny(a []interface{}, sep string) string {
	strs := []string{}
	for _, item := range a {
		strs = append(strs, fmt.Sprintf("%v", item))
	}
	return strings.Join(strs, sep)
}

func JoinInt(a []int, sep string) string {
	strs := []string{}
	for _, i := range a {
		strs = append(strs, strconv.Itoa(i))
	}
	return strings.Join(strs, sep)
}

func JoinCondenseTrimSpace(slice []string, sep string) string {
	return strings.Join(SliceLinesTrimSpace(slice, true), sep)
}

func SliceCondenseRegexps(texts []string, regexps []*regexp.Regexp, replacement string) []string {
	parts := []string{}
	for _, part := range texts {
		for _, rx := range regexps {
			part = rx.ReplaceAllString(part, replacement)
		}
		part = strings.TrimSpace(part)
		if len(part) > 0 {
			parts = append(parts, part)
		}
	}
	return parts
}

func SliceCondensePunctuation(texts []string) []string {
	parts := []string{}
	for _, part := range texts {
		part = regexp.MustCompile(`[^a-zA-Z0-9]+`).ReplaceAllString(part, " ")
		part = regexp.MustCompile(`\s+`).ReplaceAllString(part, " ")
		part = strings.TrimSpace(part)
		if len(part) > 0 {
			parts = append(parts, part)
		}
	}
	return parts
}

func SliceCondenseAndQuoteSpace(items []string, quoteLeft, quoteRight string) []string {
	return SliceCondenseAndQuote(items, " ", " ", quoteLeft, quoteRight)
}

func SliceCondenseAndQuote(items []string, trimLeft, trimRight, quoteLeft, quoteRight string) []string {
	newItems := []string{}
	for _, item := range items {
		item = strings.TrimLeft(item, trimLeft)
		item = strings.TrimRight(item, trimRight)
		if len(item) > 0 {
			item = quoteLeft + item + quoteRight
			newItems = append(newItems, item)
		}
	}
	return newItems
}

// SplitTrimSpace splits a string and trims spaces on
// remaining elements.
func SplitTrimSpace(s, sep string) []string {
	split := strings.Split(s, sep)
	strs := []string{}
	for _, str := range split {
		strs = append(strs, strings.TrimSpace(str))
	}
	return strs
}

// SliceToSingleIntOrNeg converts a single element slice
// with a string to an integer or `-1`
func SliceToSingleIntOrNeg(vals []string) int {
	if len(vals) != 1 {
		return -1
	}
	num, err := strconv.Atoi(vals[0])
	if err != nil {
		return -1
	}
	return num
}

// Dedupe returns a string slice with duplicate values
// removed. First observance is kept.
func Dedupe(vals []string) []string {
	deduped := []string{}
	seen := map[string]int{}
	for _, val := range vals {
		if _, ok := seen[val]; ok {
			continue
		}
		seen[val] = 1
		deduped = append(deduped, val)
	}
	return deduped
}

// SliceIndexOf returns the index of an element in a
// string slice. Returns -1 if not found.
func SliceIndexOf(needle string, haystack []string) int {
	for k, v := range haystack {
		if v == needle {
			return k
		}
	}
	return -1 //not found.
}

// SliceIndexOfLcTrimSpace returns the index of an element in a
// string slice. Returns -1 if not found.
func SliceIndexOfLcTrimSpace(needle string, haystack []string) int {
	needle = strings.ToLower(strings.TrimSpace(needle))
	for k, v := range haystack {
		v = strings.ToLower(strings.TrimSpace(v))
		if v == needle {
			return k
		}
	}
	return -1 //not found.
}

func SliceChooseOnePreferredLowerTrimSpace(options, preferenceOrder []string) string {
	if len(options) == 0 {
		return ""
	} else if len(preferenceOrder) == 0 {
		return strings.ToLower(strings.TrimSpace(options[0]))
	}
	optMap := map[string]int{}
	for _, opt := range options {
		opt = strings.ToLower(strings.TrimSpace(opt))
		if len(opt) > 0 {
			optMap[opt] = 1
		}
	}
	for _, pref := range preferenceOrder {
		pref = strings.ToLower(strings.TrimSpace(pref))
		if _, ok := optMap[pref]; ok {
			return pref
		}
	}
	return strings.ToLower(strings.TrimSpace(options[0]))
}

func SliceJoinQuotedMaxLength(slice []string, begQuote, endQuote, sep string, maxLength int) []string {
	words := []string{}
	curWords := []string{}
	curLength := 0
	for _, word := range slice {
		if curLength+len(begQuote+word+endQuote+sep) > maxLength {
			words = append(words, strings.Join(curWords, sep))
			curWords = []string{}
			curLength = 0
		} else {
			curWords = append(curWords, begQuote+word+endQuote)
			curLength += len(begQuote + word + endQuote + sep)
		}
	}
	if len(curWords) > 0 {
		words = append(words, strings.Join(curWords, sep))
	}
	return words
}

type JoinCustomConfig struct {
	QuoteBegin string
	QuoteEnd   string
	Separator  string
	MaxLength  int
	TrimSpace  bool
	SkipEmpty  bool
}

func JoinCustom(slice []string, cfg JoinCustomConfig) []string {
	lines := []string{}
	curWords := []string{}
	curLength := 0
	for _, word := range slice {
		if cfg.TrimSpace {
			word = strings.TrimSpace(word)
		}
		if cfg.SkipEmpty && len(word) == 0 {
			continue
		}
		if cfg.MaxLength > 0 {
			if curLength+len(cfg.QuoteBegin+word+cfg.QuoteEnd+cfg.Separator) > cfg.MaxLength {
				lines = append(lines, strings.Join(curWords, cfg.Separator))
				curWords = []string{}
				curLength = 0
			} else {
				curWords = append(curWords, cfg.QuoteBegin+word+cfg.QuoteEnd)
				curLength += len(cfg.QuoteBegin + word + cfg.QuoteEnd + cfg.Separator)
			}
		} else {
			curWords = append(curWords, cfg.QuoteBegin+word+cfg.QuoteEnd)
		}
	}
	if len(curWords) > 0 {
		lines = append(lines, strings.Join(curWords, cfg.Separator))
	}
	return lines
}

func SliceJoinQuotedMaxLengthTrimSpaceSkipEmpty(slice []string, begQuote, endQuote, sep string, maxLength int) []string {
	words := []string{}
	curWords := []string{}
	curLength := 0
	for _, word := range slice {
		if curLength+len(begQuote+word+endQuote+sep) > maxLength {
			words = append(words, strings.Join(curWords, sep))
			curWords = []string{}
			curLength = 0
		} else {
			curWords = append(curWords, begQuote+word+endQuote)
			curLength += len(begQuote + word + endQuote + sep)
		}
	}
	if len(curWords) > 0 {
		words = append(words, strings.Join(curWords, sep))
	}
	return words
}

func SliceJoinQuoted(slice []string, begQuote, endQuote, sep string) string {
	words := []string{}
	for _, word := range slice {
		words = append(words, begQuote+word+endQuote)
	}
	return strings.Join(words, sep)
}

func Remove(real, filter []string) []string {
	filtered := []string{}
	filterMap := map[string]int{}
	for _, f := range filter {
		filterMap[f] = 1
	}
	for _, r := range real {
		if _, ok := filterMap[r]; !ok {
			filtered = append(filtered, r)
		}
	}
	return filtered
}
<|endoftext|>


<commit_before><commit_msg>Fix doc<commit_after><|endoftext|>


<commit_before><commit_msg>Print parsed line with status<commit_after><|endoftext|>


<commit_before><commit_msg>Fix parse float numbers as int<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"regexp"
	"strconv"

	ircevent "github.com/thoj/go-ircevent"
)

//irc client options struct
type ircConfig struct {
	Server   string
	Channels []string
	Nickname string
	Verbose  bool
	Debug    bool
}

var (
	ircConn *ircevent.Connection //irc connection
	ircCfg  *ircConfig           //irc config
	//regex parse string for newEpisode()
	releaseWatch = regexp.MustCompile(
		`(?i)release:.+\[horriblesubs\] (.+) - ([0-9]{1,3}) \[(1080p|720p|480p)\]`)
)

//Starts the irc client
func ircConnStart(c *ircConfig) {
	ircCfg = c

	ircConn = ircevent.IRC(c.Nickname, c.Nickname)
	ircConn.Debug = c.Debug
	ircConn.VerboseCallbackHandler = c.Verbose
	ircConn.AddCallback("PRIVMSG", ircMsgHandler)
	ircConn.AddCallback("001", ircWelcomeHandler)

	err := ircConn.Connect(c.Server)
	if err != nil {
		log.Fatal("ircConnStart() => Connect() error:\t", err)
	}
	ircConn.Loop()
}

//irc client incomming message handler function
func ircMsgHandler(e *ircevent.Event) {
	//Samples new anime string:
	//Release: [Anime] [HorribleSubs] Ushio to Tora - 22 [480p].mkv
	//Release: [Anime] [HorribleSubs] Hackadoll the Animation - 09 [720p].mkv

	if releaseWatch.MatchString(e.Message()) == true {
		newEpisode(releaseWatch.FindStringSubmatch(e.Message()))
	} else if e.Arguments[0] == ircConn.GetNick() && e.Nick != ircConn.GetNick() {
		ircConn.Privmsg(e.Nick, "Discord BOT - https://github.com/gagizagi/Axolotl-GO")
		ircConn.Privmsg(e.Nick, "Current uptime is "+getUptime())
	}
}

//irc client onWelcome handler function
func ircWelcomeHandler(e *ircevent.Event) {
	for _, channel := range ircCfg.Channels {
		ircConn.Join(channel)
	}
}

//new episode handler function
//args
// 0 = full message
// 1 = anime name
// 2 = episode (1-3 length integers only)
// 3 = resolution (1080p | 720p | 480p)
func newEpisode(args []string) {
	epnum, _ := strconv.Atoi(args[2])
	episode := anime{Name: args[1], Episode: epnum}
	if episode.Exists() {
		if episode.NewEpisode() {
			episode.UpdateEp()
			if len(episode.Subs) > 0 {
				resultstr := fmt.Sprintf(
					"**New episode of %s released - Episode %d**\n",
					episode.Name, episode.Episode)

				//Add mentions for every subbed user
				for _, person := range episode.Subs {
					resultstr += fmt.Sprintf("<@%s>", person)
				}

				//Add download link
				if episode.Href != "" {
					resultstr += fmt.Sprintf("\nDownload at %s\n", episode.Href)
				} else {
					resultstr += fmt.Sprint("\nDownload at http://horriblesubs.info/\n")
				}

				//Add subscribe ID
				resultstr += fmt.Sprintf(
					"To subscribe to this anime type \"!sub %s\"",
					episode.ID)

				//Send update message to all anime channels
				//TODO make a function
				for _, channel := range discordCfg.AnimeChannels {
					discord.ChannelMessageSend(channel, resultstr)
				}
			}
		}
	} else {
		//Insert series into DB
		episode.Insert()

		//Announce new series to all anime channels
		resultstr := fmt.Sprintf("**New series started: %s - Episode %d**\n",
			episode.Name, episode.Episode)
		resultstr += fmt.Sprintf("To subscribe to this anime type \"!sub %s\"",
			episode.ID)

		//TODO make a function
		for _, channel := range discordCfg.AnimeChannels {
			discord.ChannelMessageSend(channel, resultstr)
		}
	}
}
<commit_msg>readability<commit_after>package main

import (
	"fmt"
	"log"
	"regexp"
	"strconv"

	ircevent "github.com/thoj/go-ircevent"
)

//irc client options struct
type ircConfig struct {
	Server   string
	Channels []string
	Nickname string
	Verbose  bool
	Debug    bool
}

var (
	ircConn *ircevent.Connection //irc connection
	ircCfg  *ircConfig           //irc config

	//regex parse string for newEpisode()
	releaseWatch = regexp.MustCompile(
		`(?i)release:.+\[horriblesubs\] (.+) - ([0-9]{1,3}) \[(1080p|720p|480p)\]`)
)

//Starts the irc client
func ircConnStart(c *ircConfig) {
	ircCfg = c

	ircConn = ircevent.IRC(c.Nickname, c.Nickname)
	ircConn.Debug = c.Debug
	ircConn.VerboseCallbackHandler = c.Verbose
	ircConn.AddCallback("PRIVMSG", ircMsgHandler)
	ircConn.AddCallback("001", ircWelcomeHandler)

	err := ircConn.Connect(c.Server)
	if err != nil {
		log.Fatal("ircConnStart() => Connect() error:\t", err)
	}
	ircConn.Loop()
}

//irc client incomming message handler function
func ircMsgHandler(e *ircevent.Event) {
	//Samples new anime string:
	//Release: [Anime] [HorribleSubs] Ushio to Tora - 22 [480p].mkv
	//Release: [Anime] [HorribleSubs] Hackadoll the Animation - 09 [720p].mkv

	if releaseWatch.MatchString(e.Message()) {
		newEpisode(releaseWatch.FindStringSubmatch(e.Message()))
	} else if e.Arguments[0] == ircConn.GetNick() && e.Nick != ircConn.GetNick() {
		ircConn.Privmsg(e.Nick, "Discord BOT - https://github.com/gagizagi/Axolotl-GO")
		ircConn.Privmsg(e.Nick, "Current uptime is "+getUptime())
	}
}

//irc client onWelcome handler function
func ircWelcomeHandler(e *ircevent.Event) {
	for _, channel := range ircCfg.Channels {
		ircConn.Join(channel)
	}
}

//new episode handler function
//args
// 0 = full message
// 1 = anime name
// 2 = episode (1-3 length integers only)
// 3 = resolution (1080p | 720p | 480p)
func newEpisode(args []string) {
	epnum, _ := strconv.Atoi(args[2])
	episode := anime{Name: args[1], Episode: epnum}
	if episode.Exists() {
		if episode.NewEpisode() {
			episode.UpdateEp()
			if len(episode.Subs) > 0 {
				resultstr := fmt.Sprintf(
					"**New episode of %s released - Episode %d**\n",
					episode.Name, episode.Episode)

				//Add mentions for every subbed user
				for _, person := range episode.Subs {
					resultstr += fmt.Sprintf("<@%s>", person)
				}

				//Add download link
				if episode.Href != "" {
					resultstr += fmt.Sprintf("\nDownload at %s\n", episode.Href)
				} else {
					resultstr += fmt.Sprint("\nDownload at http://horriblesubs.info/\n")
				}

				//Add subscribe ID
				resultstr += fmt.Sprintf(
					"To subscribe to this anime type \"!sub %s\"",
					episode.ID)

				//Send update message to all anime channels
				//TODO make a function
				for _, channel := range discordCfg.AnimeChannels {
					discord.ChannelMessageSend(channel, resultstr)
				}
			}
		}
	} else {
		//Insert series into DB
		episode.Insert()

		//Announce new series to all anime channels
		resultstr := fmt.Sprintf("**New series started: %s - Episode %d**\n",
			episode.Name, episode.Episode)
		resultstr += fmt.Sprintf("To subscribe to this anime type \"!sub %s\"",
			episode.ID)

		//TODO make a function
		for _, channel := range discordCfg.AnimeChannels {
			discord.ChannelMessageSend(channel, resultstr)
		}
	}
}
<|endoftext|>


<commit_before>package sqlite3

import (
	"database/sql"
	"io/ioutil"
	"os"
	"path"
	"testing"
)

func TestCorruptDbErrors(t *testing.T) {
	dirName, err := ioutil.TempDir("", "sqlite3")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dirName)

	dbFileName := path.Join(dirName, "test.db")
	f, err := os.Create(dbFileName)
	if err != nil {
		t.Error(err)
	}
	f.Write([]byte{1, 2, 3, 4, 5})
	f.Close()

	db, err := sql.Open("sqlite3", dbFileName)
	if err == nil {
		_, err = db.Exec("drop table foo")
	}

	sqliteErr := err.(Error)
	if sqliteErr.Code != ErrNotADB {
		t.Error("wrong error code for corrupted DB")
	}
	if err.Error() == "" {
		t.Error("wrong error string for corrupted DB")
	}
	db.Close()
}

func TestSqlLogicErrors(t *testing.T) {
	dirName, err := ioutil.TempDir("", "sqlite3")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dirName)

	dbFileName := path.Join(dirName, "test.db")
	db, err := sql.Open("sqlite3", dbFileName)
	if err != nil {
		t.Error(err)
	}

	_, err = db.Exec("CREATE TABLE Foo (id INT PRIMARY KEY)")
	if err != nil {
		t.Error(err)
	}

	const expectedErr = "table Foo already exists"
	_, err = db.Exec("CREATE TABLE Foo (id INT PRIMARY KEY)")
	if err.Error() != expectedErr {
		t.Errorf("Unexpected error: %s, expected %s", err.Error(), expectedErr)
	}
}
<commit_msg>Add test for error == nil<commit_after>package sqlite3

import (
	"database/sql"
	"io/ioutil"
	"os"
	"path"
	"testing"
)

func TestSimpleError(t *testing.T) {
	if ErrWarning.Error() != "" {
		t.Error("wrong error code for corrupted DB")
	}
}

func TestCorruptDbErrors(t *testing.T) {
	dirName, err := ioutil.TempDir("", "sqlite3")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dirName)

	dbFileName := path.Join(dirName, "test.db")
	f, err := os.Create(dbFileName)
	if err != nil {
		t.Error(err)
	}
	f.Write([]byte{1, 2, 3, 4, 5})
	f.Close()

	db, err := sql.Open("sqlite3", dbFileName)
	if err == nil {
		_, err = db.Exec("drop table foo")
	}

	sqliteErr := err.(Error)
	if sqliteErr.Code != ErrNotADB {
		t.Error("wrong error code for corrupted DB")
	}
	if err.Error() == "" {
		t.Error("wrong error string for corrupted DB")
	}
	db.Close()
}

func TestSqlLogicErrors(t *testing.T) {
	dirName, err := ioutil.TempDir("", "sqlite3")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dirName)

	dbFileName := path.Join(dirName, "test.db")
	db, err := sql.Open("sqlite3", dbFileName)
	if err != nil {
		t.Error(err)
	}

	_, err = db.Exec("CREATE TABLE Foo (id INT PRIMARY KEY)")
	if err != nil {
		t.Error(err)
	}

	const expectedErr = "table Foo already exists"
	_, err = db.Exec("CREATE TABLE Foo (id INT PRIMARY KEY)")
	if err.Error() != expectedErr {
		t.Errorf("Unexpected error: %s, expected %s", err.Error(), expectedErr)
	}
}
<|endoftext|>


<commit_before><commit_msg>Sum up totals and print summary of scan<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/deferpanic/dpcli/api"
	"os"
	"strings"
)

// osCheck ensures we are dealing with el capitan or above
func osCheck() string {
	good := []string{"10.11.4", "10.11.5", "10.11.6", "10.12", "10.12.2", "10.12.3"}
	out := strings.TrimSpace(runCmd("sw_vers -productVersion"))
	for i := 0; i < len(good); i++ {
		if good[i] == out {
			fmt.Println(api.GreenBold("found supported osx version"))
			return out
		}
	}

	fmt.Printf(api.RedBold(fmt.Sprintf("You are running osX version %s\n", out)))
	fmt.Printf(api.RedBold(fmt.Sprintf("This is only tested on osX %v.\n"+
		"pf_ctl is used. If using an earlier osx you might need to use natd "+
		"or contribute a patch :)\n", good)))
	os.Exit(1)
	return ""
}

// cpulimitCheck looks for cpulimit which helps languages that use a lot
// of cpu
func cpulimitCheck() {
	out := strings.TrimSpace(runCmd("/usr/bin/which cpulimit"))
	if out == "" {
		fmt.Println(api.RedBold("cpulimit not found - installing..."))
		runCmd("brew install cpulimit")
	} else {
		fmt.Println(api.GreenBold("found cpulimit"))
	}
}

func qemuCheck() {
	out := strings.TrimSpace(runCmd("which qemu-system-x86_64"))
	if out == "qemu-system-x86_64 not found" {
		fmt.Println(api.RedBold("qemu not found - installing..."))
		runCmd("brew install qemu")
	} else {
		fmt.Println(api.GreenBold("found qemu"))
	}
}

func tuntapCheck() {
	out := strings.TrimSpace(runCmd("sudo kextstat | grep tap"))
	if out != "" {
		fmt.Println(api.GreenBold("found tuntap support"))
	} else {
		fmt.Println(api.RedBold("Please download and install tuntaposx"))
		fmt.Println(api.RedBold("wget http://downloads.sourceforge.net/tuntaposx/tuntap_20150118.tar.gz"))
		os.Exit(1)
	}
}
<commit_msg>bumping version<commit_after>package main

import (
	"fmt"
	"github.com/deferpanic/dpcli/api"
	"os"
	"strings"
)

// osCheck ensures we are dealing with el capitan or above
func osCheck() string {
	good := []string{"10.11.4", "10.11.5", "10.11.6", "10.12", "10.12.2", "10.12.3", "10.12.6"}
	out := strings.TrimSpace(runCmd("sw_vers -productVersion"))
	for i := 0; i < len(good); i++ {
		if good[i] == out {
			fmt.Println(api.GreenBold("found supported osx version"))
			return out
		}
	}

	fmt.Printf(api.RedBold(fmt.Sprintf("You are running osX version %s\n", out)))
	fmt.Printf(api.RedBold(fmt.Sprintf("This is only tested on osX %v.\n"+
		"pf_ctl is used. If using an earlier osx you might need to use natd "+
		"or contribute a patch :)\n", good)))
	os.Exit(1)
	return ""
}

// cpulimitCheck looks for cpulimit which helps languages that use a lot
// of cpu
func cpulimitCheck() {
	out := strings.TrimSpace(runCmd("/usr/bin/which cpulimit"))
	if out == "" {
		fmt.Println(api.RedBold("cpulimit not found - installing..."))
		runCmd("brew install cpulimit")
	} else {
		fmt.Println(api.GreenBold("found cpulimit"))
	}
}

func qemuCheck() {
	out := strings.TrimSpace(runCmd("which qemu-system-x86_64"))
	if out == "qemu-system-x86_64 not found" {
		fmt.Println(api.RedBold("qemu not found - installing..."))
		runCmd("brew install qemu")
	} else {
		fmt.Println(api.GreenBold("found qemu"))
	}
}

func tuntapCheck() {
	out := strings.TrimSpace(runCmd("sudo kextstat | grep tap"))
	if out != "" {
		fmt.Println(api.GreenBold("found tuntap support"))
	} else {
		fmt.Println(api.RedBold("Please download and install tuntaposx"))
		fmt.Println(api.RedBold("wget http://downloads.sourceforge.net/tuntaposx/tuntap_20150118.tar.gz"))
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package dorothea

type Pin struct {
	player *Player
	field  int
}

const OUT_AREA = -1

func NewPins(p *Player) [4]*Pin {
	var pins [4]*Pin
	for i := 0; i < 4; i++ {
		pins[i] = &Pin{
			player: p,
			field:  OUT_AREA}
	}
	return pins
}
<commit_msg>new Position type for Pins<commit_after>package dorothea

type Position int

const (
	Out     Position = iota // Pin in OutArea
	OnField                 // Pin on Field
	InHome                  // Pin in HomeArea
)

type Pin struct {
	player    *Player
	position  Position
	iField    int // index in game.fields when position == OnField
	iHomeArea int // index in player.HomeArea when position == InHome
}

const NOT_IN_INDEX = -1

func NewPins(p *Player) [4]*Pin {
	var pins [4]*Pin
	for i := 0; i < 4; i++ {
		pins[i] = &Pin{
			player:    p,
			position:  Out,
			iField:    NOT_IN_INDEX,
			iHomeArea: NOT_IN_INDEX}
	}
	return pins
}
<|endoftext|>