<commit_before>package main

import (
	"archive/zip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"github.com/gorilla/mux"
)

var ROOT = ""

type FileInfoJSON struct {
	Name        string `json:"name"`
	Size        int64  `json:"size"`
	ModifiedAt  string `json:"modified_at"`
	MIMEType    string `json:"mime_type"`
	IsDirectory bool   `json:"is_directory"`
	IsHidden    bool   `json:"is_hidden"`
	IsLink      bool   `json:"is_link"`
}

// returns a pair of (filename, MIME type) strings given a `file` output line
func parseMIMEType(fileOutputLine string) (string, string, error) {
	// parse the file program output into a bare MIME type
	mimeString := strings.TrimSpace(fileOutputLine)
	splitIndex := strings.LastIndex(mimeString, ":")

	if len(fileOutputLine) <= 1 || splitIndex <= 0 {
		return "", "", fmt.Errorf("Invalid MIME string: '%s'", fileOutputLine)
	}

	return mimeString[0:splitIndex], strings.TrimSpace(mimeString[splitIndex+1:]), nil
}

func writeJSONResponse(w http.ResponseWriter, data interface{}) {
	w.Header().Add("Content-Type", "application/json")
	w.Header().Add("Cache-Control", "no-cache")

	json, err := json.Marshal(data)
	if err != nil {
		http.Error(w, "Failed to generate JSON response", 500)
		return
	}
	w.Write(json)
}

// given a path, returns a map of child name to MIME type
func getChildMIMETypes(parentPath string) map[string]string {
	result := make(map[string]string)

	// get all the children in the given directory
	children, err := filepath.Glob(path.Join(parentPath, "*"))
	if err != nil {
		return result
	}

	args := []string{"--mime-type", "--dereference", "--preserve-date"}
	args = append(args, children...)

	// call `file` for a newline-delimited list of "filename: MIME-type" pairs
	fileOutput, err := exec.Command("file", args...).Output()

	if err != nil {
		return result
	}

	for _, line := range strings.Split(string(fileOutput), "\n") {
		fileName, mimeType, err := parseMIMEType(line)
		if err == nil {
			result[fileName] = mimeType
		}
	}

	return result
}

func getMIMEType(filePath string) string {
	fileOutput, err := exec.Command(
		"file",
		"--mime-type",
		// "--dereference",
		"--preserve-date",
		"--brief",
		filePath,
	).Output()

	if err != nil {
		return ""
	}

	return strings.TrimSpace(string(fileOutput))
}

// given a root and a relative child path, returns the normalized, absolute path
// of the child. if the path is not a child of the root or is otherwise invalid,
// returns an error.
func normalizePathUnderRoot(root, child string) (string, error) {
	// clean the path, resolving any ".."s in it
	requestPath := path.Clean(path.Join(root, child))

	// if the path exited the root directory, fail
	relPath, err := filepath.Rel(root, requestPath)
	if err != nil || strings.Index(relPath, "..") >= 0 {
		// keep things vague since someone's probably trying to be sneaky anyway
		return "", fmt.Errorf("Invalid path")
	}

	return requestPath, nil
}

// this returns the info for the specified files _or_ directory, not just files
func getInfo(w http.ResponseWriter, r *http.Request) {
	// make sure our path is valid
	rawPath := mux.Vars(r)["path"]
	normalizedPath, err := normalizePathUnderRoot(ROOT, rawPath)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// stat the file so we can return its info
	fileInfo, err := os.Stat(normalizedPath)
	if err != nil {
		// don't report the raw error in case we leak server directory information
		http.Error(w, "", 404)
		return
	}

	mimeType := getMIMEType(normalizedPath)

	writeJSONResponse(w, FileInfoJSON{
		fileInfo.Name(),
		fileInfo.Size(),
		fileInfo.ModTime().Format("2006-01-02T15:04:05Z"), // ISO 8601
		mimeType,
		fileInfo.IsDir(),
		strings.HasPrefix(fileInfo.Name(), "."),
		fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink,
	})
}

func download(w http.ResponseWriter, r *http.Request) {
	// make sure our path is valid
	rawPath := mux.Vars(r)["path"]
	normalizedPath, err := normalizePathUnderRoot(ROOT, rawPath)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// stat the file so we can set appropriate response headers, and so we can
	// ensure it's a regular file and not a directory.
	file, err := os.Stat(normalizedPath)
	if err != nil {
		// don't report the raw error in case we leak server directory information
		http.Error(w, "", 404)
		return
	}

	// return different responses depending on file type
	if file.IsDir() {
		downloadDirectory(w, r, normalizedPath)
	} else {
		downloadFile(w, r, normalizedPath, file)
	}
}

func downloadFile(w http.ResponseWriter, r *http.Request, filePath string, file os.FileInfo) {
	mimeType := getMIMEType(filePath)
	w.Header().Add("Content-Type", mimeType)
	w.Header().Add("Content-Disposition", file.Name())
	w.Header().Add("Cache-Control", "no-cache")

	http.ServeFile(w, r, filePath)
}

func getDirectory(w http.ResponseWriter, r *http.Request) {
	// ensure the directory actually exists
	rawPath := mux.Vars(r)["path"]
	normalizedPath, err := normalizePathUnderRoot(ROOT, rawPath)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	children, err := ioutil.ReadDir(normalizedPath)
	if err != nil {
		// don't report the raw error in case we leak server directory information
		http.Error(w, "", 404)
		return
	}

	// get a map of all the MIME types for the directory
	mimeTypes := getChildMIMETypes(normalizedPath)

	// list the directory to a JSON response
	var files []FileInfoJSON
	for _, file := range children {
		fileName := file.Name()
		mimeType := mimeTypes[path.Join(normalizedPath, fileName)]

		files = append(files, FileInfoJSON{
			fileName,
			file.Size(),
			file.ModTime().Format("2006-01-02T15:04:05Z"), // ISO 8601
			mimeType,
			file.IsDir(),
			strings.HasPrefix(fileName, "."), // hidden?
			file.Mode()&os.ModeSymlink == os.ModeSymlink,
		})
	}

	writeJSONResponse(w, files)
}

// zip up a directory and write it to the response stream
func downloadDirectory(w http.ResponseWriter, r *http.Request, dirPath string) {
	// give the file a nice name, but replace the root directory name with
	// something generic.
	var downloadName string
	if dirPath == ROOT {
		downloadName = "files.zip"
	} else {
		downloadName = path.Base(dirPath) + ".zip"
	}

	w.Header().Add("Content-Type", "application/zip")
	w.Header().Add("Content-Disposition", downloadName)
	w.Header().Add("Cache-Control", "no-cache")

	z := zip.NewWriter(w)
	defer z.Close()

	// walk the directory and add each file to the zip file, giving up (returning
	// an error) if we encounter an error anywhere along the line.
	filepath.Walk(dirPath, func(fullFilePath string, file os.FileInfo, err error) error {
		if err != nil {
			// don't say what failed since doing so might leak the full path
			http.Error(w, "Failed to generate archive", 500)
			return err
		}

		// use the relative file path so we don't accidentally leak the full path
		// anywhere. we only use the full path to read the file from disk. we know
		// it's relative so we can ignore the error.
		filePath, _ := filepath.Rel(dirPath, fullFilePath)

		// build a header we can use to generate a ZIP archive entry
		header, err := zip.FileInfoHeader(file)
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to generate archive header for %s", filePath), 500)
			return err
		}

		// ensure the name is set to relative path within this directory so we'll
		// preserve the directory's structure within the archive.
		header.Name = filePath

		// add a directory entry for true directories so they'll show up even if
		// they have no children. adding a trailing `/` does this for us,
		// apparently.
		fileIsSymlink := file.Mode()&os.ModeSymlink == os.ModeSymlink
		if file.IsDir() && !fileIsSymlink {
			header.Name += "/"
		}

		// generate an archive entry for this file/directory/symlink
		zf, err := z.CreateHeader(header)
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to add %s to archive", filePath), 500)
			return err
		}

		// if the file is a symlink, preserve it as such
		if fileIsSymlink {
			// according to the ZIP format, symlinks must have the namesake file mode
			// with sole body content of the string path of the link's destination.
			dest, err := os.Readlink(fullFilePath)
			if err != nil {
				http.Error(w, fmt.Sprintf("Failed to resolve %s", filePath), 500)
				return err
			}

			zf.Write([]byte(dest))
		} else if file.IsDir() {
			// NOTE: do nothing since all we have to do for directories is create
			// their header entry, which has already been done.
		} else {
			// open the file for reading
			f, err := os.Open(fullFilePath)
			if err != nil {
				http.Error(w, fmt.Sprintf("Failed to read %s", filePath), 500)
				return err
			}
			defer f.Close()

			// write the file contents to the archive
			written, err := io.Copy(zf, f)
			if err != nil {
				http.Error(w, fmt.Sprintf("Failed to write %s to archive after %d bytes", filePath, written), 500)
				return err
			}
		}

		// flush what we've written so far to the client so the download will be as
		// incremental as possible. doing flushes after every file also ensures that
		// our memory usage doesn't balloon to the entire size of the zipped
		// directory, just the size of one file (which is better than nothing...).
		err = z.Flush()
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to flush data for %s", filePath), 500)
			return err
		}

		return nil
	})
}

// retrieves/caches/updates a thumbnail file given a path, or returns an error
// if no thumbnail could be geneated.
func getThumbnail(w http.ResponseWriter, r *http.Request) {
	// TODO:
	// * look up the parent file to get its modtime and ensure it exists
	// * see if we have a cached file with the same modtime
	// ** if so, use it
	// ** otherwise, generate a preview
	// *** use graphicsmagick/ffmpeg to generate a preview thumbnail
	// *** store the thumbnail to a mirrored path with the filename plus the modtime
	// * read the cached file and return its contents

	// cache preview thumbnails for a good while to lower load on this tiny
	// server, even if we are caching the preview thumbnails on-disk too.
	w.Header().Add("Cache-Control", "max-age=3600")
	w.Header().Add("Content-Type", "image/jpeg")
}

func main() {
	// ensure we have all the binaries we need
	requiredBinaries := []string{"file"}
	for _, binary := range requiredBinaries {
		if _, err := exec.LookPath(binary); err != nil {
			log.Panicf("'%s' must be installed and in the PATH\n", binary)
		}
	}

	if len(os.Args) <= 1 {
		panic("A root directory argument is required")
	}

	ROOT = path.Clean(os.Args[1])

	router := mux.NewRouter()

	// /files
	// anything with a trailing `/` indicates a directory; anything that ends
	// without a trailing slash indicates a file.
	filesJSON := router.Headers("Content-Type", "application/json").Subrouter()
	filesJSON.HandleFunc("/files/{path:.*[^/]$}", getInfo).
		Headers("Content-Type", "application/json").
		Methods("GET")
	filesJSON.HandleFunc("/files{path:.*}/", getDirectory).
		Headers("Content-Type", "application/json").
		Methods("GET")

	router.HandleFunc("/files/{path:.*}", download).
		Methods("GET")

	// /thumbnails
	router.HandleFunc("/thumbnails/{path:.*[^/]$}", getThumbnail).
		Methods("GET")

	// /resources (static files)
	router.HandleFunc("/resources/{path:.*}", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "ui/resources/" + mux.Vars(r)["path"])
	})

	router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "ui/resources/index.html")
	})

	addr := "127.0.0.1:3000"
	fmt.Printf("Serving %s to %s...\n", ROOT, addr)
	http.ListenAndServe(addr, router)
}
<commit_msg>Formatting<commit_after>package main

import (
	"archive/zip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"github.com/gorilla/mux"
)

var ROOT = ""

type FileInfoJSON struct {
	Name        string `json:"name"`
	Size        int64  `json:"size"`
	ModifiedAt  string `json:"modified_at"`
	MIMEType    string `json:"mime_type"`
	IsDirectory bool   `json:"is_directory"`
	IsHidden    bool   `json:"is_hidden"`
	IsLink      bool   `json:"is_link"`
}

// returns a pair of (filename, MIME type) strings given a `file` output line
func parseMIMEType(fileOutputLine string) (string, string, error) {
	// parse the file program output into a bare MIME type
	mimeString := strings.TrimSpace(fileOutputLine)
	splitIndex := strings.LastIndex(mimeString, ":")

	if len(fileOutputLine) <= 1 || splitIndex <= 0 {
		return "", "", fmt.Errorf("Invalid MIME string: '%s'", fileOutputLine)
	}

	return mimeString[0:splitIndex], strings.TrimSpace(mimeString[splitIndex+1:]), nil
}

func writeJSONResponse(w http.ResponseWriter, data interface{}) {
	w.Header().Add("Content-Type", "application/json")
	w.Header().Add("Cache-Control", "no-cache")

	json, err := json.Marshal(data)
	if err != nil {
		http.Error(w, "Failed to generate JSON response", 500)
		return
	}
	w.Write(json)
}

// given a path, returns a map of child name to MIME type
func getChildMIMETypes(parentPath string) map[string]string {
	result := make(map[string]string)

	// get all the children in the given directory
	children, err := filepath.Glob(path.Join(parentPath, "*"))
	if err != nil {
		return result
	}

	args := []string{"--mime-type", "--dereference", "--preserve-date"}
	args = append(args, children...)

	// call `file` for a newline-delimited list of "filename: MIME-type" pairs
	fileOutput, err := exec.Command("file", args...).Output()

	if err != nil {
		return result
	}

	for _, line := range strings.Split(string(fileOutput), "\n") {
		fileName, mimeType, err := parseMIMEType(line)
		if err == nil {
			result[fileName] = mimeType
		}
	}

	return result
}

func getMIMEType(filePath string) string {
	fileOutput, err := exec.Command(
		"file",
		"--mime-type",
		// "--dereference",
		"--preserve-date",
		"--brief",
		filePath,
	).Output()

	if err != nil {
		return ""
	}

	return strings.TrimSpace(string(fileOutput))
}

// given a root and a relative child path, returns the normalized, absolute path
// of the child. if the path is not a child of the root or is otherwise invalid,
// returns an error.
func normalizePathUnderRoot(root, child string) (string, error) {
	// clean the path, resolving any ".."s in it
	requestPath := path.Clean(path.Join(root, child))

	// if the path exited the root directory, fail
	relPath, err := filepath.Rel(root, requestPath)
	if err != nil || strings.Index(relPath, "..") >= 0 {
		// keep things vague since someone's probably trying to be sneaky anyway
		return "", fmt.Errorf("Invalid path")
	}

	return requestPath, nil
}

// this returns the info for the specified files _or_ directory, not just files
func getInfo(w http.ResponseWriter, r *http.Request) {
	// make sure our path is valid
	rawPath := mux.Vars(r)["path"]
	normalizedPath, err := normalizePathUnderRoot(ROOT, rawPath)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// stat the file so we can return its info
	fileInfo, err := os.Stat(normalizedPath)
	if err != nil {
		// don't report the raw error in case we leak server directory information
		http.Error(w, "", 404)
		return
	}

	mimeType := getMIMEType(normalizedPath)

	writeJSONResponse(w, FileInfoJSON{
		fileInfo.Name(),
		fileInfo.Size(),
		fileInfo.ModTime().Format("2006-01-02T15:04:05Z"), // ISO 8601
		mimeType,
		fileInfo.IsDir(),
		strings.HasPrefix(fileInfo.Name(), "."),
		fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink,
	})
}

func download(w http.ResponseWriter, r *http.Request) {
	// make sure our path is valid
	rawPath := mux.Vars(r)["path"]
	normalizedPath, err := normalizePathUnderRoot(ROOT, rawPath)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// stat the file so we can set appropriate response headers, and so we can
	// ensure it's a regular file and not a directory.
	file, err := os.Stat(normalizedPath)
	if err != nil {
		// don't report the raw error in case we leak server directory information
		http.Error(w, "", 404)
		return
	}

	// return different responses depending on file type
	if file.IsDir() {
		downloadDirectory(w, r, normalizedPath)
	} else {
		downloadFile(w, r, normalizedPath, file)
	}
}

func downloadFile(w http.ResponseWriter, r *http.Request, filePath string, file os.FileInfo) {
	mimeType := getMIMEType(filePath)
	w.Header().Add("Content-Type", mimeType)
	w.Header().Add("Content-Disposition", file.Name())
	w.Header().Add("Cache-Control", "no-cache")

	http.ServeFile(w, r, filePath)
}

func getDirectory(w http.ResponseWriter, r *http.Request) {
	// ensure the directory actually exists
	rawPath := mux.Vars(r)["path"]
	normalizedPath, err := normalizePathUnderRoot(ROOT, rawPath)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	children, err := ioutil.ReadDir(normalizedPath)
	if err != nil {
		// don't report the raw error in case we leak server directory information
		http.Error(w, "", 404)
		return
	}

	// get a map of all the MIME types for the directory
	mimeTypes := getChildMIMETypes(normalizedPath)

	// list the directory to a JSON response
	var files []FileInfoJSON
	for _, file := range children {
		fileName := file.Name()
		mimeType := mimeTypes[path.Join(normalizedPath, fileName)]

		files = append(files, FileInfoJSON{
			fileName,
			file.Size(),
			file.ModTime().Format("2006-01-02T15:04:05Z"), // ISO 8601
			mimeType,
			file.IsDir(),
			strings.HasPrefix(fileName, "."), // hidden?
			file.Mode()&os.ModeSymlink == os.ModeSymlink,
		})
	}

	writeJSONResponse(w, files)
}

// zip up a directory and write it to the response stream
func downloadDirectory(w http.ResponseWriter, r *http.Request, dirPath string) {
	// give the file a nice name, but replace the root directory name with
	// something generic.
	var downloadName string
	if dirPath == ROOT {
		downloadName = "files.zip"
	} else {
		downloadName = path.Base(dirPath) + ".zip"
	}

	w.Header().Add("Content-Type", "application/zip")
	w.Header().Add("Content-Disposition", downloadName)
	w.Header().Add("Cache-Control", "no-cache")

	z := zip.NewWriter(w)
	defer z.Close()

	// walk the directory and add each file to the zip file, giving up (returning
	// an error) if we encounter an error anywhere along the line.
	filepath.Walk(dirPath, func(fullFilePath string, file os.FileInfo, err error) error {
		if err != nil {
			// don't say what failed since doing so might leak the full path
			http.Error(w, "Failed to generate archive", 500)
			return err
		}

		// use the relative file path so we don't accidentally leak the full path
		// anywhere. we only use the full path to read the file from disk. we know
		// it's relative so we can ignore the error.
		filePath, _ := filepath.Rel(dirPath, fullFilePath)

		// build a header we can use to generate a ZIP archive entry
		header, err := zip.FileInfoHeader(file)
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to generate archive header for %s", filePath), 500)
			return err
		}

		// ensure the name is set to relative path within this directory so we'll
		// preserve the directory's structure within the archive.
		header.Name = filePath

		// add a directory entry for true directories so they'll show up even if
		// they have no children. adding a trailing `/` does this for us,
		// apparently.
		fileIsSymlink := file.Mode()&os.ModeSymlink == os.ModeSymlink
		if file.IsDir() && !fileIsSymlink {
			header.Name += "/"
		}

		// generate an archive entry for this file/directory/symlink
		zf, err := z.CreateHeader(header)
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to add %s to archive", filePath), 500)
			return err
		}

		// if the file is a symlink, preserve it as such
		if fileIsSymlink {
			// according to the ZIP format, symlinks must have the namesake file mode
			// with sole body content of the string path of the link's destination.
			dest, err := os.Readlink(fullFilePath)
			if err != nil {
				http.Error(w, fmt.Sprintf("Failed to resolve %s", filePath), 500)
				return err
			}

			zf.Write([]byte(dest))
		} else if file.IsDir() {
			// NOTE: do nothing since all we have to do for directories is create
			// their header entry, which has already been done.
		} else {
			// open the file for reading
			f, err := os.Open(fullFilePath)
			if err != nil {
				http.Error(w, fmt.Sprintf("Failed to read %s", filePath), 500)
				return err
			}
			defer f.Close()

			// write the file contents to the archive
			written, err := io.Copy(zf, f)
			if err != nil {
				http.Error(w, fmt.Sprintf("Failed to write %s to archive after %d bytes", filePath, written), 500)
				return err
			}
		}

		// flush what we've written so far to the client so the download will be as
		// incremental as possible. doing flushes after every file also ensures that
		// our memory usage doesn't balloon to the entire size of the zipped
		// directory, just the size of one file (which is better than nothing...).
		err = z.Flush()
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to flush data for %s", filePath), 500)
			return err
		}

		return nil
	})
}

// retrieves/caches/updates a thumbnail file given a path, or returns an error
// if no thumbnail could be geneated.
func getThumbnail(w http.ResponseWriter, r *http.Request) {
	// TODO:
	// * look up the parent file to get its modtime and ensure it exists
	// * see if we have a cached file with the same modtime
	// ** if so, use it
	// ** otherwise, generate a preview
	// *** use graphicsmagick/ffmpeg to generate a preview thumbnail
	// *** store the thumbnail to a mirrored path with the filename plus the modtime
	// * read the cached file and return its contents

	// cache preview thumbnails for a good while to lower load on this tiny
	// server, even if we are caching the preview thumbnails on-disk too.
	w.Header().Add("Cache-Control", "max-age=3600")
	w.Header().Add("Content-Type", "image/jpeg")
}

func main() {
	// ensure we have all the binaries we need
	requiredBinaries := []string{"file"}
	for _, binary := range requiredBinaries {
		if _, err := exec.LookPath(binary); err != nil {
			log.Panicf("'%s' must be installed and in the PATH\n", binary)
		}
	}

	if len(os.Args) <= 1 {
		panic("A root directory argument is required")
	}

	ROOT = path.Clean(os.Args[1])

	router := mux.NewRouter()

	// /files
	// anything with a trailing `/` indicates a directory; anything that ends
	// without a trailing slash indicates a file.
	filesJSON := router.Headers("Content-Type", "application/json").Subrouter()
	filesJSON.HandleFunc("/files/{path:.*[^/]$}", getInfo).
		Headers("Content-Type", "application/json").
		Methods("GET")
	filesJSON.HandleFunc("/files{path:.*}/", getDirectory).
		Headers("Content-Type", "application/json").
		Methods("GET")

	router.HandleFunc("/files/{path:.*}", download).
		Methods("GET")

	// /thumbnails
	router.HandleFunc("/thumbnails/{path:.*[^/]$}", getThumbnail).
		Methods("GET")

	// /resources (static files)
	router.HandleFunc("/resources/{path:.*}", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "ui/resources/"+mux.Vars(r)["path"])
	})

	router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "ui/resources/index.html")
	})

	addr := "127.0.0.1:3000"
	fmt.Printf("Serving %s to %s...\n", ROOT, addr)
	http.ListenAndServe(addr, router)
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import (
	"runtime/internal/sys"
	"unsafe"
)

const (
	_PAGE_SIZE = sys.PhysPageSize
	_EACCES    = 13
)

// NOTE: vec must be just 1 byte long here.
// Mincore returns ENOMEM if any of the pages are unmapped,
// but we want to know that all of the pages are unmapped.
// To make these the same, we can only ask about one page
// at a time. See golang.org/issue/7476.
var addrspace_vec [1]byte

func addrspace_free(v unsafe.Pointer, n uintptr) bool {
	var chunk uintptr
	for off := uintptr(0); off < n; off += chunk {
		chunk = _PAGE_SIZE * uintptr(len(addrspace_vec))
		if chunk > (n - off) {
			chunk = n - off
		}
		errval := mincore(unsafe.Pointer(uintptr(v)+off), chunk, &addrspace_vec[0])
		// ENOMEM means unmapped, which is what we want.
		// Anything else we assume means the pages are mapped.
		if errval != -_ENOMEM {
			return false
		}
	}
	return true
}

func mmap_fixed(v unsafe.Pointer, n uintptr, prot, flags, fd int32, offset uint32) unsafe.Pointer {
	p := mmap(v, n, prot, flags, fd, offset)
	// On some systems, mmap ignores v without
	// MAP_FIXED, so retry if the address space is free.
	if p != v && addrspace_free(v, n) {
		if uintptr(p) > 4096 {
			munmap(p, n)
		}
		p = mmap(v, n, prot, flags|_MAP_FIXED, fd, offset)
	}
	return p
}

// Don't split the stack as this method may be invoked without a valid G, which
// prevents us from allocating more stack.
//go:nosplit
func sysAlloc(n uintptr, sysStat *uint64) unsafe.Pointer {
	p := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
	if uintptr(p) < 4096 {
		if uintptr(p) == _EACCES {
			print("runtime: mmap: access denied\n")
			exit(2)
		}
		if uintptr(p) == _EAGAIN {
			print("runtime: mmap: too much locked memory (check 'ulimit -l').\n")
			exit(2)
		}
		return nil
	}
	mSysStatInc(sysStat, n)
	return p
}

func sysUnused(v unsafe.Pointer, n uintptr) {
	// By default, Linux's "transparent huge page" support will
	// merge pages into a huge page if there's even a single
	// present regular page, undoing the effects of the DONTNEED
	// below. On amd64, that means khugepaged can turn a single
	// 4KB page to 2MB, bloating the process's RSS by as much as
	// 512X. (See issue #8832 and Linux kernel bug
	// https://bugzilla.kernel.org/show_bug.cgi?id=93111)
	//
	// To work around this, we explicitly disable transparent huge
	// pages when we release pages of the heap. However, we have
	// to do this carefully because changing this flag tends to
	// split the VMA (memory mapping) containing v in to three
	// VMAs in order to track the different values of the
	// MADV_NOHUGEPAGE flag in the different regions. There's a
	// default limit of 65530 VMAs per address space (sysctl
	// vm.max_map_count), so we must be careful not to create too
	// many VMAs (see issue #12233).
	//
	// Since huge pages are huge, there's little use in adjusting
	// the MADV_NOHUGEPAGE flag on a fine granularity, so we avoid
	// exploding the number of VMAs by only adjusting the
	// MADV_NOHUGEPAGE flag on a large granularity. This still
	// gets most of the benefit of huge pages while keeping the
	// number of VMAs under control. With hugePageSize = 2MB, even
	// a pessimal heap can reach 128GB before running out of VMAs.
	if sys.HugePageSize != 0 {
		var s uintptr = sys.HugePageSize // division by constant 0 is a compile-time error :(

		// If it's a large allocation, we want to leave huge
		// pages enabled. Hence, we only adjust the huge page
		// flag on the huge pages containing v and v+n-1, and
		// only if those aren't aligned.
		var head, tail uintptr
		if uintptr(v)%s != 0 {
			// Compute huge page containing v.
			head = uintptr(v) &^ (s - 1)
		}
		if (uintptr(v)+n)%s != 0 {
			// Compute huge page containing v+n-1.
			tail = (uintptr(v) + n - 1) &^ (s - 1)
		}

		// Note that madvise will return EINVAL if the flag is
		// already set, which is quite likely. We ignore
		// errors.
		if head != 0 && head+sys.HugePageSize == tail {
			// head and tail are different but adjacent,
			// so do this in one call.
			madvise(unsafe.Pointer(head), 2*sys.HugePageSize, _MADV_NOHUGEPAGE)
		} else {
			// Advise the huge pages containing v and v+n-1.
			if head != 0 {
				madvise(unsafe.Pointer(head), sys.HugePageSize, _MADV_NOHUGEPAGE)
			}
			if tail != 0 && tail != head {
				madvise(unsafe.Pointer(tail), sys.HugePageSize, _MADV_NOHUGEPAGE)
			}
		}
	}

	madvise(v, n, _MADV_DONTNEED)
}

func sysUsed(v unsafe.Pointer, n uintptr) {
	if sys.HugePageSize != 0 {
		// Partially undo the NOHUGEPAGE marks from sysUnused
		// for whole huge pages between v and v+n. This may
		// leave huge pages off at the end points v and v+n
		// even though allocations may cover these entire huge
		// pages. We could detect this and undo NOHUGEPAGE on
		// the end points as well, but it's probably not worth
		// the cost because when neighboring allocations are
		// freed sysUnused will just set NOHUGEPAGE again.
		var s uintptr = sys.HugePageSize

		// Round v up to a huge page boundary.
		beg := (uintptr(v) + (s - 1)) &^ (s - 1)
		// Round v+n down to a huge page boundary.
		end := (uintptr(v) + n) &^ (s - 1)

		if beg < end {
			madvise(unsafe.Pointer(beg), end-beg, _MADV_HUGEPAGE)
		}
	}
}

// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
//go:nosplit
func sysFree(v unsafe.Pointer, n uintptr, sysStat *uint64) {
	mSysStatDec(sysStat, n)
	munmap(v, n)
}

func sysFault(v unsafe.Pointer, n uintptr) {
	mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, -1, 0)
}

func sysReserve(v unsafe.Pointer, n uintptr, reserved *bool) unsafe.Pointer {
	// On 64-bit, people with ulimit -v set complain if we reserve too
	// much address space. Instead, assume that the reservation is okay
	// if we can reserve at least 64K and check the assumption in SysMap.
	// Only user-mode Linux (UML) rejects these requests.
	if sys.PtrSize == 8 && uint64(n) > 1<<32 {
		p := mmap_fixed(v, 64<<10, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
		if p != v {
			if uintptr(p) >= 4096 {
				munmap(p, 64<<10)
			}
			return nil
		}
		munmap(p, 64<<10)
		*reserved = false
		return v
	}

	p := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
	if uintptr(p) < 4096 {
		return nil
	}
	*reserved = true
	return p
}

func sysMap(v unsafe.Pointer, n uintptr, reserved bool, sysStat *uint64) {
	mSysStatInc(sysStat, n)

	// On 64-bit, we don't actually have v reserved, so tread carefully.
	if !reserved {
		p := mmap_fixed(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
		if uintptr(p) == _ENOMEM {
			throw("runtime: out of memory")
		}
		if p != v {
			print("runtime: address space conflict: map(", v, ") = ", p, "\n")
			throw("runtime: address space conflict")
		}
		return
	}

	p := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
	if uintptr(p) == _ENOMEM {
		throw("runtime: out of memory")
	}
	if p != v {
		throw("runtime: cannot map pages in arena address space")
	}
}
<commit_msg>runtime: check that sysUnused is always physical-page aligned<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import (
	"runtime/internal/sys"
	"unsafe"
)

const (
	_PAGE_SIZE = sys.PhysPageSize
	_EACCES    = 13
)

// NOTE: vec must be just 1 byte long here.
// Mincore returns ENOMEM if any of the pages are unmapped,
// but we want to know that all of the pages are unmapped.
// To make these the same, we can only ask about one page
// at a time. See golang.org/issue/7476.
var addrspace_vec [1]byte

func addrspace_free(v unsafe.Pointer, n uintptr) bool {
	var chunk uintptr
	for off := uintptr(0); off < n; off += chunk {
		chunk = _PAGE_SIZE * uintptr(len(addrspace_vec))
		if chunk > (n - off) {
			chunk = n - off
		}
		errval := mincore(unsafe.Pointer(uintptr(v)+off), chunk, &addrspace_vec[0])
		// ENOMEM means unmapped, which is what we want.
		// Anything else we assume means the pages are mapped.
		if errval != -_ENOMEM {
			return false
		}
	}
	return true
}

func mmap_fixed(v unsafe.Pointer, n uintptr, prot, flags, fd int32, offset uint32) unsafe.Pointer {
	p := mmap(v, n, prot, flags, fd, offset)
	// On some systems, mmap ignores v without
	// MAP_FIXED, so retry if the address space is free.
	if p != v && addrspace_free(v, n) {
		if uintptr(p) > 4096 {
			munmap(p, n)
		}
		p = mmap(v, n, prot, flags|_MAP_FIXED, fd, offset)
	}
	return p
}

// Don't split the stack as this method may be invoked without a valid G, which
// prevents us from allocating more stack.
//go:nosplit
func sysAlloc(n uintptr, sysStat *uint64) unsafe.Pointer {
	p := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
	if uintptr(p) < 4096 {
		if uintptr(p) == _EACCES {
			print("runtime: mmap: access denied\n")
			exit(2)
		}
		if uintptr(p) == _EAGAIN {
			print("runtime: mmap: too much locked memory (check 'ulimit -l').\n")
			exit(2)
		}
		return nil
	}
	mSysStatInc(sysStat, n)
	return p
}

func sysUnused(v unsafe.Pointer, n uintptr) {
	// By default, Linux's "transparent huge page" support will
	// merge pages into a huge page if there's even a single
	// present regular page, undoing the effects of the DONTNEED
	// below. On amd64, that means khugepaged can turn a single
	// 4KB page to 2MB, bloating the process's RSS by as much as
	// 512X. (See issue #8832 and Linux kernel bug
	// https://bugzilla.kernel.org/show_bug.cgi?id=93111)
	//
	// To work around this, we explicitly disable transparent huge
	// pages when we release pages of the heap. However, we have
	// to do this carefully because changing this flag tends to
	// split the VMA (memory mapping) containing v in to three
	// VMAs in order to track the different values of the
	// MADV_NOHUGEPAGE flag in the different regions. There's a
	// default limit of 65530 VMAs per address space (sysctl
	// vm.max_map_count), so we must be careful not to create too
	// many VMAs (see issue #12233).
	//
	// Since huge pages are huge, there's little use in adjusting
	// the MADV_NOHUGEPAGE flag on a fine granularity, so we avoid
	// exploding the number of VMAs by only adjusting the
	// MADV_NOHUGEPAGE flag on a large granularity. This still
	// gets most of the benefit of huge pages while keeping the
	// number of VMAs under control. With hugePageSize = 2MB, even
	// a pessimal heap can reach 128GB before running out of VMAs.
	if sys.HugePageSize != 0 {
		var s uintptr = sys.HugePageSize // division by constant 0 is a compile-time error :(

		// If it's a large allocation, we want to leave huge
		// pages enabled. Hence, we only adjust the huge page
		// flag on the huge pages containing v and v+n-1, and
		// only if those aren't aligned.
		var head, tail uintptr
		if uintptr(v)%s != 0 {
			// Compute huge page containing v.
			head = uintptr(v) &^ (s - 1)
		}
		if (uintptr(v)+n)%s != 0 {
			// Compute huge page containing v+n-1.
			tail = (uintptr(v) + n - 1) &^ (s - 1)
		}

		// Note that madvise will return EINVAL if the flag is
		// already set, which is quite likely. We ignore
		// errors.
		if head != 0 && head+sys.HugePageSize == tail {
			// head and tail are different but adjacent,
			// so do this in one call.
			madvise(unsafe.Pointer(head), 2*sys.HugePageSize, _MADV_NOHUGEPAGE)
		} else {
			// Advise the huge pages containing v and v+n-1.
			if head != 0 {
				madvise(unsafe.Pointer(head), sys.HugePageSize, _MADV_NOHUGEPAGE)
			}
			if tail != 0 && tail != head {
				madvise(unsafe.Pointer(tail), sys.HugePageSize, _MADV_NOHUGEPAGE)
			}
		}
	}

	if uintptr(v)&(sys.PhysPageSize-1) != 0 || n&(sys.PhysPageSize-1) != 0 {
		// madvise will round this to any physical page
		// *covered* by this range, so an unaligned madvise
		// will release more memory than intended.
		throw("unaligned sysUnused")
	}

	madvise(v, n, _MADV_DONTNEED)
}

func sysUsed(v unsafe.Pointer, n uintptr) {
	if sys.HugePageSize != 0 {
		// Partially undo the NOHUGEPAGE marks from sysUnused
		// for whole huge pages between v and v+n. This may
		// leave huge pages off at the end points v and v+n
		// even though allocations may cover these entire huge
		// pages. We could detect this and undo NOHUGEPAGE on
		// the end points as well, but it's probably not worth
		// the cost because when neighboring allocations are
		// freed sysUnused will just set NOHUGEPAGE again.
		var s uintptr = sys.HugePageSize

		// Round v up to a huge page boundary.
		beg := (uintptr(v) + (s - 1)) &^ (s - 1)
		// Round v+n down to a huge page boundary.
		end := (uintptr(v) + n) &^ (s - 1)

		if beg < end {
			madvise(unsafe.Pointer(beg), end-beg, _MADV_HUGEPAGE)
		}
	}
}

// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
//go:nosplit
func sysFree(v unsafe.Pointer, n uintptr, sysStat *uint64) {
	mSysStatDec(sysStat, n)
	munmap(v, n)
}

func sysFault(v unsafe.Pointer, n uintptr) {
	mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, -1, 0)
}

func sysReserve(v unsafe.Pointer, n uintptr, reserved *bool) unsafe.Pointer {
	// On 64-bit, people with ulimit -v set complain if we reserve too
	// much address space. Instead, assume that the reservation is okay
	// if we can reserve at least 64K and check the assumption in SysMap.
	// Only user-mode Linux (UML) rejects these requests.
	if sys.PtrSize == 8 && uint64(n) > 1<<32 {
		p := mmap_fixed(v, 64<<10, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
		if p != v {
			if uintptr(p) >= 4096 {
				munmap(p, 64<<10)
			}
			return nil
		}
		munmap(p, 64<<10)
		*reserved = false
		return v
	}

	p := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
	if uintptr(p) < 4096 {
		return nil
	}
	*reserved = true
	return p
}

func sysMap(v unsafe.Pointer, n uintptr, reserved bool, sysStat *uint64) {
	mSysStatInc(sysStat, n)

	// On 64-bit, we don't actually have v reserved, so tread carefully.
	if !reserved {
		p := mmap_fixed(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
		if uintptr(p) == _ENOMEM {
			throw("runtime: out of memory")
		}
		if p != v {
			print("runtime: address space conflict: map(", v, ") = ", p, "\n")
			throw("runtime: address space conflict")
		}
		return
	}

	p := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
	if uintptr(p) == _ENOMEM {
		throw("runtime: out of memory")
	}
	if p != v {
		throw("runtime: cannot map pages in arena address space")
	}
}
<|endoftext|>


<commit_before>package redis

import (
	"time"

	"github.com/garyburd/redigo/redis"
)

var PrefixKey = "ratelimit:"

type bucketStore struct {
	pool *redis.Pool

	rate          int
	windowSeconds int
}

// New creates new in-memory token bucket store.
func New(pool *redis.Pool) *bucketStore {
	return &bucketStore{
		pool: pool,
	}
}

func (s *bucketStore) InitRate(rate int, window time.Duration) {
	s.rate = rate
	s.windowSeconds = int(window / time.Second)
	if s.windowSeconds <= 1 {
		s.windowSeconds = 1
	}
}

// Take implements TokenBucketStore interface. It takes token from a bucket
// referenced by a given key, if available.
func (s *bucketStore) Take(key string) (bool, int, time.Time, error) {
	c := s.pool.Get()
	defer c.Close()

	// Number of tokens in the bucket.
	bucketLen, err := redis.Int(c.Do("LLEN", PrefixKey+key))
	if err != nil {
		return false, 0, time.Time{}, err
	}

	// Bucket is full.
	if bucketLen >= s.rate {
		return false, 0, time.Time{}, nil
	}

	if bucketLen > 0 {
		// Bucket most probably exists, try to push a new token into it.
		// If RPUSHX returns 0 (ie. key expired between LLEN and RPUSHX), we need
		// to fall-back to RPUSH without returning error.
		c.Send("MULTI")
		c.Send("RPUSHX", PrefixKey+key, "")
		reply, err := redis.Ints(c.Do("EXEC"))
		if err != nil {
			return false, 0, time.Time{}, err
		}
		bucketLen = reply[0]
		if bucketLen > 0 {
			return true, s.rate - bucketLen - 1, time.Time{}, nil
		}
	}

	c.Send("MULTI")
	c.Send("RPUSH", PrefixKey+key, "")
	c.Send("EXPIRE", PrefixKey+key, s.windowSeconds)
	if _, err := c.Do("EXEC"); err != nil {
		return false, 0, time.Time{}, err
	}

	return true, s.rate - bucketLen - 1, time.Time{}, nil
}
<commit_msg>Skip few takes on unhealthy Redis<commit_after>package redis

import (
	"errors"
	"time"

	"github.com/garyburd/redigo/redis"
)

var (
	PrefixKey    = "ratelimit:"
	ErrUnhealthy = errors.New("redis is not healthy")
)

const skipOnUnhealthy = 1000

type bucketStore struct {
	pool *redis.Pool

	rate          int
	windowSeconds int
	skip          int
}

// New creates new in-memory token bucket store.
func New(pool *redis.Pool) *bucketStore {
	return &bucketStore{
		pool: pool,
	}
}

func (s *bucketStore) InitRate(rate int, window time.Duration) {
	s.rate = rate
	s.windowSeconds = int(window / time.Second)
	if s.windowSeconds <= 1 {
		s.windowSeconds = 1
	}
}

// Take implements TokenBucketStore interface. It takes token from a bucket
// referenced by a given key, if available.
func (s *bucketStore) Take(key string) (bool, int, time.Time, error) {
	if s.skip > 0 {
		s.skip--
		return false, 0, time.Time{}, ErrUnhealthy
	}
	c := s.pool.Get()
	defer c.Close()

	// Number of tokens in the bucket.
	bucketLen, err := redis.Int(c.Do("LLEN", PrefixKey+key))
	if err != nil {
		s.skip = skipOnUnhealthy
		return false, 0, time.Time{}, err
	}

	// Bucket is full.
	if bucketLen >= s.rate {
		return false, 0, time.Time{}, nil
	}

	if bucketLen > 0 {
		// Bucket most probably exists, try to push a new token into it.
		// If RPUSHX returns 0 (ie. key expired between LLEN and RPUSHX), we need
		// to fall-back to RPUSH without returning error.
		c.Send("MULTI")
		c.Send("RPUSHX", PrefixKey+key, "")
		reply, err := redis.Ints(c.Do("EXEC"))
		if err != nil {
			s.skip = skipOnUnhealthy
			return false, 0, time.Time{}, err
		}
		bucketLen = reply[0]
		if bucketLen > 0 {
			return true, s.rate - bucketLen - 1, time.Time{}, nil
		}
	}

	c.Send("MULTI")
	c.Send("RPUSH", PrefixKey+key, "")
	c.Send("EXPIRE", PrefixKey+key, s.windowSeconds)
	if _, err := c.Do("EXEC"); err != nil {
		s.skip = skipOnUnhealthy
		return false, 0, time.Time{}, err
	}

	return true, s.rate - bucketLen - 1, time.Time{}, nil
}
<|endoftext|>


<commit_before>package main

import (
	// Flogo action
	_ "github.com/TIBCOSoftware/flogo-contrib/action/flow"

	// Activities from https://github.com/TIBCOSoftware/flogo-contrib
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/actreply"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/actreturn"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/aggregate"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/app"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/awsiot"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/awssns"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/coap"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/couchbase"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/counter"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/error"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/gpio"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/inference"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/kafkapub"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/lambda"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/log"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/mapper"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/mongodb"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/rest"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/subflow"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/twilio"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/wsmessage"

	// Triggers from https://github.com/TIBCOSoftware/flogo-contrib
	_ "github.com/TIBCOSoftware/flogo-contrib/trigger/cli"
	_ "github.com/TIBCOSoftware/flogo-contrib/trigger/coap"
	_ "github.com/TIBCOSoftware/flogo-contrib/trigger/kafkasub"
	_ "github.com/TIBCOSoftware/flogo-contrib/trigger/lambda"
	_ "github.com/TIBCOSoftware/flogo-contrib/trigger/mqtt"
	_ "github.com/TIBCOSoftware/flogo-contrib/trigger/rest"
	_ "github.com/TIBCOSoftware/flogo-contrib/trigger/timer"

	// Activities from https://github.com/retgits/flogo-components
	_ "github.com/retgits/flogo-components/activity/addtodate"
	_ "github.com/retgits/flogo-components/activity/amazons3"
	_ "github.com/retgits/flogo-components/activity/amazonsqssend"
	_ "github.com/retgits/flogo-components/activity/awsssm"
	_ "github.com/retgits/flogo-components/activity/commandparser"
	_ "github.com/retgits/flogo-components/activity/downloadfile"
	_ "github.com/retgits/flogo-components/activity/dynamodbinsert"
	_ "github.com/retgits/flogo-components/activity/dynamodbquery"
	_ "github.com/retgits/flogo-components/activity/githubissues"
	_ "github.com/retgits/flogo-components/activity/gzip"
	_ "github.com/retgits/flogo-components/activity/iftttwebhook"
	_ "github.com/retgits/flogo-components/activity/null"
	_ "github.com/retgits/flogo-components/activity/pubnubpublisher"
	_ "github.com/retgits/flogo-components/activity/queryparser"
	_ "github.com/retgits/flogo-components/activity/randomnumber"
	_ "github.com/retgits/flogo-components/activity/randomstring"
	_ "github.com/retgits/flogo-components/activity/tomlreader"
	_ "github.com/retgits/flogo-components/activity/trellocard"
	_ "github.com/retgits/flogo-components/activity/writetofile"

	// Triggers from https://github.com/retgits/flogo-components
	_ "github.com/retgits/flogo-components/trigger/pubnubsubscriber"
)
<commit_msg>Removing inference activity as it doesn't work on Flogo Web<commit_after>package main

import (
	// Flogo action
	_ "github.com/TIBCOSoftware/flogo-contrib/action/flow"

	// Activities from https://github.com/TIBCOSoftware/flogo-contrib
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/actreply"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/actreturn"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/aggregate"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/app"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/awsiot"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/awssns"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/coap"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/couchbase"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/counter"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/error"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/gpio"

	//_ "github.com/TIBCOSoftware/flogo-contrib/activity/inference"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/kafkapub"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/lambda"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/log"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/mapper"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/mongodb"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/rest"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/subflow"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/twilio"
	_ "github.com/TIBCOSoftware/flogo-contrib/activity/wsmessage"

	// Triggers from https://github.com/TIBCOSoftware/flogo-contrib
	_ "github.com/TIBCOSoftware/flogo-contrib/trigger/cli"
	_ "github.com/TIBCOSoftware/flogo-contrib/trigger/coap"
	_ "github.com/TIBCOSoftware/flogo-contrib/trigger/kafkasub"
	_ "github.com/TIBCOSoftware/flogo-contrib/trigger/lambda"
	_ "github.com/TIBCOSoftware/flogo-contrib/trigger/mqtt"
	_ "github.com/TIBCOSoftware/flogo-contrib/trigger/rest"
	_ "github.com/TIBCOSoftware/flogo-contrib/trigger/timer"

	// Activities from https://github.com/retgits/flogo-components
	_ "github.com/retgits/flogo-components/activity/addtodate"
	_ "github.com/retgits/flogo-components/activity/amazons3"
	_ "github.com/retgits/flogo-components/activity/amazonsqssend"
	_ "github.com/retgits/flogo-components/activity/awsssm"
	_ "github.com/retgits/flogo-components/activity/commandparser"
	_ "github.com/retgits/flogo-components/activity/downloadfile"
	_ "github.com/retgits/flogo-components/activity/dynamodbinsert"
	_ "github.com/retgits/flogo-components/activity/dynamodbquery"
	_ "github.com/retgits/flogo-components/activity/githubissues"
	_ "github.com/retgits/flogo-components/activity/gzip"
	_ "github.com/retgits/flogo-components/activity/iftttwebhook"
	_ "github.com/retgits/flogo-components/activity/null"
	_ "github.com/retgits/flogo-components/activity/pubnubpublisher"
	_ "github.com/retgits/flogo-components/activity/queryparser"
	_ "github.com/retgits/flogo-components/activity/randomnumber"
	_ "github.com/retgits/flogo-components/activity/randomstring"
	_ "github.com/retgits/flogo-components/activity/tomlreader"
	_ "github.com/retgits/flogo-components/activity/trellocard"
	_ "github.com/retgits/flogo-components/activity/writetofile"

	// Triggers from https://github.com/retgits/flogo-components
	_ "github.com/retgits/flogo-components/trigger/pubnubsubscriber"
)
<|endoftext|>


<commit_before>package flow

import (
	"os"
	"sync"
)

/*
There are 3 running mode:
1. as normal program
	If not in distributed mode, it should not be intercepted.
2. "-driver" mode to drive in distributed mode
	context runner will register
3. "-task.[context|taskGroup].id" mode to run task in distributed mode
*/

var contextRunner ContextRunner
var taskRunner TaskRunner

// Invoked by driver task runner
func RegisterContextRunner(r ContextRunner) {
	contextRunner = r
}
func RegisterTaskRunner(r TaskRunner) {
	taskRunner = r
}

type ContextRunner interface {
	Run(*FlowContext)
	IsDriverMode() bool
	IsDriverPlotMode() bool
	Plot(*FlowContext)
}

type TaskRunner interface {
	Run(fc *FlowContext)
	IsTaskMode() bool
}

func Ready() {
	if taskRunner.IsTaskMode() {
		for _, fc := range Contexts {
			fc.Run()
		}
		os.Exit(0)
	} else if contextRunner.IsDriverMode() {
		if contextRunner.IsDriverPlotMode() {
			for _, fc := range Contexts {
				contextRunner.Plot(fc)
			}
			os.Exit(0)
		}
	} else {
	}
}

func (fc *FlowContext) Run() {

	if taskRunner.IsTaskMode() {
		taskRunner.Run(fc)
	} else if contextRunner.IsDriverMode() {
		contextRunner.Run(fc)
	} else {
		fc.runFlowContextInStandAloneMode()
	}
}

func (fc *FlowContext) runFlowContextInStandAloneMode() {

	var wg sync.WaitGroup

	isDatasetStarted := make(map[int]bool)

	// start all task edges
	for _, step := range fc.Steps {
		for _, input := range step.Inputs {
			if _, ok := isDatasetStarted[input.Id]; !ok {
				wg.Add(1)
				go func(step *Step) {
					defer wg.Done()
					input.RunDatasetInStandAloneMode()
				}(step)
				isDatasetStarted[input.Id] = true
			}
		}
		wg.Add(1)
		go func(step *Step) {
			defer wg.Done()
			step.RunStep()
		}(step)

		if step.Output != nil {
			if _, ok := isDatasetStarted[step.Output.Id]; !ok {
				wg.Add(1)
				go func(step *Step) {
					defer wg.Done()
					// println(step.Name, "start output dataset", step.Output.Id)
					step.Output.RunDatasetInStandAloneMode()
				}(step)
				isDatasetStarted[step.Output.Id] = true
			}
		}
	}
	wg.Wait()
}
<commit_msg>fix issue when running locally<commit_after>package flow

import (
	"os"
	"sync"
)

/*
There are 3 running mode:
1. as normal program
	If not in distributed mode, it should not be intercepted.
2. "-driver" mode to drive in distributed mode
	context runner will register
3. "-task.[context|taskGroup].id" mode to run task in distributed mode
*/

var contextRunner ContextRunner
var taskRunner TaskRunner

// Invoked by driver task runner
func RegisterContextRunner(r ContextRunner) {
	contextRunner = r
}
func RegisterTaskRunner(r TaskRunner) {
	taskRunner = r
}

type ContextRunner interface {
	Run(*FlowContext)
	IsDriverMode() bool
	IsDriverPlotMode() bool
	Plot(*FlowContext)
}

type TaskRunner interface {
	Run(fc *FlowContext)
	IsTaskMode() bool
}

func Ready() {
	if taskRunner.IsTaskMode() {
		for _, fc := range Contexts {
			fc.Run()
		}
		os.Exit(0)
	} else if contextRunner.IsDriverMode() {
		if contextRunner.IsDriverPlotMode() {
			for _, fc := range Contexts {
				contextRunner.Plot(fc)
			}
			os.Exit(0)
		}
	} else {
	}
}

func (fc *FlowContext) Run() {

	if taskRunner != nil && taskRunner.IsTaskMode() {
		taskRunner.Run(fc)
	} else if contextRunner != nil && contextRunner.IsDriverMode() {
		contextRunner.Run(fc)
	} else {
		fc.runFlowContextInStandAloneMode()
	}
}

func (fc *FlowContext) runFlowContextInStandAloneMode() {

	var wg sync.WaitGroup

	isDatasetStarted := make(map[int]bool)

	// start all task edges
	for _, step := range fc.Steps {
		for _, input := range step.Inputs {
			if _, ok := isDatasetStarted[input.Id]; !ok {
				wg.Add(1)
				go func(step *Step) {
					defer wg.Done()
					input.RunDatasetInStandAloneMode()
				}(step)
				isDatasetStarted[input.Id] = true
			}
		}
		wg.Add(1)
		go func(step *Step) {
			defer wg.Done()
			step.RunStep()
		}(step)

		if step.Output != nil {
			if _, ok := isDatasetStarted[step.Output.Id]; !ok {
				wg.Add(1)
				go func(step *Step) {
					defer wg.Done()
					// println(step.Name, "start output dataset", step.Output.Id)
					step.Output.RunDatasetInStandAloneMode()
				}(step)
				isDatasetStarted[step.Output.Id] = true
			}
		}
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/deevatech/manager/models/tests"
	"github.com/deevatech/manager/runner"
	. "github.com/deevatech/manager/types"
	"github.com/gin-gonic/gin"
	"log"
	"net/http"
	"os"
	"strconv"
)

func init() {
	log.Println("Deeva Manager!")
}

func main() {
	router := gin.Default()
	router.POST("/run", handleRunRequest)
	router.GET("/tests/:id", handleTestLookupRequest)
	router.POST("/tests/:id/submit", handleTestSubmitRequest)

	port := os.Getenv("DEEVA_MANAGER_PORT")
	if len(port) == 0 {
		port = "8080"
	}

	log.Printf("Starting in %s mode on port %s\n", gin.Mode(), port)
	host := fmt.Sprintf(":%s", port)
	router.Run(host)
}

func handleRunRequest(c *gin.Context) {
	var run RunParams
	if errParams := c.BindJSON(&run); errParams == nil {
		if result, errRun := runner.Run(run); errRun == nil {
			c.JSON(http.StatusOK, result)
		} else {
			c.JSON(http.StatusBadRequest, gin.H{
				"error": errRun,
			})
		}
	} else {
		c.JSON(http.StatusBadRequest, gin.H{
			"error": errParams,
		})
	}
}

func handleTestLookupRequest(c *gin.Context) {
	id, err := strconv.ParseUint(c.Param("id"), 10, 64)
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{
			"error": err,
		})
		return
	}

	test := tests.FindById(id)
	c.JSON(http.StatusOK, test)
}

func handleTestSubmitRequest(c *gin.Context) {
	id, err := strconv.ParseUint(c.Param("id"), 10, 64)
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{
			"error": err,
		})
		return
	}

	// lookup test
	test := tests.FindById(id)

	var submit TestSubmitParams
	if errParams := c.BindJSON(&submit); errParams == nil {
		run := RunParams{
			Language: test.Language,
			Source:   submit.Code,
			Spec:     test.Spec,
		}
		if result, errRun := runner.Run(run); errRun == nil {
			c.JSON(http.StatusOK, result)
		} else {
			c.JSON(http.StatusBadRequest, gin.H{
				"error": errRun,
			})
		}
	} else {
		c.JSON(http.StatusBadRequest, gin.H{
			"error": errParams,
		})
	}
}
<commit_msg>Add Access-Control-Allow-Origin header to all endpoints<commit_after>package main

import (
	"fmt"
	"github.com/deevatech/manager/models/tests"
	"github.com/deevatech/manager/runner"
	. "github.com/deevatech/manager/types"
	"github.com/gin-gonic/gin"
	"log"
	"net/http"
	"os"
	"strconv"
)

func init() {
	log.Println("Deeva Manager!")
}

func main() {
	router := gin.Default()
	router.POST("/run", handleRunRequest)
	router.GET("/tests/:id", handleTestLookupRequest)
	router.POST("/tests/:id/submit", handleTestSubmitRequest)

	port := os.Getenv("DEEVA_MANAGER_PORT")
	if len(port) == 0 {
		port = "8080"
	}

	log.Printf("Starting in %s mode on port %s\n", gin.Mode(), port)
	host := fmt.Sprintf(":%s", port)
	router.Run(host)
}

func handleRunRequest(c *gin.Context) {
	c.Header("Access-Control-Allow-Origin", "*")

	var run RunParams
	if errParams := c.BindJSON(&run); errParams == nil {
		if result, errRun := runner.Run(run); errRun == nil {
			c.JSON(http.StatusOK, result)
		} else {
			c.JSON(http.StatusBadRequest, gin.H{
				"error": errRun,
			})
		}
	} else {
		c.JSON(http.StatusBadRequest, gin.H{
			"error": errParams,
		})
	}
}

func handleTestLookupRequest(c *gin.Context) {
	c.Header("Access-Control-Allow-Origin", "*")

	id, err := strconv.ParseUint(c.Param("id"), 10, 64)
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{
			"error": err,
		})
		return
	}

	test := tests.FindById(id)
	c.JSON(http.StatusOK, test)
}

func handleTestSubmitRequest(c *gin.Context) {
	c.Header("Access-Control-Allow-Origin", "*")

	id, err := strconv.ParseUint(c.Param("id"), 10, 64)
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{
			"error": err,
		})
		return
	}

	// lookup test
	test := tests.FindById(id)

	var submit TestSubmitParams
	if errParams := c.BindJSON(&submit); errParams == nil {
		run := RunParams{
			Language: test.Language,
			Source:   submit.Code,
			Spec:     test.Spec,
		}
		if result, errRun := runner.Run(run); errRun == nil {
			c.JSON(http.StatusOK, result)
		} else {
			c.JSON(http.StatusBadRequest, gin.H{
				"error": errRun,
			})
		}
	} else {
		c.JSON(http.StatusBadRequest, gin.H{
			"error": errParams,
		})
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"github.com/globocom/config"
	"github.com/globocom/tsuru/db"
	ftesting "github.com/globocom/tsuru/fs/testing"
	"launchpad.net/gocheck"
	"os"
	"testing"
)

func Test(t *testing.T) { gocheck.TestingT(t) }

type S struct {
	collName      string
	imageCollName string
	conn          *db.Storage
	gitHost       string
	repoNamespace string
	deployCmd     string
	runBin        string
	runArgs       string
	port          string
	hostAddr      string
}

var _ = gocheck.Suite(&S{})

func (s *S) SetUpSuite(c *gocheck.C) {
	s.collName = "docker_unit"
	s.imageCollName = "docker_image"
	s.gitHost = "my.gandalf.com"
	s.repoNamespace = "tsuru"
	s.hostAddr = "10.0.0.4"
	config.Set("git:host", s.gitHost)
	config.Set("database:url", "127.0.0.1:27017")
	config.Set("database:name", "docker_provision_tests_s")
	config.Set("docker:repository-namespace", s.repoNamespace)
	config.Set("docker:binary", "docker")
	config.Set("docker:router", "fake")
	config.Set("docker:collection", s.collName)
	config.Set("docker:host-address", hostAddr)
	config.Set("docker:deploy-cmd", "/var/lib/tsuru/deploy")
	config.Set("docker:run-cmd:bin", "/usr/local/bin/circusd")
	config.Set("docker:run-cmd:args", "/etc/circus/circus.ini")
	config.Set("docker:run-cmd:port", "8888")
	config.Set("docker:ssh:add-key-cmd", "/var/lib/tsuru/add-key")
	s.deployCmd = "/var/lib/tsuru/deploy"
	s.runBin = "/usr/local/bin/circusd"
	s.runArgs = "/etc/circus/circus.ini"
	s.port = "8888"
	var err error
	s.conn, err = db.Conn()
	c.Assert(err, gocheck.IsNil)
	fsystem = &ftesting.RecordingFs{}
	f, err := fsystem.Create(os.ExpandEnv("${HOME}/.ssh/id_rsa.pub"))
	c.Assert(err, gocheck.IsNil)
	f.Write([]byte("key-content"))
	f.Close()
}

func (s *S) TearDownSuite(c *gocheck.C) {
	s.conn.Collection(s.collName).Database.DropDatabase()
	fsystem = nil
}
<commit_msg>provision/docker: fix SetUpSuite<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"github.com/globocom/config"
	"github.com/globocom/tsuru/db"
	ftesting "github.com/globocom/tsuru/fs/testing"
	"launchpad.net/gocheck"
	"os"
	"testing"
)

func Test(t *testing.T) { gocheck.TestingT(t) }

type S struct {
	collName      string
	imageCollName string
	conn          *db.Storage
	gitHost       string
	repoNamespace string
	deployCmd     string
	runBin        string
	runArgs       string
	port          string
	hostAddr      string
}

var _ = gocheck.Suite(&S{})

func (s *S) SetUpSuite(c *gocheck.C) {
	s.collName = "docker_unit"
	s.imageCollName = "docker_image"
	s.gitHost = "my.gandalf.com"
	s.repoNamespace = "tsuru"
	s.hostAddr = "10.0.0.4"
	config.Set("git:host", s.gitHost)
	config.Set("database:url", "127.0.0.1:27017")
	config.Set("database:name", "docker_provision_tests_s")
	config.Set("docker:repository-namespace", s.repoNamespace)
	config.Set("docker:binary", "docker")
	config.Set("docker:router", "fake")
	config.Set("docker:collection", s.collName)
	config.Set("docker:host-address", s.hostAddr)
	config.Set("docker:deploy-cmd", "/var/lib/tsuru/deploy")
	config.Set("docker:run-cmd:bin", "/usr/local/bin/circusd")
	config.Set("docker:run-cmd:args", "/etc/circus/circus.ini")
	config.Set("docker:run-cmd:port", "8888")
	config.Set("docker:ssh:add-key-cmd", "/var/lib/tsuru/add-key")
	s.deployCmd = "/var/lib/tsuru/deploy"
	s.runBin = "/usr/local/bin/circusd"
	s.runArgs = "/etc/circus/circus.ini"
	s.port = "8888"
	var err error
	s.conn, err = db.Conn()
	c.Assert(err, gocheck.IsNil)
	fsystem = &ftesting.RecordingFs{}
	f, err := fsystem.Create(os.ExpandEnv("${HOME}/.ssh/id_rsa.pub"))
	c.Assert(err, gocheck.IsNil)
	f.Write([]byte("key-content"))
	f.Close()
}

func (s *S) TearDownSuite(c *gocheck.C) {
	s.conn.Collection(s.collName).Database.DropDatabase()
	fsystem = nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package adapter implements a controller that interacts with gerrit instances
package adapter

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/gerrit/client"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/pjutil"
)

type kubeClient interface {
	CreateProwJob(kube.ProwJob) (kube.ProwJob, error)
}

type gerritClient interface {
	QueryChanges(lastUpdate time.Time, rateLimit int) map[string][]client.ChangeInfo
	GetBranchRevision(instance, project, branch string) (string, error)
	SetReview(instance, id, revision, message string, labels map[string]string) error
}

type configAgent interface {
	Config() *config.Config
}

// Controller manages gerrit changes.
type Controller struct {
	ca configAgent
	kc kubeClient
	gc gerritClient

	lastSyncFallback string

	lastUpdate time.Time
}

// NewController returns a new gerrit controller client
func NewController(lastSyncFallback, cookiefilePath string, projects map[string][]string, kc *kube.Client, ca *config.Agent) (*Controller, error) {
	if lastSyncFallback == "" {
		return nil, errors.New("empty lastSyncFallback")
	}

	var lastUpdate time.Time
	if buf, err := ioutil.ReadFile(lastSyncFallback); err == nil {
		unix, err := strconv.ParseInt(string(buf), 10, 64)
		if err != nil {
			return nil, err
		}
		lastUpdate = time.Unix(unix, 0)
	} else if err != nil && !os.IsNotExist(err) {
		return nil, fmt.Errorf("failed to read lastSyncFallback: %v", err)
	} else {
		logrus.Warnf("lastSyncFallback not found: %s", lastSyncFallback)
		lastUpdate = time.Now()
	}

	c, err := client.NewClient(projects)
	if err != nil {
		return nil, err
	}
	c.Start(cookiefilePath)

	return &Controller{
		kc:               kc,
		ca:               ca,
		gc:               c,
		lastUpdate:       lastUpdate,
		lastSyncFallback: lastSyncFallback,
	}, nil
}

func copyFile(srcPath, destPath string) error {
	// fallback to copying the file instead
	src, err := os.Open(srcPath)
	if err != nil {
		return err
	}
	dst, err := os.OpenFile(destPath, os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	_, err = io.Copy(dst, src)
	if err != nil {
		return err
	}
	dst.Sync()
	dst.Close()
	src.Close()
	return nil
}

// SaveLastSync saves last sync time in Unix to a volume
func (c *Controller) SaveLastSync(lastSync time.Time) error {
	if c.lastSyncFallback == "" {
		return nil
	}

	lastSyncUnix := strconv.FormatInt(lastSync.Unix(), 10)
	logrus.Infof("Writing last sync: %s", lastSyncUnix)

	tempFile, err := ioutil.TempFile(filepath.Dir(c.lastSyncFallback), "temp")
	if err != nil {
		return err
	}
	defer os.Remove(tempFile.Name())

	err = ioutil.WriteFile(tempFile.Name(), []byte(lastSyncUnix), 0644)
	if err != nil {
		return err
	}

	err = os.Rename(tempFile.Name(), c.lastSyncFallback)
	if err != nil {
		logrus.WithError(err).Info("Rename failed, fallback to copyfile")
		return copyFile(tempFile.Name(), c.lastSyncFallback)
	}
	return nil
}

// Sync looks for newly made gerrit changes
// and creates prowjobs according to specs
func (c *Controller) Sync() error {
	// gerrit timestamp only has second precision
	syncTime := time.Now().Truncate(time.Second)

	for instance, changes := range c.gc.QueryChanges(c.lastUpdate, c.ca.Config().Gerrit.RateLimit) {
		for _, change := range changes {
			if err := c.ProcessChange(instance, change); err != nil {
				logrus.WithError(err).Errorf("Failed process change %v", change.CurrentRevision)
			}
		}

		logrus.Infof("Processed %d changes for instance %s", len(changes), instance)
	}

	c.lastUpdate = syncTime
	if err := c.SaveLastSync(syncTime); err != nil {
		logrus.WithError(err).Errorf("last sync %v, cannot save to path %v", syncTime, c.lastSyncFallback)
	}

	return nil
}

func makeCloneURI(instance, project string) (*url.URL, error) {
	u, err := url.Parse(instance)
	if err != nil {
		return nil, fmt.Errorf("instance %s is not a url: %v", instance, err)
	}
	if u.Host == "" {
		return nil, errors.New("instance does not set host")
	}
	if u.Path != "" {
		return nil, errors.New("instance cannot set path (this is set by project)")
	}
	u.Path = project
	return u, nil
}

// ProcessChange creates new presubmit prowjobs base off the gerrit changes
func (c *Controller) ProcessChange(instance string, change client.ChangeInfo) error {
	rev, ok := change.Revisions[change.CurrentRevision]
	if !ok {
		return fmt.Errorf("cannot find current revision for change %v", change.ID)
	}

	logger := logrus.WithField("gerrit change", change.Number)

	cloneURI, err := makeCloneURI(instance, change.Project)
	if err != nil {
		return fmt.Errorf("failed to create clone uri: %v", err)
	}

	baseSHA, err := c.gc.GetBranchRevision(instance, change.Project, change.Branch)
	if err != nil {
		return fmt.Errorf("failed to get SHA from base branch: %v", err)
	}

	triggeredJobs := []string{}

	switch change.Status {
	case client.Merged:
		postsubmits := c.ca.Config().Postsubmits[cloneURI.String()]
		postsubmits = append(postsubmits, c.ca.Config().Postsubmits[cloneURI.Host+"/"+cloneURI.Path]...)
		for _, spec := range postsubmits {
			kr := kube.Refs{
				Org:      cloneURI.Host,  // Something like android.googlesource.com
				Repo:     change.Project, // Something like platform/build
				BaseRef:  change.Branch,
				BaseSHA:  baseSHA,
				CloneURI: cloneURI.String(), // Something like https://android.googlesource.com/platform/build
				Pulls: []kube.Pull{
					{
						Number: change.Number,
						Author: rev.Commit.Author.Name,
						SHA:    change.CurrentRevision,
						Ref:    rev.Ref,
					},
				},
			}

			pj := pjutil.NewProwJobWithAnnotation(
				pjutil.PostsubmitSpec(spec, kr),
				map[string]string{
					client.GerritRevision: change.CurrentRevision,
				},
				map[string]string{
					client.GerritID:       change.ID,
					client.GerritInstance: instance,
				},
			)

			logger.WithFields(pjutil.ProwJobFields(&pj)).Infof("Creating a new postsubmit job for change %d.", change.Number)

			if _, err := c.kc.CreateProwJob(pj); err != nil {
				logger.WithError(err).Errorf("fail to create prowjob %v", pj)
			} else {
				triggeredJobs = append(triggeredJobs, spec.Name)
			}
		}
	case client.New:
		presubmits := c.ca.Config().Presubmits[cloneURI.String()]
		presubmits = append(presubmits, c.ca.Config().Presubmits[cloneURI.Host+"/"+cloneURI.Path]...)
		for _, spec := range presubmits {
			kr := kube.Refs{
				Org:      cloneURI.Host,  // Something like android.googlesource.com
				Repo:     change.Project, // Something like platform/build
				BaseRef:  change.Branch,
				BaseSHA:  baseSHA,
				CloneURI: cloneURI.String(), // Something like https://android.googlesource.com/platform/build
				Pulls: []kube.Pull{
					{
						Number: change.Number,
						Author: rev.Commit.Author.Name,
						SHA:    change.CurrentRevision,
						Ref:    rev.Ref,
					},
				},
			}

			// TODO(krzyzacy): Support AlwaysRun and RunIfChanged

			pj := pjutil.NewProwJobWithAnnotation(
				pjutil.PresubmitSpec(spec, kr),
				map[string]string{
					client.GerritRevision: change.CurrentRevision,
				},
				map[string]string{
					client.GerritID:       change.ID,
					client.GerritInstance: instance,
				},
			)

			logger.WithFields(pjutil.ProwJobFields(&pj)).Infof("Creating a new presubmit job for change %d.", change.Number)

			if _, err := c.kc.CreateProwJob(pj); err != nil {
				logger.WithError(err).Errorf("fail to create prowjob %v", pj)
			} else {
				triggeredJobs = append(triggeredJobs, spec.Name)
			}
		}
	}

	if len(triggeredJobs) > 0 {
		// comment back to gerrit
		message := fmt.Sprintf("Triggered %d prow jobs:", len(triggeredJobs))
		for _, job := range triggeredJobs {
			message += fmt.Sprintf("\n  * Name: %s", job)
		}

		if err := c.gc.SetReview(instance, change.ID, change.CurrentRevision, message, nil); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>take in exist label/annotations for gerrit presubmits :-\<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package adapter implements a controller that interacts with gerrit instances
package adapter

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/gerrit/client"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/pjutil"
)

type kubeClient interface {
	CreateProwJob(kube.ProwJob) (kube.ProwJob, error)
}

type gerritClient interface {
	QueryChanges(lastUpdate time.Time, rateLimit int) map[string][]client.ChangeInfo
	GetBranchRevision(instance, project, branch string) (string, error)
	SetReview(instance, id, revision, message string, labels map[string]string) error
}

type configAgent interface {
	Config() *config.Config
}

// Controller manages gerrit changes.
type Controller struct {
	ca configAgent
	kc kubeClient
	gc gerritClient

	lastSyncFallback string

	lastUpdate time.Time
}

// NewController returns a new gerrit controller client
func NewController(lastSyncFallback, cookiefilePath string, projects map[string][]string, kc *kube.Client, ca *config.Agent) (*Controller, error) {
	if lastSyncFallback == "" {
		return nil, errors.New("empty lastSyncFallback")
	}

	var lastUpdate time.Time
	if buf, err := ioutil.ReadFile(lastSyncFallback); err == nil {
		unix, err := strconv.ParseInt(string(buf), 10, 64)
		if err != nil {
			return nil, err
		}
		lastUpdate = time.Unix(unix, 0)
	} else if err != nil && !os.IsNotExist(err) {
		return nil, fmt.Errorf("failed to read lastSyncFallback: %v", err)
	} else {
		logrus.Warnf("lastSyncFallback not found: %s", lastSyncFallback)
		lastUpdate = time.Now()
	}

	c, err := client.NewClient(projects)
	if err != nil {
		return nil, err
	}
	c.Start(cookiefilePath)

	return &Controller{
		kc:               kc,
		ca:               ca,
		gc:               c,
		lastUpdate:       lastUpdate,
		lastSyncFallback: lastSyncFallback,
	}, nil
}

func copyFile(srcPath, destPath string) error {
	// fallback to copying the file instead
	src, err := os.Open(srcPath)
	if err != nil {
		return err
	}
	dst, err := os.OpenFile(destPath, os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	_, err = io.Copy(dst, src)
	if err != nil {
		return err
	}
	dst.Sync()
	dst.Close()
	src.Close()
	return nil
}

// SaveLastSync saves last sync time in Unix to a volume
func (c *Controller) SaveLastSync(lastSync time.Time) error {
	if c.lastSyncFallback == "" {
		return nil
	}

	lastSyncUnix := strconv.FormatInt(lastSync.Unix(), 10)
	logrus.Infof("Writing last sync: %s", lastSyncUnix)

	tempFile, err := ioutil.TempFile(filepath.Dir(c.lastSyncFallback), "temp")
	if err != nil {
		return err
	}
	defer os.Remove(tempFile.Name())

	err = ioutil.WriteFile(tempFile.Name(), []byte(lastSyncUnix), 0644)
	if err != nil {
		return err
	}

	err = os.Rename(tempFile.Name(), c.lastSyncFallback)
	if err != nil {
		logrus.WithError(err).Info("Rename failed, fallback to copyfile")
		return copyFile(tempFile.Name(), c.lastSyncFallback)
	}
	return nil
}

// Sync looks for newly made gerrit changes
// and creates prowjobs according to specs
func (c *Controller) Sync() error {
	// gerrit timestamp only has second precision
	syncTime := time.Now().Truncate(time.Second)

	for instance, changes := range c.gc.QueryChanges(c.lastUpdate, c.ca.Config().Gerrit.RateLimit) {
		for _, change := range changes {
			if err := c.ProcessChange(instance, change); err != nil {
				logrus.WithError(err).Errorf("Failed process change %v", change.CurrentRevision)
			}
		}

		logrus.Infof("Processed %d changes for instance %s", len(changes), instance)
	}

	c.lastUpdate = syncTime
	if err := c.SaveLastSync(syncTime); err != nil {
		logrus.WithError(err).Errorf("last sync %v, cannot save to path %v", syncTime, c.lastSyncFallback)
	}

	return nil
}

func makeCloneURI(instance, project string) (*url.URL, error) {
	u, err := url.Parse(instance)
	if err != nil {
		return nil, fmt.Errorf("instance %s is not a url: %v", instance, err)
	}
	if u.Host == "" {
		return nil, errors.New("instance does not set host")
	}
	if u.Path != "" {
		return nil, errors.New("instance cannot set path (this is set by project)")
	}
	u.Path = project
	return u, nil
}

// ProcessChange creates new presubmit prowjobs base off the gerrit changes
func (c *Controller) ProcessChange(instance string, change client.ChangeInfo) error {
	rev, ok := change.Revisions[change.CurrentRevision]
	if !ok {
		return fmt.Errorf("cannot find current revision for change %v", change.ID)
	}

	logger := logrus.WithField("gerrit change", change.Number)

	cloneURI, err := makeCloneURI(instance, change.Project)
	if err != nil {
		return fmt.Errorf("failed to create clone uri: %v", err)
	}

	baseSHA, err := c.gc.GetBranchRevision(instance, change.Project, change.Branch)
	if err != nil {
		return fmt.Errorf("failed to get SHA from base branch: %v", err)
	}

	triggeredJobs := []string{}

	switch change.Status {
	case client.Merged:
		postsubmits := c.ca.Config().Postsubmits[cloneURI.String()]
		postsubmits = append(postsubmits, c.ca.Config().Postsubmits[cloneURI.Host+"/"+cloneURI.Path]...)
		for _, spec := range postsubmits {
			kr := kube.Refs{
				Org:      cloneURI.Host,  // Something like android.googlesource.com
				Repo:     change.Project, // Something like platform/build
				BaseRef:  change.Branch,
				BaseSHA:  baseSHA,
				CloneURI: cloneURI.String(), // Something like https://android.googlesource.com/platform/build
				Pulls: []kube.Pull{
					{
						Number: change.Number,
						Author: rev.Commit.Author.Name,
						SHA:    change.CurrentRevision,
						Ref:    rev.Ref,
					},
				},
			}

			labels := make(map[string]string)
			for k, v := range spec.Labels {
				labels[k] = v
			}
			labels[client.GerritRevision] = change.CurrentRevision

			pj := pjutil.NewProwJobWithAnnotation(
				pjutil.PostsubmitSpec(spec, kr),
				labels,
				map[string]string{
					client.GerritID:       change.ID,
					client.GerritInstance: instance,
				},
			)

			logger.WithFields(pjutil.ProwJobFields(&pj)).Infof("Creating a new postsubmit job for change %d.", change.Number)

			if _, err := c.kc.CreateProwJob(pj); err != nil {
				logger.WithError(err).Errorf("fail to create prowjob %v", pj)
			} else {
				triggeredJobs = append(triggeredJobs, spec.Name)
			}
		}
	case client.New:
		presubmits := c.ca.Config().Presubmits[cloneURI.String()]
		presubmits = append(presubmits, c.ca.Config().Presubmits[cloneURI.Host+"/"+cloneURI.Path]...)
		for _, spec := range presubmits {
			kr := kube.Refs{
				Org:      cloneURI.Host,  // Something like android.googlesource.com
				Repo:     change.Project, // Something like platform/build
				BaseRef:  change.Branch,
				BaseSHA:  baseSHA,
				CloneURI: cloneURI.String(), // Something like https://android.googlesource.com/platform/build
				Pulls: []kube.Pull{
					{
						Number: change.Number,
						Author: rev.Commit.Author.Name,
						SHA:    change.CurrentRevision,
						Ref:    rev.Ref,
					},
				},
			}

			// TODO(krzyzacy): Support AlwaysRun and RunIfChanged

			labels := make(map[string]string)
			for k, v := range spec.Labels {
				labels[k] = v
			}
			labels[client.GerritRevision] = change.CurrentRevision

			pj := pjutil.NewProwJobWithAnnotation(
				pjutil.PresubmitSpec(spec, kr),
				labels,
				map[string]string{
					client.GerritID:       change.ID,
					client.GerritInstance: instance,
				},
			)

			logger.WithFields(pjutil.ProwJobFields(&pj)).Infof("Creating a new presubmit job for change %d.", change.Number)

			if _, err := c.kc.CreateProwJob(pj); err != nil {
				logger.WithError(err).Errorf("fail to create prowjob %v", pj)
			} else {
				triggeredJobs = append(triggeredJobs, spec.Name)
			}
		}
	}

	if len(triggeredJobs) > 0 {
		// comment back to gerrit
		message := fmt.Sprintf("Triggered %d prow jobs:", len(triggeredJobs))
		for _, job := range triggeredJobs {
			message += fmt.Sprintf("\n  * Name: %s", job)
		}

		if err := c.gc.SetReview(instance, change.ID, change.CurrentRevision, message, nil); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2011 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package oglematchers provides a set of matchers useful in a testing or
// mocking framework. These matchers are inspired by and mostly compatible with
// Goolge Test for C++ and Google JS Test.
//
// This package is used by github.com/jacobsa/ogletest and
// github.com/jacobsa/oglemock, which may be more directly useful if you're not
// writing your own testing package or defining your own matchers.
package oglematchers

// A MatchResult is an integer equal to one of the MATCH_* constants below.
// Matchers use a tri-state logic in order to make the semantics of matchers
// that wrap other matchers make more sense. The constants below represent the
// three values that a matcher may return.
type MatchResult int

const (
	// MATCH_FALSE indicates that the supplied value didn't match. For example,
	// IsNil would return this when presented with any non-nil value, and
	// GreaterThan(17) would return this when presented with 16.
	MATCH_FALSE MatchResult = 0

	// MATCH_TRUE indicates that the supplied value did match. For example, IsNil
	// would return this when presented with nil, and GreaterThan(17) would
	// return this when presented with 19.
	MATCH_TRUE MatchResult = 1

	// MATCH_UNDEFINED indicates that the matcher doesn't process values of the
	// supplied type, or otherwise doesn't know how to handle the value. This is
	// akin to returning MATCH_FALSE, except that wrapper matchers should
	// propagagate undefined values.
	//
	// For example, if GreaterThan(17) returned MATCH_FALSE for the value "taco",
	// then Not(GreaterThan(17)) would return MATCH_TRUE. This is technically
	// correct, but is surprising and may mask failures where the wrong sort of
	// matcher is accidentally used. Instead, GreaterThan(17) can return
	// MATCH_UNDEFINED, which will be propagated by Not().
	MATCH_UNDEFINED MatchResult = -1
)

// A Matcher is some predicate implicitly defining a set of values that it
// matches. For example, GreaterThan(17) matches all numeric values greater
// than 17, and HasSubstr("taco") matches all strings with the substring
// "taco".
type Matcher interface {
	// Matches returns a MatchResult indicating whether the supplied value
	// belongs to the set defined by the matcher.
	//
	// If the result is MATCH_FALSE or MATCH_UNDEFINED, it may additionally
	// return an error describing why the value doesn't match. The error text is
	// a relative clause that is suitable for being placed after the value. For
	// example, a predicate that matches strings with a particular substring may,
	// when presented with a numerical value, return the following error text:
	//
	//     "which is not a string"
	//
	// Then the failure message may look like:
	//
	//     Expected: has substring "taco"
	//     Actual:   17, which is not a string
	//
	Matches(candidate interface{}) (MatchResult, error)

	// Description returns a string describing the property that values matching
	// this matcher have, as a verb phrase where the subject is the value. For
	// example, "is greather than 17" or "has substring "taco"".
	Description() string
}
<commit_msg>Updated Matcher interface and added FatalError, for #20.<commit_after>// Copyright 2011 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package oglematchers provides a set of matchers useful in a testing or
// mocking framework. These matchers are inspired by and mostly compatible with
// Goolge Test for C++ and Google JS Test.
//
// This package is used by github.com/jacobsa/ogletest and
// github.com/jacobsa/oglemock, which may be more directly useful if you're not
// writing your own testing package or defining your own matchers.
package oglematchers

// A Matcher is some predicate implicitly defining a set of values that it
// matches. For example, GreaterThan(17) matches all numeric values greater
// than 17, and HasSubstr("taco") matches all strings with the substring
// "taco".
type Matcher interface {
	// Matches returns a bool indicating whether the supplied value belongs to
	// the set defined by the matcher.
	//
	// If the result is false, it returns an error describing why the value
	// doesn't match. The error text is a relative clause that is suitable for
	// being placed after the value. For example, a predicate that matches
	// strings with a particular substring may, when presented with a numerical
	// value, return the following error text:
	//
	//     "which is not a string"
	//
	// Then the failure message may look like:
	//
	//     Expected: has substring "taco"
	//     Actual:   17, which is not a string
	//
	// If the error is self-apparent based on the description of the matcher, the
	// error text may be empty. For example:
	//
	//     Expected: 17
	//     Actual:   19
	//
	// If you are implementing a new matcher, see also the documentation on
	// FatalError.
	Matches(candidate interface{}) (bool, error)

	// Description returns a string describing the property that values matching
	// this matcher have, as a verb phrase where the subject is the value. For
	// example, "is greather than 17" or "has substring "taco"".
	Description() string
}

// FatalError is an implementation of the error interface that may be returned
// from matchers, indicating the error should be propagated. Returning a
// *FatalError indicates that the matcher doesn't process values of the
// supplied type, or otherwise doesn't know how to handle the value.
//
// For example, if GreaterThan(17) returned false for the value "taco" without
// a fatal error, then Not(GreaterThan(17)) would return true. This is
// technically correct, but is surprising and may mask failures where the wrong
// sort of matcher is accidentally used. Instead, GreaterThan(17) can return a
// fatal error, which will be propagated by Not().
type FatalError struct {
	errorText
}

// NewFatalError creates a FatalError struct with the supplied error text.
func NewFatalError(s string) *FatalError {
	return &FatalError{s}
}

func (e *FatalError) Error() string {
	return e.errorText
}
<|endoftext|>