<commit_before>// +build ignore

/*
Copyright 2013 Brandon Philips

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// This program builds a project and is a copy of path.go. See
// github.com/philips/path.go
//
// $ go run path.go
//
// See the README file for more details.
package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
)

const (
	DefaultThirdParty = "third_party"
)

// thirdPartyDir creates a string path to the third_party directory based on
// the current working directory.
func thirdPartyDir() string {
	root, err := os.Getwd()
	if err != nil {
		log.Fatalf("Failed to get the current working directory: %v", err)
	}
	return path.Join(root, DefaultThirdParty)
}

func srcDir() string {
	return path.Join(thirdPartyDir(), "src")
}

// binDir creates a string path to the GOBIN directory based on the current
// working directory.
func binDir() string {
	root, err := os.Getwd()
	if err != nil {
		log.Fatalf("Failed to get the current working directory: %v", err)
	}
	return path.Join(root, "bin")
}

// runEnv execs a command like a shell script piping everything to the parent's
// stderr/stdout and uses the given environment.
func runEnv(env []string, name string, arg ...string) {
	cmd := exec.Command(name, arg...)

	cmd.Env = env

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	err = cmd.Start()
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)
	cmd.Wait()
}

// run calls runEnv with the GOPATH third_party packages.
func run(name string, arg ...string) {
	env := append(os.Environ(),
		"GOPATH="+thirdPartyDir(),
		"GOBIN="+binDir(),
	)

	runEnv(env, name, arg...)
}

// setupProject does the initial setup of the third_party src directory
// including setting up the symlink to the cwd from the src directory.
func setupProject(pkg string) {
	root, err := os.Getwd()
	if err != nil {
		log.Fatalf("Failed to get the current working directory: %v", err)
	}

	src := path.Join(thirdPartyDir(), "src", pkg)
	srcdir := path.Dir(src)

	os.MkdirAll(srcdir, 0755)

	rel, err := filepath.Rel(srcdir, root)
	if err != nil {
		log.Fatalf("creating relative third party path: %v", err)
	}

	err = os.Symlink(rel, src)
	if err != nil && os.IsExist(err) == false {
		log.Fatalf("creating project third party symlink: %v", err)
	}
}

func getVc(root string) versionControl {
	for _, v := range []string{".git", ".hg"} {
		r := path.Join(root, v)
		info, err := os.Stat(r)

		if err != nil || !info.IsDir() {
			continue
		}

		base := path.Base(r)
		switch base {
		case ".git":
			return vcGit(r)
		case ".hg":
			return vcHg(r)
		}
	}
	return new(vcNoop)
}

type versionControl interface {
	commit() string
	update(string) error
}

//  Performs noops on all VC operations.
type vcNoop struct{}

func (v *vcNoop) commit() string {
	return ""
}

func (v *vcNoop) update(dir string) error {
	return nil
}

type vcHg string

// vcHg.commit returns the current HEAD commit hash for a given hg dir.
func (v vcHg) commit() string {
	out, err := exec.Command("hg", "id", "-i", "-R", string(v)).Output()
	if err != nil {
		return ""
	}
	return string(out)
}

// vcHg.udpate updates the given hg dir to ref.
func (v vcHg) update(ref string) error {
	_, err := exec.Command("hg",
		"update",
		"-r", ref,
		"-R", string(v),
		"--cwd", path.Dir(string(v)),
	).Output()
	if err != nil {
		return err
	}
	return nil
}

type vcGit string

// vcGit.commit returns the current HEAD commit hash for a given git dir.
func (v vcGit) commit() string {
	out, err := exec.Command("git", "--git-dir="+string(v), "rev-parse", "HEAD").Output()
	if err != nil {
		return ""
	}
	return string(out)
}

// vcHg.udpate updates the given git dir to ref.
func (v vcGit) update(ref string) error {
	_, err := exec.Command("git",
		"--work-tree="+path.Dir(string(v)),
		"--git-dir="+string(v),
		"reset", "--hard", ref,
	).Output()
	if err != nil {
		return err
	}
	return nil
}

// commit grabs the commit id from hg or git as a string.
func commit(dir string) string {
	return getVc(dir).commit()
}

// removeVcs removes a .git or .hg directory from the given root if it exists.
func removeVcs(root string) (bool, string) {
	for _, v := range []string{".git", ".hg"} {
		r := path.Join(root, v)
		info, err := os.Stat(r)

		if err != nil {
			continue
		}

		// We didn't find it, next!
		if info.IsDir() == false {
			continue
		}

		// We found it, grab the commit and remove the directory
		c := commit(root)
		err = os.RemoveAll(r)
		if err != nil {
			log.Fatalf("removeVcs: %v", err)
		}
		return true, c
	}

	return false, ""
}

// bump takes care of grabbing a package, getting the package git hash and
// removing all of the version control stuff.
func bump(pkg, version string) {
	tpd := thirdPartyDir()

	temp, err := ioutil.TempDir(tpd, "bump")
	if err != nil {
		log.Fatalf("bump: %v", err)
	}
	defer os.RemoveAll(temp)

	env := append(os.Environ(),
		"GOPATH="+temp,
	)

	runEnv(env, "go", "get", "-u", "-d", pkg)

	for {
		root := path.Join(temp, "src", pkg) // the temp installation root
		home := path.Join(tpd, "src", pkg)  // where the package will end up

		if version != "" {
			err := getVc(root).update(version)
			if err != nil {
				log.Fatalf("bump: %v", err)
			}
		}

		ok, c := removeVcs(root)
		if ok {
			// Create the path leading up to the package
			err := os.MkdirAll(path.Dir(home), 0755)
			if err != nil {
				log.Fatalf("bump: %v", err)
			}

			// Remove anything that might have been there
			err = os.RemoveAll(home)
			if err != nil {
				log.Fatalf("bump: %v", err)
			}

			// Finally move the package
			err = os.Rename(root, home)
			if err != nil {
				log.Fatalf("bump: %v", err)
			}

			fmt.Printf("%s %s\n", pkg, strings.TrimSpace(c))
			break
		}

		// Pop off and try to find this directory!
		pkg = path.Dir(pkg)
		if pkg == "." {
			return
		}
	}
}

// validPkg uses go list to decide if the given path is a valid go package.
// This is used by the bumpAll walk to bump all of the existing packages.
func validPkg(pkg string) bool {
	env := append(os.Environ(),
		"GOPATH="+thirdPartyDir(),
	)
	cmd := exec.Command("go", "list", pkg)
	cmd.Env = env

	out, err := cmd.Output()
	if err != nil {
		return false
	}

	if pkg == strings.TrimSpace(string(out)) {
		return true
	}

	return false
}

// bumpWalk walks the third_party directory and bumps all of the packages that it finds.
func bumpWalk(path string, info os.FileInfo, err error) error {
	if err != nil {
		return nil
	}

	// go packages are always directories
	if info.IsDir() == false {
		return nil
	}

	parts := strings.Split(path, srcDir()+"/")
	if len(parts) == 1 {
		return nil
	}

	pkg := parts[1]

	if validPkg(pkg) == false {
		return nil
	}

	bump(pkg, "")

	return nil
}

func bumpAll() {
	err := filepath.Walk(srcDir(), bumpWalk)
	if err != nil {
		log.Fatalf(err.Error())
	}
}

func main() {
	log.SetFlags(0)

	if len(os.Args) <= 1 {
		log.Fatalf("No command")
	}

	cmd := os.Args[1]

	if cmd == "setup" && len(os.Args) > 2 {
		setupProject(os.Args[2])
		return
	}

	if cmd == "bump" && len(os.Args) > 2 {
		ref := ""
		if len(os.Args) > 3 {
			ref = os.Args[3]
		}

		bump(os.Args[2], ref)
		return
	}

	if cmd == "bump-all" && len(os.Args) > 1 {
		bumpAll()
		return
	}

	run("go", os.Args[1:]...)
}
<commit_msg>chore(third_party.go): s/path.go/third_party.go/g<commit_after>// +build ignore

/*
Copyright 2013 Brandon Philips

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// This program builds a project and is a copy of third_party.go. See
// github.com/philips/third_party.go
//
// $ go run third_party.go
//
// See the README file for more details.
package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
)

const (
	DefaultThirdParty = "third_party"
)

// thirdPartyDir creates a string path to the third_party directory based on
// the current working directory.
func thirdPartyDir() string {
	root, err := os.Getwd()
	if err != nil {
		log.Fatalf("Failed to get the current working directory: %v", err)
	}
	return path.Join(root, DefaultThirdParty)
}

func srcDir() string {
	return path.Join(thirdPartyDir(), "src")
}

// binDir creates a string path to the GOBIN directory based on the current
// working directory.
func binDir() string {
	root, err := os.Getwd()
	if err != nil {
		log.Fatalf("Failed to get the current working directory: %v", err)
	}
	return path.Join(root, "bin")
}

// runEnv execs a command like a shell script piping everything to the parent's
// stderr/stdout and uses the given environment.
func runEnv(env []string, name string, arg ...string) {
	cmd := exec.Command(name, arg...)

	cmd.Env = env

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	err = cmd.Start()
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)
	cmd.Wait()
}

// run calls runEnv with the GOPATH third_party packages.
func run(name string, arg ...string) {
	env := append(os.Environ(),
		"GOPATH="+thirdPartyDir(),
		"GOBIN="+binDir(),
	)

	runEnv(env, name, arg...)
}

// setupProject does the initial setup of the third_party src directory
// including setting up the symlink to the cwd from the src directory.
func setupProject(pkg string) {
	root, err := os.Getwd()
	if err != nil {
		log.Fatalf("Failed to get the current working directory: %v", err)
	}

	src := path.Join(thirdPartyDir(), "src", pkg)
	srcdir := path.Dir(src)

	os.MkdirAll(srcdir, 0755)

	rel, err := filepath.Rel(srcdir, root)
	if err != nil {
		log.Fatalf("creating relative third party path: %v", err)
	}

	err = os.Symlink(rel, src)
	if err != nil && os.IsExist(err) == false {
		log.Fatalf("creating project third party symlink: %v", err)
	}
}

func getVc(root string) versionControl {
	for _, v := range []string{".git", ".hg"} {
		r := path.Join(root, v)
		info, err := os.Stat(r)

		if err != nil || !info.IsDir() {
			continue
		}

		base := path.Base(r)
		switch base {
		case ".git":
			return vcGit(r)
		case ".hg":
			return vcHg(r)
		}
	}
	return new(vcNoop)
}

type versionControl interface {
	commit() string
	update(string) error
}

//  Performs noops on all VC operations.
type vcNoop struct{}

func (v *vcNoop) commit() string {
	return ""
}

func (v *vcNoop) update(dir string) error {
	return nil
}

type vcHg string

// vcHg.commit returns the current HEAD commit hash for a given hg dir.
func (v vcHg) commit() string {
	out, err := exec.Command("hg", "id", "-i", "-R", string(v)).Output()
	if err != nil {
		return ""
	}
	return string(out)
}

// vcHg.udpate updates the given hg dir to ref.
func (v vcHg) update(ref string) error {
	_, err := exec.Command("hg",
		"update",
		"-r", ref,
		"-R", string(v),
		"--cwd", path.Dir(string(v)),
	).Output()
	if err != nil {
		return err
	}
	return nil
}

type vcGit string

// vcGit.commit returns the current HEAD commit hash for a given git dir.
func (v vcGit) commit() string {
	out, err := exec.Command("git", "--git-dir="+string(v), "rev-parse", "HEAD").Output()
	if err != nil {
		return ""
	}
	return string(out)
}

// vcHg.udpate updates the given git dir to ref.
func (v vcGit) update(ref string) error {
	_, err := exec.Command("git",
		"--work-tree="+path.Dir(string(v)),
		"--git-dir="+string(v),
		"reset", "--hard", ref,
	).Output()
	if err != nil {
		return err
	}
	return nil
}

// commit grabs the commit id from hg or git as a string.
func commit(dir string) string {
	return getVc(dir).commit()
}

// removeVcs removes a .git or .hg directory from the given root if it exists.
func removeVcs(root string) (bool, string) {
	for _, v := range []string{".git", ".hg"} {
		r := path.Join(root, v)
		info, err := os.Stat(r)

		if err != nil {
			continue
		}

		// We didn't find it, next!
		if info.IsDir() == false {
			continue
		}

		// We found it, grab the commit and remove the directory
		c := commit(root)
		err = os.RemoveAll(r)
		if err != nil {
			log.Fatalf("removeVcs: %v", err)
		}
		return true, c
	}

	return false, ""
}

// bump takes care of grabbing a package, getting the package git hash and
// removing all of the version control stuff.
func bump(pkg, version string) {
	tpd := thirdPartyDir()

	temp, err := ioutil.TempDir(tpd, "bump")
	if err != nil {
		log.Fatalf("bump: %v", err)
	}
	defer os.RemoveAll(temp)

	env := append(os.Environ(),
		"GOPATH="+temp,
	)

	runEnv(env, "go", "get", "-u", "-d", pkg)

	for {
		root := path.Join(temp, "src", pkg) // the temp installation root
		home := path.Join(tpd, "src", pkg)  // where the package will end up

		if version != "" {
			err := getVc(root).update(version)
			if err != nil {
				log.Fatalf("bump: %v", err)
			}
		}

		ok, c := removeVcs(root)
		if ok {
			// Create the path leading up to the package
			err := os.MkdirAll(path.Dir(home), 0755)
			if err != nil {
				log.Fatalf("bump: %v", err)
			}

			// Remove anything that might have been there
			err = os.RemoveAll(home)
			if err != nil {
				log.Fatalf("bump: %v", err)
			}

			// Finally move the package
			err = os.Rename(root, home)
			if err != nil {
				log.Fatalf("bump: %v", err)
			}

			fmt.Printf("%s %s\n", pkg, strings.TrimSpace(c))
			break
		}

		// Pop off and try to find this directory!
		pkg = path.Dir(pkg)
		if pkg == "." {
			return
		}
	}
}

// validPkg uses go list to decide if the given path is a valid go package.
// This is used by the bumpAll walk to bump all of the existing packages.
func validPkg(pkg string) bool {
	env := append(os.Environ(),
		"GOPATH="+thirdPartyDir(),
	)
	cmd := exec.Command("go", "list", pkg)
	cmd.Env = env

	out, err := cmd.Output()
	if err != nil {
		return false
	}

	if pkg == strings.TrimSpace(string(out)) {
		return true
	}

	return false
}

// bumpWalk walks the third_party directory and bumps all of the packages that it finds.
func bumpWalk(path string, info os.FileInfo, err error) error {
	if err != nil {
		return nil
	}

	// go packages are always directories
	if info.IsDir() == false {
		return nil
	}

	parts := strings.Split(path, srcDir()+"/")
	if len(parts) == 1 {
		return nil
	}

	pkg := parts[1]

	if validPkg(pkg) == false {
		return nil
	}

	bump(pkg, "")

	return nil
}

func bumpAll() {
	err := filepath.Walk(srcDir(), bumpWalk)
	if err != nil {
		log.Fatalf(err.Error())
	}
}

func main() {
	log.SetFlags(0)

	if len(os.Args) <= 1 {
		log.Fatalf("No command")
	}

	cmd := os.Args[1]

	if cmd == "setup" && len(os.Args) > 2 {
		setupProject(os.Args[2])
		return
	}

	if cmd == "bump" && len(os.Args) > 2 {
		ref := ""
		if len(os.Args) > 3 {
			ref = os.Args[3]
		}

		bump(os.Args[2], ref)
		return
	}

	if cmd == "bump-all" && len(os.Args) > 1 {
		bumpAll()
		return
	}

	run("go", os.Args[1:]...)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
)

// On performance limited OS X hosts (ex: VMs) the iPhone/iOS Simulator might time out
//  while booting. So far it seems that a simple retry solves these issues.

// This boot timeout can happen when running Unit Tests
//  with Xcode Command Line `xcodebuild`.
const timeOutMessageIPhoneSimulator = "iPhoneSimulator: Timed out waiting"

// This boot timeout can happen when running Xcode (7+) UI tests
//  with Xcode Command Line `xcodebuild`.
const timeOutMessageUITest = "Terminating app due to uncaught exception '_XCTestCaseInterruptionException'"

func exportEnvironmentWithEnvman(keyStr, valueStr string) error {
	log.Printf("Exporting: %s", keyStr)
	envman := exec.Command("envman", "add", "--key", keyStr)
	envman.Stdin = strings.NewReader(valueStr)
	envman.Stdout = os.Stdout
	envman.Stderr = os.Stderr
	return envman.Run()
}

func printConfig(projectPath, scheme, simulatorDevice, simulatorOsVersion, action, deviceDestination string, cleanBuild bool) {
	log.Println()
	log.Println("========== Configs ==========")
	log.Printf(" * project_path: %s", projectPath)
	log.Printf(" * scheme: %s", scheme)
	log.Printf(" * simulator_device: %s", simulatorDevice)
	log.Printf(" * simulator_os_version: %s", simulatorOsVersion)
	log.Printf(" * is_clean_build: %v", cleanBuild)
	log.Printf(" * project_action: %s", action)
	log.Printf(" * device_destination: %s", deviceDestination)

	cmd := exec.Command("xcodebuild", "-version")
	xcodebuildVersion, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf(" [!] Failed to get the version of xcodebuild! Error: %s", err)
	}
	fmt.Println()
	log.Println(" * xcodebuildVersion:")
	fmt.Printf("%s\n", xcodebuildVersion)
	fmt.Println()
}

func validateRequiredInput(key string) (string, error) {
	value := os.Getenv(key)
	if value == "" {
		return "", fmt.Errorf("[!] Missing required input: %s", key)
	}
	return value, nil
}

func isStringFoundInOutput(searchStr, outputToSearchIn string) bool {
	r, err := regexp.Compile("(?i)" + searchStr)
	if err != nil {
		log.Printf(" [!] Failed to compile regexp: %s", err)
		return false
	}
	return r.MatchString(outputToSearchIn)
}

func findFirstDelimiter(searchIn string, searchForDelimiters []string) (foundIdx int, foundDelim string) {
	foundIdx = -1
	for _, aDelim := range searchForDelimiters {
		aDelimFoundIdx := strings.Index(searchIn, aDelim)
		if aDelimFoundIdx >= 0 {
			if foundIdx == -1 || aDelimFoundIdx < foundIdx {
				foundIdx = aDelimFoundIdx
				foundDelim = aDelim
			}
		}
	}
	return foundIdx, foundDelim
}

func findTestSummaryInOutput(fullOutput string, isRunSucess bool) string {
	// using a list of possible delimiters, because the actual order
	//  of these delimiters varies in Xcode CLT's output,
	//  so we'll try to find the first occurance of any of the listed
	//  delimiters
	possibleDelimiters := []string{}
	if !isRunSucess {
		// Failed
		possibleDelimiters = []string{"Testing failed:", "Failing tests:", "** TEST FAILED **"}
	} else {
		// Success
		possibleDelimiters = []string{"Test Suite ", "** TEST SUCCEEDED **"}
	}

	splitIdx, _ := findFirstDelimiter(fullOutput, possibleDelimiters)
	if splitIdx < 0 {
		log.Println(" [!] Could not find any of the required test result delimiters")
		return ""
	}
	return fullOutput[splitIdx:]
}

func printableCommandArgs(fullCommandArgs []string) string {
	cmdArgsDecorated := []string{}
	for idx, anArg := range fullCommandArgs {
		quotedArg := strconv.Quote(anArg)
		if idx == 0 {
			quotedArg = anArg
		}
		cmdArgsDecorated = append(cmdArgsDecorated, quotedArg)
	}

	return strings.Join(cmdArgsDecorated, " ")
}

func runTest(action, projectPath, scheme string, cleanBuild bool, deviceDestination string, isRetryOnTimeout, isFullOutputMode bool) (string, error) {
	args := []string{action, projectPath, "-scheme", scheme}
	if cleanBuild {
		args = append(args, "clean")
	}
	// the 'build' argument is required *before* the 'test' arg, to prevent
	//  the Xcode bug described in the README, which causes:
	// 'iPhoneSimulator: Timed out waiting 120 seconds for simulator to boot, current state is 1.'
	//  in case the compilation takes a long time.
	// Related Radar link: https://openradar.appspot.com/22413115
	// Demonstration project: https://github.com/bitrise-io/simulator-launch-timeout-includes-build-time
	args = append(args, "build", "test", "-destination", deviceDestination)
	// args = append(args, "build", "test", "-destination", deviceDestination, "-sdk", "iphonesimulator")
	cmd := exec.Command("xcodebuild", args...)

	var outBuffer bytes.Buffer
	var outWriter io.Writer
	if isFullOutputMode {
		outWriter = io.MultiWriter(&outBuffer, os.Stdout)
	} else {
		outWriter = &outBuffer
	}

	cmd.Stdin = nil
	cmd.Stdout = outWriter
	cmd.Stderr = outWriter

	fmt.Println()
	log.Println("=> Compiling and running the tests...")
	cmdArgsForPrint := printableCommandArgs(cmd.Args)

	log.Printf("==> Full command: %s", cmdArgsForPrint)
	if !isFullOutputMode {
		fmt.Println()
		log.Println("=> You selected to only see test results.")
		log.Println("   This can take some time, especially if the code have to be compiled first.")
	}

	runErr := cmd.Run()
	fullOutputStr := outBuffer.String()
	if runErr != nil {
		if isStringFoundInOutput(timeOutMessageIPhoneSimulator, fullOutputStr) {
			log.Println("=> Simulator Timeout detected")
			if isRetryOnTimeout {
				log.Println("==> isRetryOnTimeout=true - retrying...")
				return runTest(action, projectPath, scheme, false, deviceDestination, false, isFullOutputMode)
			}
			log.Println(" [!] isRetryOnTimeout=false, no more retry, stopping the test!")
			return fullOutputStr, runErr
		}

		if isStringFoundInOutput(timeOutMessageUITest, fullOutputStr) {
			log.Println("=> Simulator Timeout detected: isUITestTimeoutFound")
			if isRetryOnTimeout {
				log.Println("==> isRetryOnTimeout=true - retrying...")
				return runTest(action, projectPath, scheme, false, deviceDestination, false, isFullOutputMode)
			}
			log.Println(" [!] isRetryOnTimeout=false, no more retry, stopping the test!")
			return fullOutputStr, runErr
		}

		return fullOutputStr, runErr
	}

	return fullOutputStr, nil
}

func main() {
	//
	// Required parameters
	projectPath, err := validateRequiredInput("project_path")
	if err != nil {
		log.Fatalf("Input validation failed, err: %s", err)
	}

	scheme, err := validateRequiredInput("scheme")
	if err != nil {
		log.Fatalf("Input validation failed, err: %s", err)
	}

	simulatorDevice, err := validateRequiredInput("simulator_device")
	if err != nil {
		log.Fatalf("Input validation failed, err: %s", err)
	}

	simulatorOsVersion, err := validateRequiredInput("simulator_os_version")
	if err != nil {
		log.Fatalf("Input validation failed, err: %s", err)
	}

	//
	// Not required parameters
	cleanBuild := false
	if os.Getenv("is_clean_build") == "yes" {
		cleanBuild = true
	}
	isFullOutputMode := !(os.Getenv("is_full_output") == "no")

	//
	// Project-or-Workspace flag
	action := ""
	if strings.HasSuffix(projectPath, ".xcodeproj") {
		action = "-project"
	} else if strings.HasSuffix(projectPath, ".xcworkspace") {
		action = "-workspace"
	} else {
		log.Fatalf("Failed to get valid project file (invalid project file): %s", projectPath)
	}

	//
	// Device Destination
	// xcodebuild -project ./BitriseSampleWithYML.xcodeproj -scheme BitriseSampleWithYML  test -destination "platform=iOS Simulator,name=iPhone 6 Plus,OS=latest" -sdk iphonesimulator -verbose
	deviceDestination := fmt.Sprintf("platform=iOS Simulator,name=%s,OS=%s", simulatorDevice, simulatorOsVersion)

	//
	// Print configs
	printConfig(projectPath, scheme, simulatorDevice, simulatorOsVersion, action, deviceDestination, cleanBuild)

	//
	// Run
	fullOutputStr, runErr := runTest(action, projectPath, scheme, cleanBuild, deviceDestination, true, isFullOutputMode)

	//
	isRunSuccess := (runErr == nil)
	if isRunSuccess {
		exportEnvironmentWithEnvman("BITRISE_XCODE_TEST_RESULT", "succeeded")
	} else {
		exportEnvironmentWithEnvman("BITRISE_XCODE_TEST_RESULT", "failed")
	}
	testResultsSummary := findTestSummaryInOutput(fullOutputStr, isRunSuccess)
	if testResultsSummary == "" {
		testResultsSummary = fmt.Sprintf(" [!] No test summary found in the output - most likely it was a compilation error.\n\n Full output was: %s", fullOutputStr)
	}
	exportEnvironmentWithEnvman("BITRISE_XCODE_TEST_FULL_RESULTS_TEXT", testResultsSummary)

	if !isFullOutputMode {
		fmt.Println()
		fmt.Println("========= TEST RESULTS: =========")
		fmt.Println(testResultsSummary)
		fmt.Println()
	}

	if runErr != nil {
		log.Fatalf("xcode test failed with error: %s", runErr)
	}
}
<commit_msg>major version<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
)

// On performance limited OS X hosts (ex: VMs) the iPhone/iOS Simulator might time out
//  while booting. So far it seems that a simple retry solves these issues.

// This boot timeout can happen when running Unit Tests
//  with Xcode Command Line `xcodebuild`.
const timeOutMessageIPhoneSimulator = "iPhoneSimulator: Timed out waiting"

// This boot timeout can happen when running Xcode (7+) UI tests
//  with Xcode Command Line `xcodebuild`.
const timeOutMessageUITest = "Terminating app due to uncaught exception '_XCTestCaseInterruptionException'"

func exportEnvironmentWithEnvman(keyStr, valueStr string) error {
	log.Printf("Exporting: %s", keyStr)
	envman := exec.Command("envman", "add", "--key", keyStr)
	envman.Stdin = strings.NewReader(valueStr)
	envman.Stdout = os.Stdout
	envman.Stderr = os.Stderr
	return envman.Run()
}

func printConfig(projectPath, scheme, simulatorDevice, simulatorOsVersion, action, deviceDestination string, cleanBuild bool) {
	log.Println()
	log.Println("========== Configs ==========")
	log.Printf(" * project_path: %s", projectPath)
	log.Printf(" * scheme: %s", scheme)
	log.Printf(" * simulator_device: %s", simulatorDevice)
	log.Printf(" * simulator_os_version: %s", simulatorOsVersion)
	log.Printf(" * is_clean_build: %v", cleanBuild)
	log.Printf(" * project_action: %s", action)
	log.Printf(" * device_destination: %s", deviceDestination)

	cmd := exec.Command("xcodebuild", "-version")
	xcodebuildVersion, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf(" [!] Failed to get the version of xcodebuild! Error: %s", err)
	}
	fmt.Println()
	log.Println(" * xcodebuildVersion:")
	fmt.Printf("%s\n", xcodebuildVersion)
	fmt.Println()
}

func validateRequiredInput(key string) (string, error) {
	value := os.Getenv(key)
	if value == "" {
		return "", fmt.Errorf("[!] Missing required input: %s", key)
	}
	return value, nil
}

func isStringFoundInOutput(searchStr, outputToSearchIn string) bool {
	r, err := regexp.Compile("(?i)" + searchStr)
	if err != nil {
		log.Printf(" [!] Failed to compile regexp: %s", err)
		return false
	}
	return r.MatchString(outputToSearchIn)
}

func findFirstDelimiter(searchIn string, searchForDelimiters []string) (foundIdx int, foundDelim string) {
	foundIdx = -1
	for _, aDelim := range searchForDelimiters {
		aDelimFoundIdx := strings.Index(searchIn, aDelim)
		if aDelimFoundIdx >= 0 {
			if foundIdx == -1 || aDelimFoundIdx < foundIdx {
				foundIdx = aDelimFoundIdx
				foundDelim = aDelim
			}
		}
	}
	return foundIdx, foundDelim
}

func findTestSummaryInOutput(fullOutput string, isRunSucess bool) string {
	// using a list of possible delimiters, because the actual order
	//  of these delimiters varies in Xcode CLT's output,
	//  so we'll try to find the first occurance of any of the listed
	//  delimiters
	possibleDelimiters := []string{}
	if !isRunSucess {
		// Failed
		possibleDelimiters = []string{"Testing failed:", "Failing tests:", "** TEST FAILED **"}
	} else {
		// Success
		possibleDelimiters = []string{"Test Suite ", "** TEST SUCCEEDED **"}
	}

	splitIdx, _ := findFirstDelimiter(fullOutput, possibleDelimiters)
	if splitIdx < 0 {
		log.Println(" [!] Could not find any of the required test result delimiters")
		return ""
	}
	return fullOutput[splitIdx:]
}

func printableCommandArgs(fullCommandArgs []string) string {
	cmdArgsDecorated := []string{}
	for idx, anArg := range fullCommandArgs {
		quotedArg := strconv.Quote(anArg)
		if idx == 0 {
			quotedArg = anArg
		}
		cmdArgsDecorated = append(cmdArgsDecorated, quotedArg)
	}

	return strings.Join(cmdArgsDecorated, " ")
}

func majorXcodeVersion() (int, error) {
	outBuffer := bytes.Buffer{}
	errBuffer := bytes.Buffer{}

	cmd := exec.Command("xcodebuild", "-version")
	cmd.Stdout = io.Writer(&outBuffer)
	cmd.Stderr = io.Writer(&errBuffer)

	if err := cmd.Run(); err != nil {
		return -1, fmt.Errorf("xcodebuild -version failed, err: %s", err)
	}

	outStr := outBuffer.String()
	idx := strings.Index(outStr, "Xcode ")
	if idx < 0 {
		return -1, fmt.Errorf("No prefix (Xcode ) found in xcodebuild -version output (%s)", outBuffer.String())
	}
	if idx+len("Xcode ")+1 >= len(outStr) {
		return -1, fmt.Errorf("No version number found in xcodebuild -version output (%s)", outBuffer.String())
	}
	majorVersionStr := outStr[idx+len("Xcode ") : idx+len("Xcode ")+1]
	return strconv.Atoi(majorVersionStr)
}

func runTest(action, projectPath, scheme string, cleanBuild bool, deviceDestination string, isRetryOnTimeout, isFullOutputMode bool) (string, error) {
	args := []string{action, projectPath, "-scheme", scheme}
	if cleanBuild {
		args = append(args, "clean")
	}
	// the 'build' argument is required *before* the 'test' arg, to prevent
	//  the Xcode bug described in the README, which causes:
	// 'iPhoneSimulator: Timed out waiting 120 seconds for simulator to boot, current state is 1.'
	//  in case the compilation takes a long time.
	// Related Radar link: https://openradar.appspot.com/22413115
	// Demonstration project: https://github.com/bitrise-io/simulator-launch-timeout-includes-build-time
	args = append(args, "build", "test", "-destination", deviceDestination)

	majorXcodeVersion, err := majorXcodeVersion()
	if err != nil {
		return "", fmt.Errorf("Faild to get major xcode version, err: %s", err)
	}
	if majorXcodeVersion < 7 {
		args = append(args, "-sdk", "iphonesimulator")
	}

	cmd := exec.Command("xcodebuild", args...)

	var outBuffer bytes.Buffer
	var outWriter io.Writer
	if isFullOutputMode {
		outWriter = io.MultiWriter(&outBuffer, os.Stdout)
	} else {
		outWriter = &outBuffer
	}

	cmd.Stdin = nil
	cmd.Stdout = outWriter
	cmd.Stderr = outWriter

	fmt.Println()
	log.Println("=> Compiling and running the tests...")
	cmdArgsForPrint := printableCommandArgs(cmd.Args)

	log.Printf("==> Full command: %s", cmdArgsForPrint)
	if !isFullOutputMode {
		fmt.Println()
		log.Println("=> You selected to only see test results.")
		log.Println("   This can take some time, especially if the code have to be compiled first.")
	}

	runErr := cmd.Run()
	fullOutputStr := outBuffer.String()
	if runErr != nil {
		if isStringFoundInOutput(timeOutMessageIPhoneSimulator, fullOutputStr) {
			log.Println("=> Simulator Timeout detected")
			if isRetryOnTimeout {
				log.Println("==> isRetryOnTimeout=true - retrying...")
				return runTest(action, projectPath, scheme, false, deviceDestination, false, isFullOutputMode)
			}
			log.Println(" [!] isRetryOnTimeout=false, no more retry, stopping the test!")
			return fullOutputStr, runErr
		}

		if isStringFoundInOutput(timeOutMessageUITest, fullOutputStr) {
			log.Println("=> Simulator Timeout detected: isUITestTimeoutFound")
			if isRetryOnTimeout {
				log.Println("==> isRetryOnTimeout=true - retrying...")
				return runTest(action, projectPath, scheme, false, deviceDestination, false, isFullOutputMode)
			}
			log.Println(" [!] isRetryOnTimeout=false, no more retry, stopping the test!")
			return fullOutputStr, runErr
		}

		return fullOutputStr, runErr
	}

	return fullOutputStr, nil
}

func main() {
	//
	// Required parameters
	projectPath, err := validateRequiredInput("project_path")
	if err != nil {
		log.Fatalf("Input validation failed, err: %s", err)
	}

	scheme, err := validateRequiredInput("scheme")
	if err != nil {
		log.Fatalf("Input validation failed, err: %s", err)
	}

	simulatorDevice, err := validateRequiredInput("simulator_device")
	if err != nil {
		log.Fatalf("Input validation failed, err: %s", err)
	}

	simulatorOsVersion, err := validateRequiredInput("simulator_os_version")
	if err != nil {
		log.Fatalf("Input validation failed, err: %s", err)
	}

	//
	// Not required parameters
	cleanBuild := false
	if os.Getenv("is_clean_build") == "yes" {
		cleanBuild = true
	}
	isFullOutputMode := !(os.Getenv("is_full_output") == "no")

	//
	// Project-or-Workspace flag
	action := ""
	if strings.HasSuffix(projectPath, ".xcodeproj") {
		action = "-project"
	} else if strings.HasSuffix(projectPath, ".xcworkspace") {
		action = "-workspace"
	} else {
		log.Fatalf("Failed to get valid project file (invalid project file): %s", projectPath)
	}

	//
	// Device Destination
	// xcodebuild -project ./BitriseSampleWithYML.xcodeproj -scheme BitriseSampleWithYML  test -destination "platform=iOS Simulator,name=iPhone 6 Plus,OS=latest" -sdk iphonesimulator -verbose
	deviceDestination := fmt.Sprintf("platform=iOS Simulator,name=%s,OS=%s", simulatorDevice, simulatorOsVersion)

	//
	// Print configs
	printConfig(projectPath, scheme, simulatorDevice, simulatorOsVersion, action, deviceDestination, cleanBuild)

	//
	// Run
	fullOutputStr, runErr := runTest(action, projectPath, scheme, cleanBuild, deviceDestination, true, isFullOutputMode)

	//
	isRunSuccess := (runErr == nil)
	if isRunSuccess {
		exportEnvironmentWithEnvman("BITRISE_XCODE_TEST_RESULT", "succeeded")
	} else {
		exportEnvironmentWithEnvman("BITRISE_XCODE_TEST_RESULT", "failed")
	}
	testResultsSummary := findTestSummaryInOutput(fullOutputStr, isRunSuccess)
	if testResultsSummary == "" {
		testResultsSummary = fmt.Sprintf(" [!] No test summary found in the output - most likely it was a compilation error.\n\n Full output was: %s", fullOutputStr)
	}
	exportEnvironmentWithEnvman("BITRISE_XCODE_TEST_FULL_RESULTS_TEXT", testResultsSummary)

	if !isFullOutputMode {
		fmt.Println()
		fmt.Println("========= TEST RESULTS: =========")
		fmt.Println(testResultsSummary)
		fmt.Println()
	}

	if runErr != nil {
		log.Fatalf("xcode test failed with error: %s", runErr)
	}
}
<|endoftext|>


<commit_before>package clickhouse

import (
	"bytes"
	"context"
	"database/sql/driver"
	"unicode"

	"github.com/ClickHouse/clickhouse-go/lib/data"
)

type stmt struct {
	ch       *clickhouse
	query    string
	counter  int
	numInput int
	isInsert bool
}

var emptyResult = &result{}

func (stmt *stmt) NumInput() int {
	switch {
	case stmt.ch.block != nil:
		return len(stmt.ch.block.Columns)
	case stmt.numInput < 0:
		return 0
	}
	return stmt.numInput
}

func (stmt *stmt) Exec(args []driver.Value) (driver.Result, error) {
	return stmt.execContext(context.Background(), args)
}

func (stmt *stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) {
	dargs := make([]driver.Value, len(args))
	for i, nv := range args {
		dargs[i] = nv.Value
	}
	return stmt.execContext(ctx, dargs)
}

func (stmt *stmt) execContext(ctx context.Context, args []driver.Value) (driver.Result, error) {
	finish := stmt.ch.watchCancel(ctx)
	defer finish()
	if stmt.isInsert {
		stmt.counter++
		if err := stmt.ch.block.AppendRow(args); err != nil {
			return nil, err
		}
		if (stmt.counter % stmt.ch.blockSize) == 0 {
			stmt.ch.logf("[exec] flush block")
			if err := stmt.ch.writeBlock(stmt.ch.block); err != nil {
				return nil, err
			}
			if err := stmt.ch.encoder.Flush(); err != nil {
				return nil, err
			}
		}
		return emptyResult, nil
	}
	if err := stmt.ch.sendQuery(stmt.bind(convertOldArgs(args))); err != nil {
		return nil, err
	}
	if err := stmt.ch.process(); err != nil {
		return nil, err
	}
	return emptyResult, nil
}

func (stmt *stmt) Query(args []driver.Value) (driver.Rows, error) {
	return stmt.queryContext(context.Background(), convertOldArgs(args))
}

func (stmt *stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) {
	return stmt.queryContext(ctx, args)
}

func (stmt *stmt) queryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) {
	finish := stmt.ch.watchCancel(ctx)
	if err := stmt.ch.sendQuery(stmt.bind(args)); err != nil {
		finish()
		return nil, err
	}
	meta, err := stmt.ch.readMeta()
	if err != nil {
		finish()
		return nil, err
	}
	rows := rows{
		ch:           stmt.ch,
		finish:       finish,
		stream:       make(chan *data.Block, 50),
		columns:      meta.ColumnNames(),
		blockColumns: meta.Columns,
	}
	go rows.receiveData()
	return &rows, nil
}

func (stmt *stmt) Close() error {
	stmt.ch.logf("[stmt] close")
	return nil
}

func (stmt *stmt) bind(args []driver.NamedValue) string {
	var (
		buf       bytes.Buffer
		index     int
		keyword   bool
		inBetween bool
		like      = newMatcher("like")
		limit     = newMatcher("limit")
		between   = newMatcher("between")
		and       = newMatcher("and")
	)
	switch {
	case stmt.NumInput() != 0:
		reader := bytes.NewReader([]byte(stmt.query))
		for {
			if char, _, err := reader.ReadRune(); err == nil {
				switch char {
				case '@':
					if param := paramParser(reader); len(param) != 0 {
						for _, v := range args {
							if len(v.Name) != 0 && v.Name == param {
								buf.WriteString(quote(v.Value))
							}
						}
					}
				case '?':
					if keyword && index < len(args) && len(args[index].Name) == 0 {
						buf.WriteString(quote(args[index].Value))
						index++
					} else {
						buf.WriteRune(char)
					}
				default:
					switch {
					case
						char == '=',
						char == '<',
						char == '>',
						char == '(',
						char == ',',
						char == '+',
						char == '-',
						char == '*',
						char == '/',
						char == '[':
						keyword = true
					default:
						if limit.matchRune(char) || like.matchRune(char) {
							keyword = true
						} else if between.matchRune(char) {
							keyword = true
							inBetween = true
						} else if inBetween && and.matchRune(char) {
							keyword = true
							inBetween = false
						} else {
							keyword = keyword && unicode.IsSpace(char)
						}
					}
					buf.WriteRune(char)
				}
			} else {
				break
			}
		}
	default:
		buf.WriteString(stmt.query)
	}
	return buf.String()
}

func convertOldArgs(args []driver.Value) []driver.NamedValue {
	dargs := make([]driver.NamedValue, len(args))
	for i, v := range args {
		dargs[i] = driver.NamedValue{
			Ordinal: i + 1,
			Value:   v,
		}
	}
	return dargs
}
<commit_msg>Remove watchcancel in stmt.execContext<commit_after>package clickhouse

import (
	"bytes"
	"context"
	"database/sql/driver"
	"unicode"

	"github.com/ClickHouse/clickhouse-go/lib/data"
)

type stmt struct {
	ch       *clickhouse
	query    string
	counter  int
	numInput int
	isInsert bool
}

var emptyResult = &result{}

func (stmt *stmt) NumInput() int {
	switch {
	case stmt.ch.block != nil:
		return len(stmt.ch.block.Columns)
	case stmt.numInput < 0:
		return 0
	}
	return stmt.numInput
}

func (stmt *stmt) Exec(args []driver.Value) (driver.Result, error) {
	return stmt.execContext(context.Background(), args)
}

func (stmt *stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) {
	dargs := make([]driver.Value, len(args))
	for i, nv := range args {
		dargs[i] = nv.Value
	}
	return stmt.execContext(ctx, dargs)
}

func (stmt *stmt) execContext(ctx context.Context, args []driver.Value) (driver.Result, error) {
	if stmt.isInsert {
		stmt.counter++
		if err := stmt.ch.block.AppendRow(args); err != nil {
			return nil, err
		}
		if (stmt.counter % stmt.ch.blockSize) == 0 {
			stmt.ch.logf("[exec] flush block")
			if err := stmt.ch.writeBlock(stmt.ch.block); err != nil {
				return nil, err
			}
			if err := stmt.ch.encoder.Flush(); err != nil {
				return nil, err
			}
		}
		return emptyResult, nil
	}
	if err := stmt.ch.sendQuery(stmt.bind(convertOldArgs(args))); err != nil {
		return nil, err
	}
	if err := stmt.ch.process(); err != nil {
		return nil, err
	}
	return emptyResult, nil
}

func (stmt *stmt) Query(args []driver.Value) (driver.Rows, error) {
	return stmt.queryContext(context.Background(), convertOldArgs(args))
}

func (stmt *stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) {
	return stmt.queryContext(ctx, args)
}

func (stmt *stmt) queryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) {
	finish := stmt.ch.watchCancel(ctx)
	if err := stmt.ch.sendQuery(stmt.bind(args)); err != nil {
		finish()
		return nil, err
	}
	meta, err := stmt.ch.readMeta()
	if err != nil {
		finish()
		return nil, err
	}
	rows := rows{
		ch:           stmt.ch,
		finish:       finish,
		stream:       make(chan *data.Block, 50),
		columns:      meta.ColumnNames(),
		blockColumns: meta.Columns,
	}
	go rows.receiveData()
	return &rows, nil
}

func (stmt *stmt) Close() error {
	stmt.ch.logf("[stmt] close")
	return nil
}

func (stmt *stmt) bind(args []driver.NamedValue) string {
	var (
		buf       bytes.Buffer
		index     int
		keyword   bool
		inBetween bool
		like      = newMatcher("like")
		limit     = newMatcher("limit")
		between   = newMatcher("between")
		and       = newMatcher("and")
	)
	switch {
	case stmt.NumInput() != 0:
		reader := bytes.NewReader([]byte(stmt.query))
		for {
			if char, _, err := reader.ReadRune(); err == nil {
				switch char {
				case '@':
					if param := paramParser(reader); len(param) != 0 {
						for _, v := range args {
							if len(v.Name) != 0 && v.Name == param {
								buf.WriteString(quote(v.Value))
							}
						}
					}
				case '?':
					if keyword && index < len(args) && len(args[index].Name) == 0 {
						buf.WriteString(quote(args[index].Value))
						index++
					} else {
						buf.WriteRune(char)
					}
				default:
					switch {
					case
						char == '=',
						char == '<',
						char == '>',
						char == '(',
						char == ',',
						char == '+',
						char == '-',
						char == '*',
						char == '/',
						char == '[':
						keyword = true
					default:
						if limit.matchRune(char) || like.matchRune(char) {
							keyword = true
						} else if between.matchRune(char) {
							keyword = true
							inBetween = true
						} else if inBetween && and.matchRune(char) {
							keyword = true
							inBetween = false
						} else {
							keyword = keyword && unicode.IsSpace(char)
						}
					}
					buf.WriteRune(char)
				}
			} else {
				break
			}
		}
	default:
		buf.WriteString(stmt.query)
	}
	return buf.String()
}

func convertOldArgs(args []driver.Value) []driver.NamedValue {
	dargs := make([]driver.NamedValue, len(args))
	for i, v := range args {
		dargs[i] = driver.NamedValue{
			Ordinal: i + 1,
			Value:   v,
		}
	}
	return dargs
}
<|endoftext|>


<commit_before>package cpio

import (
	"bytes"
	"io"
	"os"
	"strconv"
	"time"
)

const (
	svr4MaxNameSize = 4096 // MAX_PATH
	svr4MaxFileSize = 4294967295
)

func readHex(s string) int64 {
	// errors are ignored and 0 returned
	i, _ := strconv.ParseInt(s, 16, 64)
	return i
}

func readSVR4Header(r io.Reader) (*Header, error) {
	var buf [110]byte
	if _, err := r.Read(buf[:]); err != nil {
		return nil, err
	}

	// TODO: check endianness

	// check magic
	hasCRC := false
	if !bytes.HasPrefix(buf[:], []byte{0x30, 0x37, 0x30, 0x37, 0x30}) { // 07070
		return nil, ErrHeader
	}
	if buf[5] == 0x32 { // '2'
		hasCRC = true
	} else if buf[5] != 0x31 { // '1'
		return nil, ErrHeader
	}

	// create invariant that all bytes are valid hex values in ascii
	for i := 0; i < len(buf); i++ {
		if buf[i] < 0x30 || buf[i] > 0x46 { // ASCII '0' to 'F'
			return nil, ErrHeader
		}
	}

	asc := string(buf[:])
	hdr := &Header{}

	hdr.Inode = readHex(asc[6:14])
	hdr.Mode = os.FileMode(readHex(asc[14:22]))
	hdr.UID = int(readHex(asc[22:30]))
	hdr.GID = int(readHex(asc[30:38]))
	hdr.Links = int(readHex(asc[38:46]))
	hdr.ModTime = time.Unix(readHex(asc[46:54]), 0)
	hdr.Size = readHex(asc[54:62])
	if hdr.Size > svr4MaxFileSize {
		return nil, ErrHeader
	}
	nameSize := readHex(asc[94:102])
	if nameSize > svr4MaxNameSize {
		return nil, ErrHeader
	}
	hdr.Checksum = Checksum(readHex(asc[102:110]))
	if !hasCRC && hdr.Checksum != 0 {
		return nil, ErrHeader
	}

	name := make([]byte, nameSize)
	if _, err := io.ReadFull(r, name); err != nil {
		return nil, err
	}
	if bytes.Equal(name, headerEOF) {
		return nil, io.EOF
	}
	hdr.Name = string(name[:nameSize-1])

	// skip to end of header - padding to a multiple of 4
	pad := (4 - (len(buf)+len(name))%4) % 4
	if pad > 0 {
		if _, err := io.ReadFull(r, buf[:pad]); err != nil {
			return nil, nil
		}
	}

	return hdr, nil
}
<commit_msg>Removed ascii check which fails on lower case<commit_after>package cpio

import (
	"bytes"
	"io"
	"os"
	"strconv"
	"time"
)

const (
	svr4MaxNameSize = 4096 // MAX_PATH
	svr4MaxFileSize = 4294967295
)

func readHex(s string) int64 {
	// errors are ignored and 0 returned
	i, _ := strconv.ParseInt(s, 16, 64)
	return i
}

func readSVR4Header(r io.Reader) (*Header, error) {
	var buf [110]byte
	if _, err := r.Read(buf[:]); err != nil {
		return nil, err
	}

	// TODO: check endianness

	// check magic
	hasCRC := false
	if !bytes.HasPrefix(buf[:], []byte{0x30, 0x37, 0x30, 0x37, 0x30}) { // 07070
		return nil, ErrHeader
	}
	if buf[5] == 0x32 { // '2'
		hasCRC = true
	} else if buf[5] != 0x31 { // '1'
		return nil, ErrHeader
	}

	asc := string(buf[:])
	hdr := &Header{}

	hdr.Inode = readHex(asc[6:14])
	hdr.Mode = os.FileMode(readHex(asc[14:22]))
	hdr.UID = int(readHex(asc[22:30]))
	hdr.GID = int(readHex(asc[30:38]))
	hdr.Links = int(readHex(asc[38:46]))
	hdr.ModTime = time.Unix(readHex(asc[46:54]), 0)
	hdr.Size = readHex(asc[54:62])
	if hdr.Size > svr4MaxFileSize {
		return nil, ErrHeader
	}
	nameSize := readHex(asc[94:102])
	if nameSize > svr4MaxNameSize {
		return nil, ErrHeader
	}
	hdr.Checksum = Checksum(readHex(asc[102:110]))
	if !hasCRC && hdr.Checksum != 0 {
		return nil, ErrHeader
	}

	name := make([]byte, nameSize)
	if _, err := io.ReadFull(r, name); err != nil {
		return nil, err
	}
	if bytes.Equal(name, headerEOF) {
		return nil, io.EOF
	}
	hdr.Name = string(name[:nameSize-1])

	// skip to end of header - padding to a multiple of 4
	pad := (4 - (len(buf)+len(name))%4) % 4
	if pad > 0 {
		if _, err := io.ReadFull(r, buf[:pad]); err != nil {
			return nil, nil
		}
	}

	return hdr, nil
}
<|endoftext|>


<commit_before><commit_msg>js: Bug fix: Release the callbacks correctly<commit_after><|endoftext|>


<commit_before>package main

import (
	_ "time"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"github.com/gorilla/sessions"
	"io"
	"net/http"
	"database/sql"
	_ "github.com/go-sql-driver/mysql"
	"runtime"
	"errors"
	"html/template"
	"os/exec"
)

var templates = template.Must(template.ParseFiles("index.html"))

const MESSAGE_QUEUE_SIZE = 10

const STATUS_FAILURE = "{\"status\":\"failure\"}"

var authKey = []byte("somesecretauth")
var store sessions.Store

var pool *Pool
var clients map[int64]*Client

var db *sql.DB

type Pool struct {
	in  chan *Client
	out chan *Room
}

type Client struct {
	id      int64
	otherid int64
	in      chan string
	out     chan string
	retChan chan *Room
}

type Room struct {
	id      []byte
	client1 *Client
	client2 *Client
	Question *Question
}

type Question struct {
	Id 	int64			`json:"id"`
	Title string		`json:"title"`
	Body string			`json:"body"`
	Difficulty int 		`json:"diff"`
}

func (p *Pool) Pair() {
	for {
		c1, c2 := <-p.in, <-p.in

		for c1.id == c2.id {
			c2 = <- p.in
		}

		fmt.Println("match found for ", c1.id, " and ", c2.id)

		b := make([]byte, 32)
		n, err := io.ReadFull(rand.Reader, b)
		if err != nil || n != 32 {
			return
		}

		row := db.QueryRow("SELECT * FROM questions ORDER BY RAND()")
		q := new(Question)
		err = row.Scan(&q.Id, &q.Title, &q.Body, &q.Difficulty)
		if err !=  nil {
			fmt.Println(err)
		}

		room := &Room{b, c1, c2, q}

		c1.otherid, c2.otherid = c2.id, c1.id
		c1.in, c2.in = c2.out, c1.out

		c1.retChan <- room
		c2.retChan <- room
	}
}

func newPool() *Pool {
	pool := &Pool{
		in:  make(chan *Client),
		out: make(chan *Room),
	}

	go pool.Pair()

	return pool
}

func UIDFromSession(w http.ResponseWriter, r *http.Request) (int64, error) {
	session, _ := store.Get(r, "session")
	userid := session.Values["userid"]

	if userid == nil {
		return 0, errors.New("no cookie set")
	}
	return userid.(int64), nil
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	db, _ = sql.Open("mysql", "root:pass@/suitup")
	defer db.Close()

	store = sessions.NewCookieStore(authKey)

	pool = newPool()
	clients = make(map[int64]*Client)

	http.HandleFunc("/", mainHandle)

	http.HandleFunc("/login", login)

	http.HandleFunc("/message/check", checkMessage)
	http.HandleFunc("/message/send", sendMessage)

	http.HandleFunc("/question/new", newQuestion)
	http.HandleFunc("/question/submit", testCode)

	http.HandleFunc("/chatroom/join", joinChatRoom)
	http.HandleFunc("/chatroom/leave", leaveChatRoom)

	http.Handle("/assets/", http.StripPrefix("/assets/", http.FileServer(http.Dir("/home/suitup/hackmit/assets/"))))
	http.ListenAndServe(":8080", nil)
}

func mainHandle(w http.ResponseWriter, r *http.Request) {
	templates.ExecuteTemplate(w, "index.html", nil)
}

func joinChatRoom(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

	fmt.Println("join ", uid)

	retChan := make(chan *Room)
	client := &Client{
		id:      uid,
		in:      nil,
		out:     make(chan string, MESSAGE_QUEUE_SIZE),
		retChan: retChan,
	}

	clients[uid] = client
	pool.in <- client

	chatroom := <- retChan

	fmt.Println("joinChatRoom-chatroom.id: ", chatroom.id)

	qjs, err := json.Marshal(chatroom.Question)

	fmt.Println("qjs: ", string(qjs))

	fmt.Fprint(w, "{\"status\":\"success\",\"crid\":\"", asciify(chatroom.id), "\",\"question\":", string(qjs), "}")
}

func asciify(ba []byte) string {
	ret := make([]byte, len(ba))
	for i, b := range ba {
		ret[i] = (b % 26) + 97
	}
	return string(ret)
}

func testCode(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	code := r.FormValue("submission")
	_ = r.FormValue("cvid")
	qid := "1"
	app := "/home/suitup/hackmit/secure.sh"
	out1, err1 := exec.Command("/usr/bin/secure", qid, code).Output()
	if err1 != nil {
    	fmt.Println(err1)
	}
	fmt.Printf("The date is %s\n", out1)
	out, err := exec.Command(app, qid, code).Output()
    if err != nil {
    	fmt.Println("LOL", err)
    	return
    }
    fmt.Println(string(out))
    fmt.Fprint(w, "{\"status\":\"success\", \"data\": \"", string(out), "\"}" )
    return
}

func leaveChatRoom(w http.ResponseWriter, r *http.Request) {
	uid, _ := UIDFromSession(w, r)
	client := clients[uid]

	if client != nil {
		fmt.Println("leaving")
		delete(clients, client.otherid)
		delete(clients, uid)
	}

	fmt.Println("leave ", uid)

	fmt.Fprint(w, "{\"status\":\"success\"}")
}

func sendMessage(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

	fmt.Println("send ", uid)

	r.ParseForm()
	message := r.FormValue("s")

	client := clients[uid]

	if client != nil {
		client.out <- message
		fmt.Fprint(w, "{\"status\":\"success\"}")
	} else {
		fmt.Fprint(w, STATUS_FAILURE)
	}
}

func checkMessage(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

	fmt.Println("check ", uid)

	client := clients[uid]

	if client != nil {
		select {
		case message, ok := <- client.in:
			if ok {
				fmt.Fprint(w, "{\"status\":\"success\",\"s\":\"", message, "\"}")
			} else {
				delete(clients, uid)
				fmt.Fprint(w, "{\"status\":\"failure\"}")
			}
		default:
			fmt.Fprint(w, "{\"status\":\"success\",\"s\":\"\"}")
		}

	} else {
		fmt.Fprint(w, "{\"status\":\"failure\",\"s\":\"\"}")
	}
}

type User struct {
    Id            	int64      	`json:"id"`
    FacebookId  	string		`json:fbid"`
    Username 		string		`json:username"`
    Email 			string		`json:email"`
    Level 			int64		`json:lvl"`
    Score 			int64		`json:score"`
}

func newQuestion(w http.ResponseWriter, r *http.Request) {
	row := db.QueryRow("SELECT * FROM questions ORDER BY RAND()")
	q := new(Question)
	err := row.Scan(&q.Id, &q.Title, &q.Body, &q.Difficulty)

	if err != nil {
		fmt.Println(err)
		fmt.Fprint(w, STATUS_FAILURE)
	}

	b, err := json.Marshal(q)
	if err != nil {
		fmt.Println(err)
		fmt.Fprint(w, STATUS_FAILURE)
	}

	fmt.Fprint(w, string(b))
}

func login(w http.ResponseWriter, r *http.Request) {
	inputToken := r.FormValue("access_token")
	if len(inputToken) != 0 {
		uid := GetMe(inputToken)

		// row := db.QueryRow("SELECT id FROM users")
		row := db.QueryRow("SELECT id FROM users WHERE facebook_id=?", string(uid))
		user := new(User)
		err := row.Scan(&user.Id)

		if err != nil {
			_, err = db.Exec("INSERT INTO users (facebook_id, username, email, level, points) VALUES (?, ?, ?, 0, 0)", uid, "", "")
			if err != nil {
				fmt.Fprint(w, STATUS_FAILURE)
				return
			} else {
				row = db.QueryRow("SELECT id FROM users WHERE facebook_id=?", string(uid))
				err = row.Scan(&user.Id)
				if err != nil {
					fmt.Fprint(w, STATUS_FAILURE)
					return
				}
			}

		}

		session, _ := store.Get(r, "session")
		session.Values["userid"] = user.Id
		session.Save(r, w)

		fmt.Fprint(w, "{\"status\":\"success\"}")
	}
}

func readHttpBody(response *http.Response) string {
	bodyBuffer := make([]byte, 1000)
	var str string

	count, err := response.Body.Read(bodyBuffer)

	for ; count > 0; count, err = response.Body.Read(bodyBuffer) {

		if err != nil {

		}

		str += string(bodyBuffer[:count])
	}

	return str

}

func getUncachedResponse(uri string) (*http.Response, error) {
	request, err := http.NewRequest("GET", uri, nil)

	if err == nil {
		request.Header.Add("Cache-Control", "no-cache")

		client := new(http.Client)

		return client.Do(request)
	}

	if (err != nil) {
	}
	return nil, err

}

func GetMe(token string) string {
	response, err := getUncachedResponse("https://graph.facebook.com/me?access_token="+token)

	if err == nil {

		var jsonBlob interface{}

		responseBody := readHttpBody(response)

		if responseBody != "" {
			err = json.Unmarshal([]byte(responseBody), &jsonBlob)

			if err == nil {
				jsonObj := jsonBlob.(map[string]interface{})
				return jsonObj["id"].(string)
			}
		}
		return err.Error()
	}

	return err.Error()
}

func handleError(err error) {
	if err != nil {
		fmt.Println(err)
	}
}
<commit_msg>adding changes to bash call<commit_after>package main

import (
	_ "time"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"github.com/gorilla/sessions"
	"io"
	"net/http"
	"database/sql"
	_ "github.com/go-sql-driver/mysql"
	"runtime"
	"errors"
	"html/template"
	"os/exec"
)

var templates = template.Must(template.ParseFiles("index.html"))

const MESSAGE_QUEUE_SIZE = 10

const STATUS_FAILURE = "{\"status\":\"failure\"}"

var authKey = []byte("somesecretauth")
var store sessions.Store

var pool *Pool
var clients map[int64]*Client

var db *sql.DB

type Pool struct {
	in  chan *Client
	out chan *Room
}

type Client struct {
	id      int64
	otherid int64
	in      chan string
	out     chan string
	retChan chan *Room
}

type Room struct {
	id      []byte
	client1 *Client
	client2 *Client
	Question *Question
}

type Question struct {
	Id 	int64			`json:"id"`
	Title string		`json:"title"`
	Body string			`json:"body"`
	Difficulty int 		`json:"diff"`
}

func (p *Pool) Pair() {
	for {
		c1, c2 := <-p.in, <-p.in

		for c1.id == c2.id {
			c2 = <- p.in
		}

		fmt.Println("match found for ", c1.id, " and ", c2.id)

		b := make([]byte, 32)
		n, err := io.ReadFull(rand.Reader, b)
		if err != nil || n != 32 {
			return
		}

		row := db.QueryRow("SELECT * FROM questions ORDER BY RAND()")
		q := new(Question)
		err = row.Scan(&q.Id, &q.Title, &q.Body, &q.Difficulty)
		if err !=  nil {
			fmt.Println(err)
		}

		room := &Room{b, c1, c2, q}

		c1.otherid, c2.otherid = c2.id, c1.id
		c1.in, c2.in = c2.out, c1.out

		c1.retChan <- room
		c2.retChan <- room
	}
}

func newPool() *Pool {
	pool := &Pool{
		in:  make(chan *Client),
		out: make(chan *Room),
	}

	go pool.Pair()

	return pool
}

func UIDFromSession(w http.ResponseWriter, r *http.Request) (int64, error) {
	session, _ := store.Get(r, "session")
	userid := session.Values["userid"]

	if userid == nil {
		return 0, errors.New("no cookie set")
	}
	return userid.(int64), nil
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	db, _ = sql.Open("mysql", "root:pass@/suitup")
	defer db.Close()

	store = sessions.NewCookieStore(authKey)

	pool = newPool()
	clients = make(map[int64]*Client)

	http.HandleFunc("/", mainHandle)

	http.HandleFunc("/login", login)

	http.HandleFunc("/message/check", checkMessage)
	http.HandleFunc("/message/send", sendMessage)

	http.HandleFunc("/question/new", newQuestion)
	http.HandleFunc("/question/submit", testCode)

	http.HandleFunc("/chatroom/join", joinChatRoom)
	http.HandleFunc("/chatroom/leave", leaveChatRoom)

	http.Handle("/assets/", http.StripPrefix("/assets/", http.FileServer(http.Dir("/home/suitup/hackmit/assets/"))))
	http.ListenAndServe(":8080", nil)
}

func mainHandle(w http.ResponseWriter, r *http.Request) {
	templates.ExecuteTemplate(w, "index.html", nil)
}

func joinChatRoom(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

	fmt.Println("join ", uid)

	retChan := make(chan *Room)
	client := &Client{
		id:      uid,
		in:      nil,
		out:     make(chan string, MESSAGE_QUEUE_SIZE),
		retChan: retChan,
	}

	clients[uid] = client
	pool.in <- client

	chatroom := <- retChan

	fmt.Println("joinChatRoom-chatroom.id: ", chatroom.id)

	qjs, err := json.Marshal(chatroom.Question)

	fmt.Println("qjs: ", string(qjs))

	fmt.Fprint(w, "{\"status\":\"success\",\"crid\":\"", asciify(chatroom.id), "\",\"question\":", string(qjs), "}")
}

func asciify(ba []byte) string {
	ret := make([]byte, len(ba))
	for i, b := range ba {
		ret[i] = (b % 26) + 97
	}
	return string(ret)
}

func testCode(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	code := r.FormValue("submission")
	_ = r.FormValue("cvid")
	qid := "1"
	app := "/home/suitup/hackmit/secure.sh"
	// out1, err1 := exec.Command("/usr/bin/secure", qid, code).Output()
	// if err1 != nil {
 //    	fmt.Println(err1)
	// }
	// fmt.Printf("The date is %s\n", out1)
	out, err := exec.Command(app, qid, code).Output()
    if err != nil {
    	fmt.Println("LOL", err)
    	return
    }
    fmt.Println(string(out))
    fmt.Fprint(w, "{\"status\":\"success\", \"data\": \"", string(out), "\"}" )
    return
}

func leaveChatRoom(w http.ResponseWriter, r *http.Request) {
	uid, _ := UIDFromSession(w, r)
	client := clients[uid]

	if client != nil {
		fmt.Println("leaving")
		delete(clients, client.otherid)
		delete(clients, uid)
	}

	fmt.Println("leave ", uid)

	fmt.Fprint(w, "{\"status\":\"success\"}")
}

func sendMessage(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

	fmt.Println("send ", uid)

	r.ParseForm()
	message := r.FormValue("s")

	client := clients[uid]

	if client != nil {
		client.out <- message
		fmt.Fprint(w, "{\"status\":\"success\"}")
	} else {
		fmt.Fprint(w, STATUS_FAILURE)
	}
}

func checkMessage(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

	fmt.Println("check ", uid)

	client := clients[uid]

	if client != nil {
		select {
		case message, ok := <- client.in:
			if ok {
				fmt.Fprint(w, "{\"status\":\"success\",\"s\":\"", message, "\"}")
			} else {
				delete(clients, uid)
				fmt.Fprint(w, "{\"status\":\"failure\"}")
			}
		default:
			fmt.Fprint(w, "{\"status\":\"success\",\"s\":\"\"}")
		}

	} else {
		fmt.Fprint(w, "{\"status\":\"failure\",\"s\":\"\"}")
	}
}

type User struct {
    Id            	int64      	`json:"id"`
    FacebookId  	string		`json:fbid"`
    Username 		string		`json:username"`
    Email 			string		`json:email"`
    Level 			int64		`json:lvl"`
    Score 			int64		`json:score"`
}

func newQuestion(w http.ResponseWriter, r *http.Request) {
	row := db.QueryRow("SELECT * FROM questions ORDER BY RAND()")
	q := new(Question)
	err := row.Scan(&q.Id, &q.Title, &q.Body, &q.Difficulty)

	if err != nil {
		fmt.Println(err)
		fmt.Fprint(w, STATUS_FAILURE)
	}

	b, err := json.Marshal(q)
	if err != nil {
		fmt.Println(err)
		fmt.Fprint(w, STATUS_FAILURE)
	}

	fmt.Fprint(w, string(b))
}

func login(w http.ResponseWriter, r *http.Request) {
	inputToken := r.FormValue("access_token")
	if len(inputToken) != 0 {
		uid := GetMe(inputToken)

		// row := db.QueryRow("SELECT id FROM users")
		row := db.QueryRow("SELECT id FROM users WHERE facebook_id=?", string(uid))
		user := new(User)
		err := row.Scan(&user.Id)

		if err != nil {
			_, err = db.Exec("INSERT INTO users (facebook_id, username, email, level, points) VALUES (?, ?, ?, 0, 0)", uid, "", "")
			if err != nil {
				fmt.Fprint(w, STATUS_FAILURE)
				return
			} else {
				row = db.QueryRow("SELECT id FROM users WHERE facebook_id=?", string(uid))
				err = row.Scan(&user.Id)
				if err != nil {
					fmt.Fprint(w, STATUS_FAILURE)
					return
				}
			}

		}

		session, _ := store.Get(r, "session")
		session.Values["userid"] = user.Id
		session.Save(r, w)

		fmt.Fprint(w, "{\"status\":\"success\"}")
	}
}

func readHttpBody(response *http.Response) string {
	bodyBuffer := make([]byte, 1000)
	var str string

	count, err := response.Body.Read(bodyBuffer)

	for ; count > 0; count, err = response.Body.Read(bodyBuffer) {

		if err != nil {

		}

		str += string(bodyBuffer[:count])
	}

	return str

}

func getUncachedResponse(uri string) (*http.Response, error) {
	request, err := http.NewRequest("GET", uri, nil)

	if err == nil {
		request.Header.Add("Cache-Control", "no-cache")

		client := new(http.Client)

		return client.Do(request)
	}

	if (err != nil) {
	}
	return nil, err

}

func GetMe(token string) string {
	response, err := getUncachedResponse("https://graph.facebook.com/me?access_token="+token)

	if err == nil {

		var jsonBlob interface{}

		responseBody := readHttpBody(response)

		if responseBody != "" {
			err = json.Unmarshal([]byte(responseBody), &jsonBlob)

			if err == nil {
				jsonObj := jsonBlob.(map[string]interface{})
				return jsonObj["id"].(string)
			}
		}
		return err.Error()
	}

	return err.Error()
}

func handleError(err error) {
	if err != nil {
		fmt.Println(err)
	}
}
<|endoftext|>


<commit_before><commit_msg>web: add timeout handler<commit_after><|endoftext|>


<commit_before>package index

import (
	"compress/gzip"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"
	"unicode/utf8"

	"github.com/etsy/hound/codesearch/index"
	"github.com/etsy/hound/codesearch/regexp"
)

const (
	matchLimit               = 5000
	manifestFilename         = "metadata.gob"
	excludedFileJsonFilename = "excluded_files.json"
	filePeekSize             = 2048
)

const (
	reasonDotFile     = "Dot files are excluded."
	reasonInvalidMode = "Invalid file mode."
	reasonNotText     = "Not a text file."
)

type Index struct {
	Ref *IndexRef
	idx *index.Index
	lck sync.RWMutex
}

type IndexOptions struct {
	ExcludeDotFiles bool
	SpecialFiles    []string
}

type SearchOptions struct {
	IgnoreCase     bool
	LinesOfContext uint
	FileRegexp     string
	Offset         int
	Limit          int
}

type Match struct {
	Line       string
	LineNumber int
	Before     []string
	After      []string
}

type SearchResponse struct {
	Matches        []*FileMatch
	FilesWithMatch int
	FilesOpened    int           `json:"-"`
	Duration       time.Duration `json:"-"`
	Revision       string
}

type FileMatch struct {
	Filename string
	Matches  []*Match
}

type ExcludedFile struct {
	Filename string
	Reason   string
}

type IndexRef struct {
	Url  string
	Rev  string
	Time time.Time
	dir  string
}

func (r *IndexRef) Dir() string {
	return r.dir
}

func (r *IndexRef) writeManifest() error {
	w, err := os.Create(filepath.Join(r.dir, manifestFilename))
	if err != nil {
		return err
	}
	defer w.Close()

	return gob.NewEncoder(w).Encode(r)
}

func (r *IndexRef) Open() (*Index, error) {
	return &Index{
		Ref: r,
		idx: index.Open(filepath.Join(r.dir, "tri")),
	}, nil
}

func (r *IndexRef) Remove() error {
	return os.RemoveAll(r.dir)
}

func (n *Index) Close() error {
	n.lck.Lock()
	defer n.lck.Unlock()
	return n.idx.Close()
}

func (n *Index) Destroy() error {
	n.lck.Lock()
	defer n.lck.Unlock()
	if err := n.idx.Close(); err != nil {
		return err
	}
	return n.Ref.Remove()
}

func (n *Index) GetDir() string {
	return n.Ref.dir
}

func toStrings(lines [][]byte) []string {
	strs := make([]string, len(lines))
	for i, n := 0, len(lines); i < n; i++ {
		strs[i] = string(lines[i])
	}
	return strs
}

func GetRegexpPattern(pat string, ignoreCase bool) string {
	if ignoreCase {
		return "(?i)(?m)" + pat
	}
	return "(?m)" + pat
}

func (n *Index) Search(pat string, opt *SearchOptions) (*SearchResponse, error) {
	startedAt := time.Now()

	n.lck.RLock()
	defer n.lck.RUnlock()

	re, err := regexp.Compile(GetRegexpPattern(pat, opt.IgnoreCase))
	if err != nil {
		return nil, err
	}

	var (
		g                grepper
		results          []*FileMatch
		filesOpened      int
		filesFound       int
		filesCollected   int
		matchesCollected int
	)

	var fre *regexp.Regexp
	if opt.FileRegexp != "" {
		fre, err = regexp.Compile(opt.FileRegexp)
		if err != nil {
			return nil, err
		}
	}

	files := n.idx.PostingQuery(index.RegexpQuery(re.Syntax))
	for _, file := range files {
		var matches []*Match
		name := n.idx.Name(file)
		hasMatch := false

		// reject files that do not match the file pattern
		if fre != nil && fre.MatchString(name, true, true) < 0 {
			continue
		}

		filesOpened++
		if err := g.grep2File(filepath.Join(n.Ref.dir, "raw", name), re, int(opt.LinesOfContext),
			func(line []byte, lineno int, before [][]byte, after [][]byte) (bool, error) {

				hasMatch = true
				if filesFound < opt.Offset || (opt.Limit > 0 && filesCollected >= opt.Limit) {
					return false, nil
				}

				matchesCollected++
				matches = append(matches, &Match{
					Line:       string(line),
					LineNumber: lineno,
					Before:     toStrings(before),
					After:      toStrings(after),
				})

				if matchesCollected > matchLimit {
					return false, fmt.Errorf("search exceeds limit on matches: %d", matchLimit)
				}

				return true, nil
			}); err != nil {
			return nil, err
		}

		if !hasMatch {
			continue
		}

		filesFound++
		if len(matches) > 0 {
			filesCollected++
			results = append(results, &FileMatch{
				Filename: name,
				Matches:  matches,
			})
		}
	}

	return &SearchResponse{
		Matches:        results,
		FilesWithMatch: filesFound,
		FilesOpened:    filesOpened,
		Duration:       time.Now().Sub(startedAt),
		Revision:       n.Ref.Rev,
	}, nil
}

func isTextFile(filename string) (bool, error) {
	buf := make([]byte, filePeekSize)
	r, err := os.Open(filename)
	if err != nil {
		return false, err
	}
	defer r.Close()

	n, err := io.ReadFull(r, buf)
	if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
		return false, err
	}

	buf = buf[:n]

	if n < filePeekSize {
		// read the whole file, must be valid.
		return utf8.Valid(buf), nil
	}

	// read a prefix, allow trailing partial runes.
	return validUTF8IgnoringPartialTrailingRune(buf), nil

}

// Determines if the buffer contains valid UTF8 encoded string data. The buffer is assumed
// to be a prefix of a larger buffer so if the buffer ends with the start of a rune, it
// is still considered valid.
//
// Basic logic copied from https://golang.org/pkg/unicode/utf8/#Valid
func validUTF8IgnoringPartialTrailingRune(p []byte) bool {
	i := 0
	n := len(p)

	for i < n {
		if p[i] < utf8.RuneSelf {
			i++
		} else {
			_, size := utf8.DecodeRune(p[i:])
			if size == 1 {
				// All valid runes of size 1 (those below RuneSelf) were handled above. This must be a RuneError.
				// If we're encountering this error within UTFMax of the end and the current byte could be a
				// valid start, we'll just ignore the assumed partial rune.
				return n-i < utf8.UTFMax && utf8.RuneStart(p[i])
			}
			i += size
		}
	}
	return true
}

func addFileToIndex(ix *index.IndexWriter, dst, src, path string) (string, error) {
	rel, err := filepath.Rel(src, path)
	if err != nil {
		return "", err
	}

	r, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer r.Close()

	dup := filepath.Join(dst, "raw", rel)
	w, err := os.Create(dup)
	if err != nil {
		return "", err
	}
	defer w.Close()

	g := gzip.NewWriter(w)
	defer g.Close()

	return ix.Add(rel, io.TeeReader(r, g)), nil
}

func addDirToIndex(dst, src, path string) error {
	rel, err := filepath.Rel(src, path)
	if err != nil {
		return err
	}

	if rel == "." {
		return nil
	}

	dup := filepath.Join(dst, "raw", rel)
	return os.Mkdir(dup, os.ModePerm)
}

// write the list of excluded files to the given filename.
func writeExcludedFilesJson(filename string, files []*ExcludedFile) error {
	w, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer w.Close()

	return json.NewEncoder(w).Encode(files)
}

func containsString(haystack []string, needle string) bool {
	for i, n := 0, len(haystack); i < n; i++ {
		if haystack[i] == needle {
			return true
		}
	}
	return false
}

func indexAllFiles(opt *IndexOptions, dst, src string) error {
	ix := index.Create(filepath.Join(dst, "tri"))
	defer ix.Close()

	excluded := []*ExcludedFile{}

	// Make a file to store the excluded files for this repo
	fileHandle, err := os.Create(filepath.Join(dst, "excluded_files.json"))
	if err != nil {
		return err
	}
	defer fileHandle.Close()

	if err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
		name := info.Name()
		rel, err := filepath.Rel(src, path)
		if err != nil {
			return err
		}

		// Is this file considered "special", this means it's not even a part
		// of the source repository (like .git or .svn).
		if containsString(opt.SpecialFiles, name) {
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if opt.ExcludeDotFiles && name[0] == '.' {
			if info.IsDir() {
				return filepath.SkipDir
			}

			excluded = append(excluded, &ExcludedFile{
				rel,
				reasonDotFile,
			})
			return nil
		}

		if info.IsDir() {
			return addDirToIndex(dst, src, path)
		}

		if info.Mode()&os.ModeType != 0 {
			excluded = append(excluded, &ExcludedFile{
				rel,
				reasonInvalidMode,
			})
			return nil
		}

		txt, err := isTextFile(path)
		if err != nil {
			return err
		}

		if !txt {
			excluded = append(excluded, &ExcludedFile{
				rel,
				reasonNotText,
			})
			return nil
		}

		reasonForExclusion, err := addFileToIndex(ix, dst, src, path)
		if err != nil {
			return err
		}
		if reasonForExclusion != "" {
			excluded = append(excluded, &ExcludedFile{rel, reasonForExclusion})
		}

		return nil
	}); err != nil {
		return err
	}

	if err := writeExcludedFilesJson(
		filepath.Join(dst, excludedFileJsonFilename),
		excluded); err != nil {
		return err
	}

	ix.Flush()

	return nil
}

// Read the metadata for the index directory. Note that even if this
// returns a non-nil error, a Metadata object will be returned with
// all the information that is known about the index (this might
// include only the path)
func Read(dir string) (*IndexRef, error) {
	m := &IndexRef{
		dir: dir,
	}

	r, err := os.Open(filepath.Join(dir, manifestFilename))
	if err != nil {
		return m, err
	}
	defer r.Close()

	if err := gob.NewDecoder(r).Decode(m); err != nil {
		return m, err
	}

	return m, nil
}

func Build(opt *IndexOptions, dst, src, url, rev string) (*IndexRef, error) {
	if _, err := os.Stat(dst); err != nil {
		if err := os.MkdirAll(dst, os.ModePerm); err != nil {
			return nil, err
		}
	}

	if err := os.Mkdir(filepath.Join(dst, "raw"), os.ModePerm); err != nil {
		return nil, err
	}

	if err := indexAllFiles(opt, dst, src); err != nil {
		return nil, err
	}

	r := &IndexRef{
		Url:  url,
		Rev:  rev,
		Time: time.Now(),
		dir:  dst,
	}

	if err := r.writeManifest(); err != nil {
		return nil, err
	}

	return r, nil
}

// Open the index in dir for searching.
func Open(dir string) (*Index, error) {
	r, err := Read(dir)
	if err != nil {
		return nil, err
	}

	return r.Open()
}
<commit_msg>index: add a search option to exclude files given a regexp<commit_after>package index

import (
	"compress/gzip"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"
	"unicode/utf8"

	"github.com/etsy/hound/codesearch/index"
	"github.com/etsy/hound/codesearch/regexp"
)

const (
	matchLimit               = 5000
	manifestFilename         = "metadata.gob"
	excludedFileJsonFilename = "excluded_files.json"
	filePeekSize             = 2048
)

const (
	reasonDotFile     = "Dot files are excluded."
	reasonInvalidMode = "Invalid file mode."
	reasonNotText     = "Not a text file."
)

type Index struct {
	Ref *IndexRef
	idx *index.Index
	lck sync.RWMutex
}

type IndexOptions struct {
	ExcludeDotFiles bool
	SpecialFiles    []string
}

type SearchOptions struct {
	IgnoreCase        bool
	LinesOfContext    uint
	FileRegexp        string
	ExcludeFileRegexp string
	Offset            int
	Limit             int
}

type Match struct {
	Line       string
	LineNumber int
	Before     []string
	After      []string
}

type SearchResponse struct {
	Matches        []*FileMatch
	FilesWithMatch int
	FilesOpened    int           `json:"-"`
	Duration       time.Duration `json:"-"`
	Revision       string
}

type FileMatch struct {
	Filename string
	Matches  []*Match
}

type ExcludedFile struct {
	Filename string
	Reason   string
}

type IndexRef struct {
	Url  string
	Rev  string
	Time time.Time
	dir  string
}

func (r *IndexRef) Dir() string {
	return r.dir
}

func (r *IndexRef) writeManifest() error {
	w, err := os.Create(filepath.Join(r.dir, manifestFilename))
	if err != nil {
		return err
	}
	defer w.Close()

	return gob.NewEncoder(w).Encode(r)
}

func (r *IndexRef) Open() (*Index, error) {
	return &Index{
		Ref: r,
		idx: index.Open(filepath.Join(r.dir, "tri")),
	}, nil
}

func (r *IndexRef) Remove() error {
	return os.RemoveAll(r.dir)
}

func (n *Index) Close() error {
	n.lck.Lock()
	defer n.lck.Unlock()
	return n.idx.Close()
}

func (n *Index) Destroy() error {
	n.lck.Lock()
	defer n.lck.Unlock()
	if err := n.idx.Close(); err != nil {
		return err
	}
	return n.Ref.Remove()
}

func (n *Index) GetDir() string {
	return n.Ref.dir
}

func toStrings(lines [][]byte) []string {
	strs := make([]string, len(lines))
	for i, n := 0, len(lines); i < n; i++ {
		strs[i] = string(lines[i])
	}
	return strs
}

func GetRegexpPattern(pat string, ignoreCase bool) string {
	if ignoreCase {
		return "(?i)(?m)" + pat
	}
	return "(?m)" + pat
}

func (n *Index) Search(pat string, opt *SearchOptions) (*SearchResponse, error) {
	startedAt := time.Now()

	n.lck.RLock()
	defer n.lck.RUnlock()

	re, err := regexp.Compile(GetRegexpPattern(pat, opt.IgnoreCase))
	if err != nil {
		return nil, err
	}

	var (
		g                grepper
		results          []*FileMatch
		filesOpened      int
		filesFound       int
		filesCollected   int
		matchesCollected int
	)

	var fre *regexp.Regexp
	if opt.FileRegexp != "" {
		fre, err = regexp.Compile(opt.FileRegexp)
		if err != nil {
			return nil, err
		}
	}

	var efre *regexp.Regexp
	if opt.ExcludeFileRegexp != "" {
		efre, err = regexp.Compile(opt.ExcludeFileRegexp)
		if err != nil {
			return nil, err
		}
	}

	files := n.idx.PostingQuery(index.RegexpQuery(re.Syntax))
	for _, file := range files {
		var matches []*Match
		name := n.idx.Name(file)
		hasMatch := false

		// reject files that do not match the file pattern
		if fre != nil && fre.MatchString(name, true, true) < 0 {
			continue
		}

		// reject files that match the exclude file pattern
		if efre != nil && efre.MatchString(name, true, true) > 0 {
			continue
		}

		filesOpened++
		if err := g.grep2File(filepath.Join(n.Ref.dir, "raw", name), re, int(opt.LinesOfContext),
			func(line []byte, lineno int, before [][]byte, after [][]byte) (bool, error) {

				hasMatch = true
				if filesFound < opt.Offset || (opt.Limit > 0 && filesCollected >= opt.Limit) {
					return false, nil
				}

				matchesCollected++
				matches = append(matches, &Match{
					Line:       string(line),
					LineNumber: lineno,
					Before:     toStrings(before),
					After:      toStrings(after),
				})

				if matchesCollected > matchLimit {
					return false, fmt.Errorf("search exceeds limit on matches: %d", matchLimit)
				}

				return true, nil
			}); err != nil {
			return nil, err
		}

		if !hasMatch {
			continue
		}

		filesFound++
		if len(matches) > 0 {
			filesCollected++
			results = append(results, &FileMatch{
				Filename: name,
				Matches:  matches,
			})
		}
	}

	return &SearchResponse{
		Matches:        results,
		FilesWithMatch: filesFound,
		FilesOpened:    filesOpened,
		Duration:       time.Now().Sub(startedAt),
		Revision:       n.Ref.Rev,
	}, nil
}

func isTextFile(filename string) (bool, error) {
	buf := make([]byte, filePeekSize)
	r, err := os.Open(filename)
	if err != nil {
		return false, err
	}
	defer r.Close()

	n, err := io.ReadFull(r, buf)
	if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
		return false, err
	}

	buf = buf[:n]

	if n < filePeekSize {
		// read the whole file, must be valid.
		return utf8.Valid(buf), nil
	}

	// read a prefix, allow trailing partial runes.
	return validUTF8IgnoringPartialTrailingRune(buf), nil

}

// Determines if the buffer contains valid UTF8 encoded string data. The buffer is assumed
// to be a prefix of a larger buffer so if the buffer ends with the start of a rune, it
// is still considered valid.
//
// Basic logic copied from https://golang.org/pkg/unicode/utf8/#Valid
func validUTF8IgnoringPartialTrailingRune(p []byte) bool {
	i := 0
	n := len(p)

	for i < n {
		if p[i] < utf8.RuneSelf {
			i++
		} else {
			_, size := utf8.DecodeRune(p[i:])
			if size == 1 {
				// All valid runes of size 1 (those below RuneSelf) were handled above. This must be a RuneError.
				// If we're encountering this error within UTFMax of the end and the current byte could be a
				// valid start, we'll just ignore the assumed partial rune.
				return n-i < utf8.UTFMax && utf8.RuneStart(p[i])
			}
			i += size
		}
	}
	return true
}

func addFileToIndex(ix *index.IndexWriter, dst, src, path string) (string, error) {
	rel, err := filepath.Rel(src, path)
	if err != nil {
		return "", err
	}

	r, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer r.Close()

	dup := filepath.Join(dst, "raw", rel)
	w, err := os.Create(dup)
	if err != nil {
		return "", err
	}
	defer w.Close()

	g := gzip.NewWriter(w)
	defer g.Close()

	return ix.Add(rel, io.TeeReader(r, g)), nil
}

func addDirToIndex(dst, src, path string) error {
	rel, err := filepath.Rel(src, path)
	if err != nil {
		return err
	}

	if rel == "." {
		return nil
	}

	dup := filepath.Join(dst, "raw", rel)
	return os.Mkdir(dup, os.ModePerm)
}

// write the list of excluded files to the given filename.
func writeExcludedFilesJson(filename string, files []*ExcludedFile) error {
	w, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer w.Close()

	return json.NewEncoder(w).Encode(files)
}

func containsString(haystack []string, needle string) bool {
	for i, n := 0, len(haystack); i < n; i++ {
		if haystack[i] == needle {
			return true
		}
	}
	return false
}

func indexAllFiles(opt *IndexOptions, dst, src string) error {
	ix := index.Create(filepath.Join(dst, "tri"))
	defer ix.Close()

	excluded := []*ExcludedFile{}

	// Make a file to store the excluded files for this repo
	fileHandle, err := os.Create(filepath.Join(dst, "excluded_files.json"))
	if err != nil {
		return err
	}
	defer fileHandle.Close()

	if err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
		name := info.Name()
		rel, err := filepath.Rel(src, path)
		if err != nil {
			return err
		}

		// Is this file considered "special", this means it's not even a part
		// of the source repository (like .git or .svn).
		if containsString(opt.SpecialFiles, name) {
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if opt.ExcludeDotFiles && name[0] == '.' {
			if info.IsDir() {
				return filepath.SkipDir
			}

			excluded = append(excluded, &ExcludedFile{
				rel,
				reasonDotFile,
			})
			return nil
		}

		if info.IsDir() {
			return addDirToIndex(dst, src, path)
		}

		if info.Mode()&os.ModeType != 0 {
			excluded = append(excluded, &ExcludedFile{
				rel,
				reasonInvalidMode,
			})
			return nil
		}

		txt, err := isTextFile(path)
		if err != nil {
			return err
		}

		if !txt {
			excluded = append(excluded, &ExcludedFile{
				rel,
				reasonNotText,
			})
			return nil
		}

		reasonForExclusion, err := addFileToIndex(ix, dst, src, path)
		if err != nil {
			return err
		}
		if reasonForExclusion != "" {
			excluded = append(excluded, &ExcludedFile{rel, reasonForExclusion})
		}

		return nil
	}); err != nil {
		return err
	}

	if err := writeExcludedFilesJson(
		filepath.Join(dst, excludedFileJsonFilename),
		excluded); err != nil {
		return err
	}

	ix.Flush()

	return nil
}

// Read the metadata for the index directory. Note that even if this
// returns a non-nil error, a Metadata object will be returned with
// all the information that is known about the index (this might
// include only the path)
func Read(dir string) (*IndexRef, error) {
	m := &IndexRef{
		dir: dir,
	}

	r, err := os.Open(filepath.Join(dir, manifestFilename))
	if err != nil {
		return m, err
	}
	defer r.Close()

	if err := gob.NewDecoder(r).Decode(m); err != nil {
		return m, err
	}

	return m, nil
}

func Build(opt *IndexOptions, dst, src, url, rev string) (*IndexRef, error) {
	if _, err := os.Stat(dst); err != nil {
		if err := os.MkdirAll(dst, os.ModePerm); err != nil {
			return nil, err
		}
	}

	if err := os.Mkdir(filepath.Join(dst, "raw"), os.ModePerm); err != nil {
		return nil, err
	}

	if err := indexAllFiles(opt, dst, src); err != nil {
		return nil, err
	}

	r := &IndexRef{
		Url:  url,
		Rev:  rev,
		Time: time.Now(),
		dir:  dst,
	}

	if err := r.writeManifest(); err != nil {
		return nil, err
	}

	return r, nil
}

// Open the index in dir for searching.
func Open(dir string) (*Index, error) {
	r, err := Read(dir)
	if err != nil {
		return nil, err
	}

	return r.Open()
}
<|endoftext|>


<commit_before><commit_msg>Initial import<commit_after><|endoftext|>


<commit_before><commit_msg>added default handler<commit_after><|endoftext|>


<commit_before>package main

import (
 "fmt"
 "math")

 func main() {
     var y float64 = 8
 x := math.Log2(y)
    fmt.Printf("Hello, Vebjørn\n")
    fmt.Println(x)
    fmt.Println("Jeg bare skriver det samme som dere skrev, men har ikke mye peiling på hva det betyr")
    fmt.Println("JJ is back»)
    fmt.Println("Lønner seg å sette seg ned med tutorialen :)\nEr ganske likt java\nFunksjonene er bare fra standardbiblioteket")
  fmt.Println("Håper du kan lære meg litt av dette Vebbis!")
<<<<<<< HEAD
    fmt.Println("Men da fikk vi det til da")
=======
  fmt.println("Kim was here v2")
  fmt.println("Hei,hei!")
  fmt.println("Nå da?")
<<<<<<< HEAD
  fmt.println("Goodbye")
=======
  fmt.Println("sigh")
>>>>>>> 7dfcbf7a388311b9fd2d266f1cc34bb6e06cf14e
>>>>>>> a4c2d2728cf6c87f636d5c395410d076d57e46fe

 }
<commit_msg>la til tekst<commit_after>package main

import (
 "fmt"
 "math")

 func main() {
     var y float64 = 8
 x := math.Log2(y)
    fmt.Printf("Hello, Vebjørn\n")
    fmt.Println(x)
    fmt.Println("Jeg bare skriver det samme som dere skrev, men har ikke mye peiling på hva det betyr")
    fmt.Println("JJ is back»)
    fmt.Println("Lønner seg å sette seg ned med tutorialen :)\nEr ganske likt java\nFunksjonene er bare fra standardbiblioteket")
  fmt.Println("Håper du kan lære meg litt av dette Vebbis!")
<<<<<<< HEAD
    fmt.Println("Men da fikk vi det til da")
=======
  fmt.println("Kim was here v2")
  fmt.println("Hei,hei!")
  fmt.println("Nå da?")
  fmt.println("Testtest123")
<<<<<<< HEAD
  fmt.println("Goodbye")
=======
  fmt.Println("sigh")
>>>>>>> 7dfcbf7a388311b9fd2d266f1cc34bb6e06cf14e
>>>>>>> a4c2d2728cf6c87f636d5c395410d076d57e46fe

 }
<|endoftext|>