<commit_before>package main

import (
	"fmt"
	"log"
	"math/rand"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/bwmarrin/discordgo"
)

//OnMessageCreate gets called when a new message occurs in the Session
func OnMessageCreate(s *discordgo.Session, m discordgo.Message) {
	log.Printf("[%5s]: %5s > %s\n", m.ChannelID, m.Author.Username, m.Content)

	if strings.HasPrefix(m.Content, "!help") {
		SendMeuk(s, m)
	}

	if strings.HasPrefix(m.Content, "!slope") {
		s.ChannelMessageSend(m.ChannelID, "http://i.imgur.com/d5ML2op.png")
	}

	if strings.HasPrefix(m.Content, "!a") {
		s.ChannelMessageSend(m.ChannelID, "AAAA "+m.Author.Username+", LET MIE TINK OF THAT KWESTJUN")
		s.ChannelMessageSend(m.ChannelID, "THAT IS:")
		s.ChannelMessageSend(m.ChannelID, string(ReturnRandomLine("ball8")))
	}

	if strings.HasPrefix(m.Content, "!go") {
		out, err := exec.Command("go", "version").Output()
		if err != nil {
			fmt.Printf("%s", err)
		} else {
			s.ChannelMessageSend(m.ChannelID, string(out))
		}
	}

	if strings.HasPrefix(m.Content, "!timer") {
		var second = strings.Split(m.Content, " ")
		if len(second) != 1 {
			s.ChannelMessageSend(m.ChannelID, "Ok, <@"+m.Author.ID+">, timer voor "+second[1]+" minuten!")

			n, err := strconv.Atoi(second[1])
			if err != nil {
				log.Printf("%s", err)
			}

			timer1 := time.NewTimer(time.Minute * time.Duration(n))
			<-timer1.C
			fmt.Println("Timer 1 expired")
			s.ChannelMessageSend(m.ChannelID, "<@"+m.Author.ID+">, einde timer!")
		}

	}

	if strings.HasPrefix(m.Content, "!rain") {
		db, err := ConnectDB()
		if err != nil {
			log.Fatal(err)
		}

		if m.ChannelID == Server.SFWChannel { //SFW Check
			s.ChannelMessageSend(m.ChannelID, "Doe niet, dit is de SFW Channel")
			return
		}

		var Pictures = []Picture{}
		db.Find(&Pictures)
		for i := 0; i < len(Pictures); i++ {
			s.ChannelMessageSend(m.ChannelID, Pictures[i].URL)
		}

		db.Close()
	}

	if strings.HasPrefix(m.Content, "!ecchi") {
		db, err := ConnectDB()
		if err != nil {
			log.Fatal(err)
		}

		if m.ChannelID == Server.SFWChannel { //SFW Check
			s.ChannelMessageSend(m.ChannelID, "Doe niet, dit is de SFW Channel")
			return
		}
		var param = strings.Split(m.Content, " ")
		var Tag = []Tag{}
		var Pictures = []Picture{}

		if len(param) == 1 { //Return random from all
			db.Find(&Pictures)
			s.ChannelMessageSend(m.ChannelID, Pictures[rand.Intn(len(Pictures))].URL)
			db.Close()
			return
		}

		if len(param) == 2 {
			db.Preload("Pictures").Find(&Tag, "name = ?", param[1])
			if len(Tag) != 0 {
				s.ChannelMessageSend(m.ChannelID, Tag[0].Pictures[rand.Intn(len(Tag[0].Pictures))].URL) //Nice
			}
			db.Close()
			return
		}

		if len(param) == 3 {
			db.Preload("Pictures").Find(&Tag, "name = ?", param[1])
			if len(Tag) != 0 {
				for i := 0; i < len(Tag[0].Pictures); i++ {
					s.ChannelMessageSend(m.ChannelID, Tag[0].Pictures[i].URL)
				}
			}
			db.Close()
		}

	}

	if strings.HasPrefix(m.Content, "!token") {
		token, message, err := NewAuthSession(m)

		if message != "" {
			s.ChannelMessageSend(m.ChannelID, message)
		}

		if err != nil {
			log.Println(err)
			return
		}

		s.ChannelMessageSend(m.ChannelID, token)

	}

	if strings.HasPrefix(m.Content, "!web") {
		s.ChannelMessageSend(m.ChannelID, "Hier link naar Webinterface")
	}

	if strings.HasPrefix(m.Content, "!tags") {
		db, err := ConnectDB()
		if err != nil {
			log.Fatal(err)
		}

		if m.ChannelID == Server.SFWChannel { //SFW Check
			s.ChannelMessageSend(m.ChannelID, "Doe niet, dit is de SFW Channel")
			return
		}

		var Tags = []Tag{}
		db.Find(&Tags)
		for i := 0; i < len(Tags); i++ {
			s.ChannelMessageSend(m.ChannelID, Tags[i].Name)
		}

		db.Close()
	}

}

//OnReady gets called when a OnReady event happens in the Session
//This event is neccessary to keep a websocket connection with the Discord API
func OnReady(s *discordgo.Session, st discordgo.Ready) {
	fmt.Println("* OnReady fired.")
	// start the Heartbeat
	s.Heartbeat(st.HeartbeatInterval)
}
<commit_msg>Changed !ecchi to !picture, removed !web, fixes #1<commit_after>package main

import (
	"fmt"
	"log"
	"math/rand"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/bwmarrin/discordgo"
)

//OnMessageCreate gets called when a new message occurs in the Session
func OnMessageCreate(s *discordgo.Session, m discordgo.Message) {
	log.Printf("[%5s]: %5s > %s\n", m.ChannelID, m.Author.Username, m.Content)

	if strings.HasPrefix(m.Content, "!help") {
		SendMeuk(s, m)
	}

	if strings.HasPrefix(m.Content, "!slope") {
		s.ChannelMessageSend(m.ChannelID, "http://i.imgur.com/d5ML2op.png")
	}

	if strings.HasPrefix(m.Content, "!a") {
		s.ChannelMessageSend(m.ChannelID, "AAAA "+m.Author.Username+", LET MIE TINK OF THAT KWESTJUN")
		s.ChannelMessageSend(m.ChannelID, "THAT IS:")
		s.ChannelMessageSend(m.ChannelID, string(ReturnRandomLine("ball8")))
	}

	if strings.HasPrefix(m.Content, "!go") {
		out, err := exec.Command("go", "version").Output()
		if err != nil {
			fmt.Printf("%s", err)
		} else {
			s.ChannelMessageSend(m.ChannelID, string(out))
		}
	}

	if strings.HasPrefix(m.Content, "!timer") {
		var second = strings.Split(m.Content, " ")
		if len(second) != 1 {
			s.ChannelMessageSend(m.ChannelID, "Ok, <@"+m.Author.ID+">, timer voor "+second[1]+" minuten!")

			n, err := strconv.Atoi(second[1])
			if err != nil {
				log.Printf("%s", err)
			}

			timer1 := time.NewTimer(time.Minute * time.Duration(n))
			<-timer1.C
			fmt.Println("Timer 1 expired")
			s.ChannelMessageSend(m.ChannelID, "<@"+m.Author.ID+">, einde timer!")
		}

	}

	if strings.HasPrefix(m.Content, "!rain") {
		db, err := ConnectDB()
		if err != nil {
			log.Fatal(err)
		}

		if m.ChannelID == Server.SFWChannel { //SFW Check
			s.ChannelMessageSend(m.ChannelID, "Doe niet, dit is de SFW Channel")
			return
		}

		var Pictures = []Picture{}
		db.Find(&Pictures)
		for i := 0; i < len(Pictures); i++ {
			s.ChannelMessageSend(m.ChannelID, Pictures[i].URL)
		}

		db.Close()
	}

	if strings.HasPrefix(m.Content, "!picture") {
		db, err := ConnectDB()
		if err != nil {
			log.Fatal(err)
		}

		if m.ChannelID == Server.SFWChannel { //SFW Check
			s.ChannelMessageSend(m.ChannelID, "Doe niet, dit is de SFW Channel")
			return
		}
		var param = strings.Split(m.Content, " ")
		var Tag = []Tag{}
		var Pictures = []Picture{}

		if len(param) == 1 { //Return random from all
			db.Find(&Pictures)
			s.ChannelMessageSend(m.ChannelID, Pictures[rand.Intn(len(Pictures))].URL)
			db.Close()
			return
		}

		if len(param) == 2 {
			db.Preload("Pictures").Find(&Tag, "name = ?", param[1])
			if len(Tag) != 0 {
				s.ChannelMessageSend(m.ChannelID, Tag[0].Pictures[rand.Intn(len(Tag[0].Pictures))].URL) //Nice
			}
			db.Close()
			return
		}

		if len(param) == 3 {
			db.Preload("Pictures").Find(&Tag, "name = ?", param[1])
			if len(Tag) != 0 {
				for i := 0; i < len(Tag[0].Pictures); i++ {
					s.ChannelMessageSend(m.ChannelID, Tag[0].Pictures[i].URL)
				}
			}
			db.Close()
		}

	}

	if strings.HasPrefix(m.Content, "!token") {
		token, message, err := NewAuthSession(m)

		if message != "" {
			s.ChannelMessageSend(m.ChannelID, message)
		}

		if err != nil {
			log.Println(err)
			return
		}

		s.ChannelMessageSend(m.ChannelID, token)

	}

	if strings.HasPrefix(m.Content, "!tags") {
		db, err := ConnectDB()
		if err != nil {
			log.Fatal(err)
		}

		if m.ChannelID == Server.SFWChannel { //SFW Check
			s.ChannelMessageSend(m.ChannelID, "Doe niet, dit is de SFW Channel")
			return
		}

		var Tags = []Tag{}
		db.Find(&Tags)
		for i := 0; i < len(Tags); i++ {
			s.ChannelMessageSend(m.ChannelID, Tags[i].Name)
		}

		db.Close()
	}

}

//OnReady gets called when a OnReady event happens in the Session
//This event is neccessary to keep a websocket connection with the Discord API
func OnReady(s *discordgo.Session, st discordgo.Ready) {
	fmt.Println("* OnReady fired.")
	// start the Heartbeat
	s.Heartbeat(st.HeartbeatInterval)
}
<|endoftext|>


<commit_before>// +build windows

package sshprovider

import (
	"net"
	"regexp"
	"strings"

	"github.com/Microsoft/go-winio"
	"github.com/pkg/errors"
	"golang.org/x/sys/windows"
)

// Returns the Windows OpenSSH agent named pipe path, but
// only if the agent is running. Returns an error otherwise.
func getFallbackAgentPath() (string, error) {
	// Windows OpenSSH agent uses a named pipe rather
	// than a UNIX socket. These pipes do not play nice
	// with os.Stat (which tries to open its target), so
	// use a FindFirstFile syscall to check for existence.
	var fd windows.Win32finddata

	path := `\\.\pipe\openssh-ssh-agent`
	pathPtr, _ := windows.UTF16PtrFromString(path)
	handle, err := windows.FindFirstFile(pathPtr, &fd)

	if err != nil {
		msg := "Windows OpenSSH agent not available at %s." +
			" Enable the SSH agent service or set SSH_AUTH_SOCK."
		return "", errors.Errorf(msg, path)
	}

	_ = windows.CloseHandle(handle)

	return path, nil
}

// Returns true if the path references a named pipe.
func isWindowsPipePath(path string) bool {
	// If path matches \\*\pipe\* then it references a named pipe
	// and requires winio.DialPipe() rather than DialTimeout("unix").
	// Slashes and backslashes may be used interchangeably in the path.
	// Path separators may consist of multiple consecutive (back)slashes.
	pipePattern := strings.ReplaceAll("^[/]{2}[^/]+[/]+pipe[/]+", "[/]", `[\\/]`)
	ok, _ := regexp.MatchString(pipePattern, path)
	return ok
}

func getWindowsPipeDialer(path string) *socketDialer {
	if isWindowsPipePath(path) {
		return &socketDialer{path: path, dialer: windowsPipeDialer}
	}

	return nil
}

func windowsPipeDialer(path string) (net.Conn, error) {
	return winio.DialPipe(path, nil)
}
<commit_msg>Fix regular expression to test for Windows named pipe in SSH agent path; allowed backslashes in host.<commit_after>// +build windows

package sshprovider

import (
	"net"
	"regexp"
	"strings"

	"github.com/Microsoft/go-winio"
	"github.com/pkg/errors"
	"golang.org/x/sys/windows"
)

// Returns the Windows OpenSSH agent named pipe path, but
// only if the agent is running. Returns an error otherwise.
func getFallbackAgentPath() (string, error) {
	// Windows OpenSSH agent uses a named pipe rather
	// than a UNIX socket. These pipes do not play nice
	// with os.Stat (which tries to open its target), so
	// use a FindFirstFile syscall to check for existence.
	var fd windows.Win32finddata

	path := `\\.\pipe\openssh-ssh-agent`
	pathPtr, _ := windows.UTF16PtrFromString(path)
	handle, err := windows.FindFirstFile(pathPtr, &fd)

	if err != nil {
		msg := "Windows OpenSSH agent not available at %s." +
			" Enable the SSH agent service or set SSH_AUTH_SOCK."
		return "", errors.Errorf(msg, path)
	}

	_ = windows.CloseHandle(handle)

	return path, nil
}

// Returns true if the path references a named pipe.
func isWindowsPipePath(path string) bool {
	// If path matches \\*\pipe\* then it references a named pipe
	// and requires winio.DialPipe() rather than DialTimeout("unix").
	// Slashes and backslashes may be used interchangeably in the path.
	// Path separators may consist of multiple consecutive (back)slashes.
	pipePattern := strings.ReplaceAll("^[/]{2}[^/]+[/]+pipe[/]+", "/", `\\/`)
	ok, _ := regexp.MatchString(pipePattern, path)
	return ok
}

func getWindowsPipeDialer(path string) *socketDialer {
	if isWindowsPipePath(path) {
		return &socketDialer{path: path, dialer: windowsPipeDialer}
	}

	return nil
}

func windowsPipeDialer(path string) (net.Conn, error) {
	return winio.DialPipe(path, nil)
}
<|endoftext|>


<commit_before>//
// Copyright 2015, Sander van Harmelen
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

// PushEvent represents a push event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#push-events
type PushEvent struct {
	ObjectKind        string      `json:"object_kind"`
	Before            string      `json:"before"`
	After             string      `json:"after"`
	Ref               string      `json:"ref"`
	CheckoutSha       string      `json:"checkout_sha"`
	UserID            int         `json:"user_id"`
	UserName          string      `json:"user_name"`
	UserEmail         string      `json:"user_email"`
	UserAvatar        string      `json:"user_avatar"`
	ProjectID         int         `json:"project_id"`
	Project           *Project    `json:"project"`
	Repository        *Repository `json:"repository"`
	Commits           []*Commit   `json:"commits"`
	TotalCommitsCount int         `json:"total_commits_count"`
}

// TagEvent represents a tag event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#tag-events
type TagEvent struct {
	ObjectKind        string      `json:"object_kind"`
	Before            string      `json:"before"`
	After             string      `json:"after"`
	Ref               string      `json:"ref"`
	CheckoutSha       string      `json:"checkout_sha"`
	UserID            int         `json:"user_id"`
	UserName          string      `json:"user_name"`
	UserAvatar        string      `json:"user_avatar"`
	ProjectID         int         `json:"project_id"`
	Project           *Project    `json:"project"`
	Repository        *Repository `json:"repository"`
	Commits           []*Commit   `json:"commits"`
	TotalCommitsCount int         `json:"total_commits_count"`
}

// IssueEvent represents a issue event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#issues-events
type IssueEvent struct {
	ObjectKind       string      `json:"object_kind"`
	User             *User       `json:"user"`
	Project          *Project    `json:"project"`
	Repository       *Repository `json:"repository"`
	ObjectAttributes struct {
		ID          int    `json:"id"`
		Title       string `json:"title"`
		AssigneeID  int    `json:"assignee_id"`
		AuthorID    int    `json:"author_id"`
		ProjectID   int    `json:"project_id"`
		CreatedAt   string `json:"created_at"` // Should be *time.Time (see Gitlab issue #21468)
		UpdatedAt   string `json:"updated_at"` // Should be *time.Time (see Gitlab issue #21468)
		Position    int    `json:"position"`
		BranchName  string `json:"branch_name"`
		Description string `json:"description"`
		MilestoneID int    `json:"milestone_id"`
		State       string `json:"state"`
		Iid         int    `json:"iid"`
		URL         string `json:"url"`
		Action      string `json:"action"`
	} `json:"object_attributes"`
	Assignee struct {
		Name      string `json:"name"`
		Username  string `json:"username"`
		AvatarURL string `json:"avatar_url"`
	} `json:"assignee"`
}

// CommitCommentEvent represents a comment on a commit event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#comment-on-commit
type CommitCommentEvent struct {
	ObjectKind       string      `json:"object_kind"`
	User             *User       `json:"user"`
	ProjectID        int         `json:"project_id"`
	Project          *Project    `json:"project"`
	Repository       *Repository `json:"repository"`
	ObjectAttributes struct {
		ID           int    `json:"id"`
		Note         string `json:"note"`
		NoteableType string `json:"noteable_type"`
		AuthorID     int    `json:"author_id"`
		CreatedAt    string `json:"created_at"`
		UpdatedAt    string `json:"updated_at"`
		ProjectID    int    `json:"project_id"`
		Attachment   string `json:"attachment"`
		LineCode     string `json:"line_code"`
		CommitID     string `json:"commit_id"`
		NoteableID   int    `json:"noteable_id"`
		System       bool   `json:"system"`
		StDiff       struct {
			Diff        string `json:"diff"`
			NewPath     string `json:"new_path"`
			OldPath     string `json:"old_path"`
			AMode       string `json:"a_mode"`
			BMode       string `json:"b_mode"`
			NewFile     bool   `json:"new_file"`
			RenamedFile bool   `json:"renamed_file"`
			DeletedFile bool   `json:"deleted_file"`
		} `json:"st_diff"`
	} `json:"object_attributes"`
	Commit *Commit `json:"commit"`
}

// MergeCommentEvent represents a comment on a merge event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#comment-on-merge-request
type MergeCommentEvent struct {
	ObjectKind       string      `json:"object_kind"`
	User             *User       `json:"user"`
	ProjectID        int         `json:"project_id"`
	Project          *Project    `json:"project"`
	Repository       *Repository `json:"repository"`
	ObjectAttributes struct {
		ID           int    `json:"id"`
		Note         string `json:"note"`
		NoteableType string `json:"noteable_type"`
		AuthorID     int    `json:"author_id"`
		CreatedAt    string `json:"created_at"`
		UpdatedAt    string `json:"updated_at"`
		ProjectID    int    `json:"project_id"`
		Attachment   string `json:"attachment"`
		LineCode     string `json:"line_code"`
		CommitID     string `json:"commit_id"`
		NoteableID   int    `json:"noteable_id"`
		System       bool   `json:"system"`
		StDiff       *Diff  `json:"st_diff"`
		URL          string `json:"url"`
	} `json:"object_attributes"`
	MergeRequest *MergeRequest `json:"merge_request"`
}

// IssueCommentEvent represents a comment on an issue event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#comment-on-issue
type IssueCommentEvent struct {
	ObjectKind       string      `json:"object_kind"`
	User             *User       `json:"user"`
	ProjectID        int         `json:"project_id"`
	Project          *Project    `json:"project"`
	Repository       *Repository `json:"repository"`
	ObjectAttributes struct {
		ID           int     `json:"id"`
		Note         string  `json:"note"`
		NoteableType string  `json:"noteable_type"`
		AuthorID     int     `json:"author_id"`
		CreatedAt    string  `json:"created_at"`
		UpdatedAt    string  `json:"updated_at"`
		ProjectID    int     `json:"project_id"`
		Attachment   string  `json:"attachment"`
		LineCode     string  `json:"line_code"`
		CommitID     string  `json:"commit_id"`
		NoteableID   int     `json:"noteable_id"`
		System       bool    `json:"system"`
		StDiff       []*Diff `json:"st_diff"`
		URL          string  `json:"url"`
	} `json:"object_attributes"`
	Issue *Issue `json:"issue"`
}

// SnippetCommentEvent represents a comment on a snippet event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#comment-on-code-snippet
type SnippetCommentEvent struct {
	ObjectKind       string      `json:"object_kind"`
	User             *User       `json:"user"`
	ProjectID        int         `json:"project_id"`
	Project          *Project    `json:"project"`
	Repository       *Repository `json:"repository"`
	ObjectAttributes struct {
		ID           int    `json:"id"`
		Note         string `json:"note"`
		NoteableType string `json:"noteable_type"`
		AuthorID     int    `json:"author_id"`
		CreatedAt    string `json:"created_at"`
		UpdatedAt    string `json:"updated_at"`
		ProjectID    int    `json:"project_id"`
		Attachment   string `json:"attachment"`
		LineCode     string `json:"line_code"`
		CommitID     string `json:"commit_id"`
		NoteableID   int    `json:"noteable_id"`
		System       bool   `json:"system"`
		StDiff       *Diff  `json:"st_diff"`
		URL          string `json:"url"`
	} `json:"object_attributes"`
	Snippet *Snippet `json:"snippet"`
}

// MergeEvent represents a merge event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#merge-request-events
type MergeEvent struct {
	ObjectKind       string `json:"object_kind"`
	User             *User  `json:"user"`
	ObjectAttributes struct {
		ID              int         `json:"id"`
		TargetBranch    string      `json:"target_branch"`
		SourceBranch    string      `json:"source_branch"`
		SourceProjectID int         `json:"source_project_id"`
		AuthorID        int         `json:"author_id"`
		AssigneeID      int         `json:"assignee_id"`
		Title           string      `json:"title"`
		CreatedAt       string      `json:"created_at"` // Should be *time.Time (see Gitlab issue #21468)
		UpdatedAt       string      `json:"updated_at"` // Should be *time.Time (see Gitlab issue #21468)
		StCommits       []*Commit   `json:"st_commits"`
		StDiffs         []*Diff     `json:"st_diffs"`
		MilestoneID     int         `json:"milestone_id"`
		State           string      `json:"state"`
		MergeStatus     string      `json:"merge_status"`
		TargetProjectID int         `json:"target_project_id"`
		Iid             int         `json:"iid"`
		Description     string      `json:"description"`
		Source          *Repository `json:"source"`
		Target          *Repository `json:"target"`
		LastCommit      *Commit     `json:"last_commit"`
		WorkInProgress  bool        `json:"work_in_progress"`
		URL             string      `json:"url"`
		Action          string      `json:"action"`
		Assignee        struct {
			Name      string `json:"name"`
			Username  string `json:"username"`
			AvatarURL string `json:"avatar_url"`
		} `json:"assignee"`
	} `json:"object_attributes"`
}

// WikiPageEvent represents a wiki page event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#wiki-page-events
type WikiPageEvent struct {
	ObjectKind string   `json:"object_kind"`
	User       *User    `json:"user"`
	Project    *Project `json:"project"`
	Wiki       struct {
		WebURL            string `json:"web_url"`
		GitSSHURL         string `json:"git_ssh_url"`
		GitHTTPURL        string `json:"git_http_url"`
		PathWithNamespace string `json:"path_with_namespace"`
		DefaultBranch     string `json:"default_branch"`
	} `json:"wiki"`
	ObjectAttributes struct {
		Title   string `json:"title"`
		Content string `json:"content"`
		Format  string `json:"format"`
		Message string `json:"message"`
		Slug    string `json:"slug"`
		URL     string `json:"url"`
		Action  string `json:"action"`
	} `json:"object_attributes"`
}

// PipelineEvent represents a pipline event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#pipeline-events
type PipelineEvent struct {
	ObjectKind       string `json:"object_kind"`
	ObjectAttributes struct {
		ID         int      `json:"id"`
		Ref        string   `json:"ref"`
		Tag        bool     `json:"tag"`
		Sha        string   `json:"sha"`
		BeforeSha  string   `json:"before_sha"`
		Status     string   `json:"status"`
		Stages     []string `json:"stages"`
		CreatedAt  string   `json:"created_at"`
		FinishedAt string   `json:"finished_at"`
		Duration   int      `json:"duration"`
	} `json:"object_attributes"`
	User    *User    `json:"user"`
	Project *Project `json:"project"`
	Commit  *Commit  `json:"commit"`
	Builds  []*Build `json:"builds"`
}
<commit_msg>Fix push event struct (#97)<commit_after>//
// Copyright 2015, Sander van Harmelen
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

// PushEvent represents a push event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#push-events
type PushEvent struct {
	ObjectKind  string `json:"object_kind"`
	Before      string `json:"before"`
	After       string `json:"after"`
	Ref         string `json:"ref"`
	CheckoutSha string `json:"checkout_sha"`
	UserID      int    `json:"user_id"`
	UserName    string `json:"user_name"`
	UserEmail   string `json:"user_email"`
	UserAvatar  string `json:"user_avatar"`
	ProjectID   int    `json:"project_id"`
	Project     struct {
		Name              string               `json:"name"`
		Description       string               `json:"description"`
		AvatarURL         string               `json:"avatar_url"`
		GitSSHURL         string               `json:"git_ssh_url"`
		GitHTTPURL        string               `json:"git_http_url"`
		Namespace         string               `json:"namespace"`
		PathWithNamespace string               `json:"path_with_namespace"`
		DefaultBranch     string               `json:"default_branch"`
		Homepage          string               `json:"homepage"`
		URL               string               `json:"url"`
		SSHURL            string               `json:"ssh_url"`
		HTTPURL           string               `json:"http_url"`
		WebURL            string               `json:"web_url"`
		VisibilityLevel   VisibilityLevelValue `json:"visibility_level"`
	} `json:"project"`
	Repository        *Repository `json:"repository"`
	Commits           []*Commit   `json:"commits"`
	TotalCommitsCount int         `json:"total_commits_count"`
}

// TagEvent represents a tag event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#tag-events
type TagEvent struct {
	ObjectKind        string      `json:"object_kind"`
	Before            string      `json:"before"`
	After             string      `json:"after"`
	Ref               string      `json:"ref"`
	CheckoutSha       string      `json:"checkout_sha"`
	UserID            int         `json:"user_id"`
	UserName          string      `json:"user_name"`
	UserAvatar        string      `json:"user_avatar"`
	ProjectID         int         `json:"project_id"`
	Project           *Project    `json:"project"`
	Repository        *Repository `json:"repository"`
	Commits           []*Commit   `json:"commits"`
	TotalCommitsCount int         `json:"total_commits_count"`
}

// IssueEvent represents a issue event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#issues-events
type IssueEvent struct {
	ObjectKind       string      `json:"object_kind"`
	User             *User       `json:"user"`
	Project          *Project    `json:"project"`
	Repository       *Repository `json:"repository"`
	ObjectAttributes struct {
		ID          int    `json:"id"`
		Title       string `json:"title"`
		AssigneeID  int    `json:"assignee_id"`
		AuthorID    int    `json:"author_id"`
		ProjectID   int    `json:"project_id"`
		CreatedAt   string `json:"created_at"` // Should be *time.Time (see Gitlab issue #21468)
		UpdatedAt   string `json:"updated_at"` // Should be *time.Time (see Gitlab issue #21468)
		Position    int    `json:"position"`
		BranchName  string `json:"branch_name"`
		Description string `json:"description"`
		MilestoneID int    `json:"milestone_id"`
		State       string `json:"state"`
		Iid         int    `json:"iid"`
		URL         string `json:"url"`
		Action      string `json:"action"`
	} `json:"object_attributes"`
	Assignee struct {
		Name      string `json:"name"`
		Username  string `json:"username"`
		AvatarURL string `json:"avatar_url"`
	} `json:"assignee"`
}

// CommitCommentEvent represents a comment on a commit event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#comment-on-commit
type CommitCommentEvent struct {
	ObjectKind       string      `json:"object_kind"`
	User             *User       `json:"user"`
	ProjectID        int         `json:"project_id"`
	Project          *Project    `json:"project"`
	Repository       *Repository `json:"repository"`
	ObjectAttributes struct {
		ID           int    `json:"id"`
		Note         string `json:"note"`
		NoteableType string `json:"noteable_type"`
		AuthorID     int    `json:"author_id"`
		CreatedAt    string `json:"created_at"`
		UpdatedAt    string `json:"updated_at"`
		ProjectID    int    `json:"project_id"`
		Attachment   string `json:"attachment"`
		LineCode     string `json:"line_code"`
		CommitID     string `json:"commit_id"`
		NoteableID   int    `json:"noteable_id"`
		System       bool   `json:"system"`
		StDiff       struct {
			Diff        string `json:"diff"`
			NewPath     string `json:"new_path"`
			OldPath     string `json:"old_path"`
			AMode       string `json:"a_mode"`
			BMode       string `json:"b_mode"`
			NewFile     bool   `json:"new_file"`
			RenamedFile bool   `json:"renamed_file"`
			DeletedFile bool   `json:"deleted_file"`
		} `json:"st_diff"`
	} `json:"object_attributes"`
	Commit *Commit `json:"commit"`
}

// MergeCommentEvent represents a comment on a merge event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#comment-on-merge-request
type MergeCommentEvent struct {
	ObjectKind       string      `json:"object_kind"`
	User             *User       `json:"user"`
	ProjectID        int         `json:"project_id"`
	Project          *Project    `json:"project"`
	Repository       *Repository `json:"repository"`
	ObjectAttributes struct {
		ID           int    `json:"id"`
		Note         string `json:"note"`
		NoteableType string `json:"noteable_type"`
		AuthorID     int    `json:"author_id"`
		CreatedAt    string `json:"created_at"`
		UpdatedAt    string `json:"updated_at"`
		ProjectID    int    `json:"project_id"`
		Attachment   string `json:"attachment"`
		LineCode     string `json:"line_code"`
		CommitID     string `json:"commit_id"`
		NoteableID   int    `json:"noteable_id"`
		System       bool   `json:"system"`
		StDiff       *Diff  `json:"st_diff"`
		URL          string `json:"url"`
	} `json:"object_attributes"`
	MergeRequest *MergeRequest `json:"merge_request"`
}

// IssueCommentEvent represents a comment on an issue event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#comment-on-issue
type IssueCommentEvent struct {
	ObjectKind       string      `json:"object_kind"`
	User             *User       `json:"user"`
	ProjectID        int         `json:"project_id"`
	Project          *Project    `json:"project"`
	Repository       *Repository `json:"repository"`
	ObjectAttributes struct {
		ID           int     `json:"id"`
		Note         string  `json:"note"`
		NoteableType string  `json:"noteable_type"`
		AuthorID     int     `json:"author_id"`
		CreatedAt    string  `json:"created_at"`
		UpdatedAt    string  `json:"updated_at"`
		ProjectID    int     `json:"project_id"`
		Attachment   string  `json:"attachment"`
		LineCode     string  `json:"line_code"`
		CommitID     string  `json:"commit_id"`
		NoteableID   int     `json:"noteable_id"`
		System       bool    `json:"system"`
		StDiff       []*Diff `json:"st_diff"`
		URL          string  `json:"url"`
	} `json:"object_attributes"`
	Issue *Issue `json:"issue"`
}

// SnippetCommentEvent represents a comment on a snippet event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#comment-on-code-snippet
type SnippetCommentEvent struct {
	ObjectKind       string      `json:"object_kind"`
	User             *User       `json:"user"`
	ProjectID        int         `json:"project_id"`
	Project          *Project    `json:"project"`
	Repository       *Repository `json:"repository"`
	ObjectAttributes struct {
		ID           int    `json:"id"`
		Note         string `json:"note"`
		NoteableType string `json:"noteable_type"`
		AuthorID     int    `json:"author_id"`
		CreatedAt    string `json:"created_at"`
		UpdatedAt    string `json:"updated_at"`
		ProjectID    int    `json:"project_id"`
		Attachment   string `json:"attachment"`
		LineCode     string `json:"line_code"`
		CommitID     string `json:"commit_id"`
		NoteableID   int    `json:"noteable_id"`
		System       bool   `json:"system"`
		StDiff       *Diff  `json:"st_diff"`
		URL          string `json:"url"`
	} `json:"object_attributes"`
	Snippet *Snippet `json:"snippet"`
}

// MergeEvent represents a merge event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#merge-request-events
type MergeEvent struct {
	ObjectKind       string `json:"object_kind"`
	User             *User  `json:"user"`
	ObjectAttributes struct {
		ID              int         `json:"id"`
		TargetBranch    string      `json:"target_branch"`
		SourceBranch    string      `json:"source_branch"`
		SourceProjectID int         `json:"source_project_id"`
		AuthorID        int         `json:"author_id"`
		AssigneeID      int         `json:"assignee_id"`
		Title           string      `json:"title"`
		CreatedAt       string      `json:"created_at"` // Should be *time.Time (see Gitlab issue #21468)
		UpdatedAt       string      `json:"updated_at"` // Should be *time.Time (see Gitlab issue #21468)
		StCommits       []*Commit   `json:"st_commits"`
		StDiffs         []*Diff     `json:"st_diffs"`
		MilestoneID     int         `json:"milestone_id"`
		State           string      `json:"state"`
		MergeStatus     string      `json:"merge_status"`
		TargetProjectID int         `json:"target_project_id"`
		Iid             int         `json:"iid"`
		Description     string      `json:"description"`
		Source          *Repository `json:"source"`
		Target          *Repository `json:"target"`
		LastCommit      *Commit     `json:"last_commit"`
		WorkInProgress  bool        `json:"work_in_progress"`
		URL             string      `json:"url"`
		Action          string      `json:"action"`
		Assignee        struct {
			Name      string `json:"name"`
			Username  string `json:"username"`
			AvatarURL string `json:"avatar_url"`
		} `json:"assignee"`
	} `json:"object_attributes"`
}

// WikiPageEvent represents a wiki page event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#wiki-page-events
type WikiPageEvent struct {
	ObjectKind string   `json:"object_kind"`
	User       *User    `json:"user"`
	Project    *Project `json:"project"`
	Wiki       struct {
		WebURL            string `json:"web_url"`
		GitSSHURL         string `json:"git_ssh_url"`
		GitHTTPURL        string `json:"git_http_url"`
		PathWithNamespace string `json:"path_with_namespace"`
		DefaultBranch     string `json:"default_branch"`
	} `json:"wiki"`
	ObjectAttributes struct {
		Title   string `json:"title"`
		Content string `json:"content"`
		Format  string `json:"format"`
		Message string `json:"message"`
		Slug    string `json:"slug"`
		URL     string `json:"url"`
		Action  string `json:"action"`
	} `json:"object_attributes"`
}

// PipelineEvent represents a pipline event.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/web_hooks/web_hooks.html#pipeline-events
type PipelineEvent struct {
	ObjectKind       string `json:"object_kind"`
	ObjectAttributes struct {
		ID         int      `json:"id"`
		Ref        string   `json:"ref"`
		Tag        bool     `json:"tag"`
		Sha        string   `json:"sha"`
		BeforeSha  string   `json:"before_sha"`
		Status     string   `json:"status"`
		Stages     []string `json:"stages"`
		CreatedAt  string   `json:"created_at"`
		FinishedAt string   `json:"finished_at"`
		Duration   int      `json:"duration"`
	} `json:"object_attributes"`
	User    *User    `json:"user"`
	Project *Project `json:"project"`
	Commit  *Commit  `json:"commit"`
	Builds  []*Build `json:"builds"`
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package exec

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"runtime"
	"strconv"
	"strings"
	"testing"
)

func helperCommand(s ...string) *Cmd {
	cs := []string{"-test.run=TestHelperProcess", "--"}
	cs = append(cs, s...)
	cmd := Command(os.Args[0], cs...)
	cmd.Env = append([]string{"GO_WANT_HELPER_PROCESS=1"}, os.Environ()...)
	return cmd
}

func TestEcho(t *testing.T) {
	bs, err := helperCommand("echo", "foo bar", "baz").Output()
	if err != nil {
		t.Errorf("echo: %v", err)
	}
	if g, e := string(bs), "foo bar baz\n"; g != e {
		t.Errorf("echo: want %q, got %q", e, g)
	}
}

func TestCatStdin(t *testing.T) {
	// Cat, testing stdin and stdout.
	input := "Input string\nLine 2"
	p := helperCommand("cat")
	p.Stdin = strings.NewReader(input)
	bs, err := p.Output()
	if err != nil {
		t.Errorf("cat: %v", err)
	}
	s := string(bs)
	if s != input {
		t.Errorf("cat: want %q, got %q", input, s)
	}
}

func TestCatGoodAndBadFile(t *testing.T) {
	// Testing combined output and error values.
	bs, err := helperCommand("cat", "/bogus/file.foo", "exec_test.go").CombinedOutput()
	if _, ok := err.(*ExitError); !ok {
		t.Errorf("expected *ExitError from cat combined; got %T: %v", err, err)
	}
	s := string(bs)
	sp := strings.SplitN(s, "\n", 2)
	if len(sp) != 2 {
		t.Fatalf("expected two lines from cat; got %q", s)
	}
	errLine, body := sp[0], sp[1]
	if !strings.HasPrefix(errLine, "Error: open /bogus/file.foo") {
		t.Errorf("expected stderr to complain about file; got %q", errLine)
	}
	if !strings.Contains(body, "func TestHelperProcess(t *testing.T)") {
		t.Errorf("expected test code; got %q (len %d)", body, len(body))
	}
}

func TestNoExistBinary(t *testing.T) {
	// Can't run a non-existent binary
	err := Command("/no-exist-binary").Run()
	if err == nil {
		t.Error("expected error from /no-exist-binary")
	}
}

func TestExitStatus(t *testing.T) {
	// Test that exit values are returned correctly
	err := helperCommand("exit", "42").Run()
	if werr, ok := err.(*ExitError); ok {
		if s, e := werr.Error(), "exit status 42"; s != e {
			t.Errorf("from exit 42 got exit %q, want %q", s, e)
		}
	} else {
		t.Fatalf("expected *ExitError from exit 42; got %T: %v", err, err)
	}
}

func TestPipes(t *testing.T) {
	check := func(what string, err error) {
		if err != nil {
			t.Fatalf("%s: %v", what, err)
		}
	}
	// Cat, testing stdin and stdout.
	c := helperCommand("pipetest")
	stdin, err := c.StdinPipe()
	check("StdinPipe", err)
	stdout, err := c.StdoutPipe()
	check("StdoutPipe", err)
	stderr, err := c.StderrPipe()
	check("StderrPipe", err)

	outbr := bufio.NewReader(stdout)
	errbr := bufio.NewReader(stderr)
	line := func(what string, br *bufio.Reader) string {
		line, _, err := br.ReadLine()
		if err != nil {
			t.Fatalf("%s: %v", what, err)
		}
		return string(line)
	}

	err = c.Start()
	check("Start", err)

	_, err = stdin.Write([]byte("O:I am output\n"))
	check("first stdin Write", err)
	if g, e := line("first output line", outbr), "O:I am output"; g != e {
		t.Errorf("got %q, want %q", g, e)
	}

	_, err = stdin.Write([]byte("E:I am error\n"))
	check("second stdin Write", err)
	if g, e := line("first error line", errbr), "E:I am error"; g != e {
		t.Errorf("got %q, want %q", g, e)
	}

	_, err = stdin.Write([]byte("O:I am output2\n"))
	check("third stdin Write 3", err)
	if g, e := line("second output line", outbr), "O:I am output2"; g != e {
		t.Errorf("got %q, want %q", g, e)
	}

	stdin.Close()
	err = c.Wait()
	check("Wait", err)
}

func TestExtraFiles(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Logf("no operating system support; skipping")
		return
	}

	// Force network usage, to verify the epoll (or whatever) fd
	// doesn't leak to the child,
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	defer ln.Close()

	tf, err := ioutil.TempFile("", "")
	if err != nil {
		t.Fatalf("TempFile: %v", err)
	}
	defer os.Remove(tf.Name())
	defer tf.Close()

	const text = "Hello, fd 3!"
	_, err = tf.Write([]byte(text))
	if err != nil {
		t.Fatalf("Write: %v", err)
	}
	_, err = tf.Seek(0, os.SEEK_SET)
	if err != nil {
		t.Fatalf("Seek: %v", err)
	}

	c := helperCommand("read3")
	c.ExtraFiles = []*os.File{tf}
	bs, err := c.CombinedOutput()
	if err != nil {
		t.Fatalf("CombinedOutput: %v; output %q", err, bs)
	}
	if string(bs) != text {
		t.Errorf("got %q; want %q", string(bs), text)
	}
}

// TestHelperProcess isn't a real test. It's used as a helper process
// for TestParameterRun.
func TestHelperProcess(*testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
		return
	}
	defer os.Exit(0)

	args := os.Args
	for len(args) > 0 {
		if args[0] == "--" {
			args = args[1:]
			break
		}
		args = args[1:]
	}
	if len(args) == 0 {
		fmt.Fprintf(os.Stderr, "No command\n")
		os.Exit(2)
	}

	cmd, args := args[0], args[1:]
	switch cmd {
	case "echo":
		iargs := []interface{}{}
		for _, s := range args {
			iargs = append(iargs, s)
		}
		fmt.Println(iargs...)
	case "cat":
		if len(args) == 0 {
			io.Copy(os.Stdout, os.Stdin)
			return
		}
		exit := 0
		for _, fn := range args {
			f, err := os.Open(fn)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error: %v\n", err)
				exit = 2
			} else {
				defer f.Close()
				io.Copy(os.Stdout, f)
			}
		}
		os.Exit(exit)
	case "pipetest":
		bufr := bufio.NewReader(os.Stdin)
		for {
			line, _, err := bufr.ReadLine()
			if err == io.EOF {
				break
			} else if err != nil {
				os.Exit(1)
			}
			if bytes.HasPrefix(line, []byte("O:")) {
				os.Stdout.Write(line)
				os.Stdout.Write([]byte{'\n'})
			} else if bytes.HasPrefix(line, []byte("E:")) {
				os.Stderr.Write(line)
				os.Stderr.Write([]byte{'\n'})
			} else {
				os.Exit(1)
			}
		}
	case "read3": // read fd 3
		fd3 := os.NewFile(3, "fd3")
		bs, err := ioutil.ReadAll(fd3)
		if err != nil {
			fmt.Printf("ReadAll from fd 3: %v", err)
			os.Exit(1)
		}
		// Now verify that there are no other open fds.
		var files []*os.File
		for wantfd := os.Stderr.Fd() + 2; wantfd <= 100; wantfd++ {
			f, err := os.Open(os.Args[0])
			if err != nil {
				fmt.Printf("error opening file with expected fd %d: %v", wantfd, err)
				os.Exit(1)
			}
			if got := f.Fd(); got != wantfd {
				fmt.Printf("leaked parent file. fd = %d; want %d", got, wantfd)
				os.Exit(1)
			}
			files = append(files, f)
		}
		for _, f := range files {
			f.Close()
		}
		os.Stderr.Write(bs)
	case "exit":
		n, _ := strconv.Atoi(args[0])
		os.Exit(n)
	default:
		fmt.Fprintf(os.Stderr, "Unknown command %q\n", cmd)
		os.Exit(2)
	}
}
<commit_msg>exec: disable new test to fix build<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package exec

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"runtime"
	"strconv"
	"strings"
	"testing"
)

func helperCommand(s ...string) *Cmd {
	cs := []string{"-test.run=TestHelperProcess", "--"}
	cs = append(cs, s...)
	cmd := Command(os.Args[0], cs...)
	cmd.Env = append([]string{"GO_WANT_HELPER_PROCESS=1"}, os.Environ()...)
	return cmd
}

func TestEcho(t *testing.T) {
	bs, err := helperCommand("echo", "foo bar", "baz").Output()
	if err != nil {
		t.Errorf("echo: %v", err)
	}
	if g, e := string(bs), "foo bar baz\n"; g != e {
		t.Errorf("echo: want %q, got %q", e, g)
	}
}

func TestCatStdin(t *testing.T) {
	// Cat, testing stdin and stdout.
	input := "Input string\nLine 2"
	p := helperCommand("cat")
	p.Stdin = strings.NewReader(input)
	bs, err := p.Output()
	if err != nil {
		t.Errorf("cat: %v", err)
	}
	s := string(bs)
	if s != input {
		t.Errorf("cat: want %q, got %q", input, s)
	}
}

func TestCatGoodAndBadFile(t *testing.T) {
	// Testing combined output and error values.
	bs, err := helperCommand("cat", "/bogus/file.foo", "exec_test.go").CombinedOutput()
	if _, ok := err.(*ExitError); !ok {
		t.Errorf("expected *ExitError from cat combined; got %T: %v", err, err)
	}
	s := string(bs)
	sp := strings.SplitN(s, "\n", 2)
	if len(sp) != 2 {
		t.Fatalf("expected two lines from cat; got %q", s)
	}
	errLine, body := sp[0], sp[1]
	if !strings.HasPrefix(errLine, "Error: open /bogus/file.foo") {
		t.Errorf("expected stderr to complain about file; got %q", errLine)
	}
	if !strings.Contains(body, "func TestHelperProcess(t *testing.T)") {
		t.Errorf("expected test code; got %q (len %d)", body, len(body))
	}
}

func TestNoExistBinary(t *testing.T) {
	// Can't run a non-existent binary
	err := Command("/no-exist-binary").Run()
	if err == nil {
		t.Error("expected error from /no-exist-binary")
	}
}

func TestExitStatus(t *testing.T) {
	// Test that exit values are returned correctly
	err := helperCommand("exit", "42").Run()
	if werr, ok := err.(*ExitError); ok {
		if s, e := werr.Error(), "exit status 42"; s != e {
			t.Errorf("from exit 42 got exit %q, want %q", s, e)
		}
	} else {
		t.Fatalf("expected *ExitError from exit 42; got %T: %v", err, err)
	}
}

func TestPipes(t *testing.T) {
	check := func(what string, err error) {
		if err != nil {
			t.Fatalf("%s: %v", what, err)
		}
	}
	// Cat, testing stdin and stdout.
	c := helperCommand("pipetest")
	stdin, err := c.StdinPipe()
	check("StdinPipe", err)
	stdout, err := c.StdoutPipe()
	check("StdoutPipe", err)
	stderr, err := c.StderrPipe()
	check("StderrPipe", err)

	outbr := bufio.NewReader(stdout)
	errbr := bufio.NewReader(stderr)
	line := func(what string, br *bufio.Reader) string {
		line, _, err := br.ReadLine()
		if err != nil {
			t.Fatalf("%s: %v", what, err)
		}
		return string(line)
	}

	err = c.Start()
	check("Start", err)

	_, err = stdin.Write([]byte("O:I am output\n"))
	check("first stdin Write", err)
	if g, e := line("first output line", outbr), "O:I am output"; g != e {
		t.Errorf("got %q, want %q", g, e)
	}

	_, err = stdin.Write([]byte("E:I am error\n"))
	check("second stdin Write", err)
	if g, e := line("first error line", errbr), "E:I am error"; g != e {
		t.Errorf("got %q, want %q", g, e)
	}

	_, err = stdin.Write([]byte("O:I am output2\n"))
	check("third stdin Write 3", err)
	if g, e := line("second output line", outbr), "O:I am output2"; g != e {
		t.Errorf("got %q, want %q", g, e)
	}

	stdin.Close()
	err = c.Wait()
	check("Wait", err)
}

func TestExtraFiles(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Logf("no operating system support; skipping")
		return
	}

	// Force network usage, to verify the epoll (or whatever) fd
	// doesn't leak to the child,
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	defer ln.Close()

	tf, err := ioutil.TempFile("", "")
	if err != nil {
		t.Fatalf("TempFile: %v", err)
	}
	defer os.Remove(tf.Name())
	defer tf.Close()

	const text = "Hello, fd 3!"
	_, err = tf.Write([]byte(text))
	if err != nil {
		t.Fatalf("Write: %v", err)
	}
	_, err = tf.Seek(0, os.SEEK_SET)
	if err != nil {
		t.Fatalf("Seek: %v", err)
	}

	c := helperCommand("read3")
	c.ExtraFiles = []*os.File{tf}
	bs, err := c.CombinedOutput()
	if err != nil {
		t.Fatalf("CombinedOutput: %v; output %q", err, bs)
	}
	if string(bs) != text {
		t.Errorf("got %q; want %q", string(bs), text)
	}
}

// TestHelperProcess isn't a real test. It's used as a helper process
// for TestParameterRun.
func TestHelperProcess(*testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
		return
	}
	defer os.Exit(0)

	args := os.Args
	for len(args) > 0 {
		if args[0] == "--" {
			args = args[1:]
			break
		}
		args = args[1:]
	}
	if len(args) == 0 {
		fmt.Fprintf(os.Stderr, "No command\n")
		os.Exit(2)
	}

	cmd, args := args[0], args[1:]
	switch cmd {
	case "echo":
		iargs := []interface{}{}
		for _, s := range args {
			iargs = append(iargs, s)
		}
		fmt.Println(iargs...)
	case "cat":
		if len(args) == 0 {
			io.Copy(os.Stdout, os.Stdin)
			return
		}
		exit := 0
		for _, fn := range args {
			f, err := os.Open(fn)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error: %v\n", err)
				exit = 2
			} else {
				defer f.Close()
				io.Copy(os.Stdout, f)
			}
		}
		os.Exit(exit)
	case "pipetest":
		bufr := bufio.NewReader(os.Stdin)
		for {
			line, _, err := bufr.ReadLine()
			if err == io.EOF {
				break
			} else if err != nil {
				os.Exit(1)
			}
			if bytes.HasPrefix(line, []byte("O:")) {
				os.Stdout.Write(line)
				os.Stdout.Write([]byte{'\n'})
			} else if bytes.HasPrefix(line, []byte("E:")) {
				os.Stderr.Write(line)
				os.Stderr.Write([]byte{'\n'})
			} else {
				os.Exit(1)
			}
		}
	case "read3": // read fd 3
		fd3 := os.NewFile(3, "fd3")
		bs, err := ioutil.ReadAll(fd3)
		if err != nil {
			fmt.Printf("ReadAll from fd 3: %v", err)
			os.Exit(1)
		}
		// TODO(bradfitz,iant): the rest of this test is disabled
		// for now. remove this block once we figure out why it fails.
		{
			os.Stderr.Write(bs)
			os.Exit(0)
		}
		// Now verify that there are no other open fds.
		var files []*os.File
		for wantfd := os.Stderr.Fd() + 2; wantfd <= 100; wantfd++ {
			f, err := os.Open(os.Args[0])
			if err != nil {
				fmt.Printf("error opening file with expected fd %d: %v", wantfd, err)
				os.Exit(1)
			}
			if got := f.Fd(); got != wantfd {
				fmt.Printf("leaked parent file. fd = %d; want %d", got, wantfd)
				os.Exit(1)
			}
			files = append(files, f)
		}
		for _, f := range files {
			f.Close()
		}
		os.Stderr.Write(bs)
	case "exit":
		n, _ := strconv.Atoi(args[0])
		os.Exit(n)
	default:
		fmt.Fprintf(os.Stderr, "Unknown command %q\n", cmd)
		os.Exit(2)
	}
}
<|endoftext|>


<commit_before>// +build linux

package main

import (
	"encoding/json"
	"os"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/opencontainers/runc/libcontainer"
)

// event struct for encoding the event data to json.
type event struct {
	Type string      `json:"type"`
	ID   string      `json:"id"`
	Data interface{} `json:"data,omitempty"`
}

var eventsCommand = cli.Command{
	Name:  "events",
	Usage: "display container events such as OOM notifications, cpu, memory, IO and network stats",
	ArgsUsage: `<container-id>

Where "<container-id>" is the name for the instance of the container.`,
	Description: `The events command displays information about the container. By default the
information is displayed once every 5 seconds.`,
	Flags: []cli.Flag{
		cli.DurationFlag{Name: "interval", Value: 5 * time.Second, Usage: "set the stats collection interval"},
		cli.BoolFlag{Name: "stats", Usage: "display the container's stats then exit"},
	},
	Action: func(context *cli.Context) {
		container, err := getContainer(context)
		if err != nil {
			fatal(err)
		}
		status, err := container.Status()
		if err != nil {
			fatal(err)
		}
		if status == libcontainer.Destroyed {
			fatalf("container with id %s is not running", container.ID())
		}
		var (
			stats  = make(chan *libcontainer.Stats, 1)
			events = make(chan *event, 1024)
			group  = &sync.WaitGroup{}
		)
		group.Add(1)
		go func() {
			defer group.Done()
			enc := json.NewEncoder(os.Stdout)
			for e := range events {
				if err := enc.Encode(e); err != nil {
					logrus.Error(err)
				}
			}
		}()
		if context.Bool("stats") {
			s, err := container.Stats()
			if err != nil {
				fatal(err)
			}
			events <- &event{Type: "stats", ID: container.ID(), Data: s}
			close(events)
			group.Wait()
			return
		}
		go func() {
			for range time.Tick(context.Duration("interval")) {
				s, err := container.Stats()
				if err != nil {
					logrus.Error(err)
					continue
				}
				stats <- s
			}
		}()
		n, err := container.NotifyOOM()
		if err != nil {
			fatal(err)
		}
		for {
			select {
			case _, ok := <-n:
				if ok {
					// this means an oom event was received, if it is !ok then
					// the channel was closed because the container stopped and
					// the cgroups no longer exist.
					events <- &event{Type: "oom", ID: container.ID()}
				} else {
					n = nil
				}
			case s := <-stats:
				events <- &event{Type: "stats", ID: container.ID(), Data: s}
			}
			if n == nil {
				close(events)
				break
			}
		}
		group.Wait()
	},
}
<commit_msg>Improve stats output<commit_after>// +build linux

package main

import (
	"encoding/json"
	"os"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/opencontainers/runc/libcontainer"
	"github.com/opencontainers/runc/libcontainer/cgroups"
)

// event struct for encoding the event data to json.
type event struct {
	Type string      `json:"type"`
	ID   string      `json:"id"`
	Data interface{} `json:"data,omitempty"`
}

// stats is the runc specific stats structure for stability when encoding and decoding stats.
type stats struct {
	Cpu     cpu                `json:"cpu"`
	Memory  memory             `json:"memory"`
	Pids    pids               `json:"pids"`
	Blkio   blkio              `json:"blkio"`
	Hugetlb map[string]hugetlb `json:"hugetlb"`
}

type hugetlb struct {
	Usage   uint64 `json:"usage,omitempty"`
	Max     uint64 `json:"max,omitempty"`
	Failcnt uint64 `json:"failcnt"`
}

type blkioEntry struct {
	Major uint64 `json:"major,omitempty"`
	Minor uint64 `json:"minor,omitempty"`
	Op    string `json:"op,omitempty"`
	Value uint64 `json:"value,omitempty"`
}

type blkio struct {
	IoServiceBytesRecursive []blkioEntry `json:"ioServiceBytesRecursive,omitempty"`
	IoServicedRecursive     []blkioEntry `json:"ioServicedRecursive,omitempty"`
	IoQueuedRecursive       []blkioEntry `json:"ioQueueRecursive,omitempty"`
	IoServiceTimeRecursive  []blkioEntry `json:"ioServiceTimeRecursive,omitempty"`
	IoWaitTimeRecursive     []blkioEntry `json:"ioWaitTimeRecursive,omitempty"`
	IoMergedRecursive       []blkioEntry `json:"ioMergedRecursive,omitempty"`
	IoTimeRecursive         []blkioEntry `json:"ioTimeRecursive,omitempty"`
	SectorsRecursive        []blkioEntry `json:"sectorsRecursive,omitempty"`
}

type pids struct {
	Current uint64 `json:"current,omitempty"`
	Limit   uint64 `json:"limit,omitempty"`
}

type throttling struct {
	Periods          uint64 `json:"periods,omitempty"`
	ThrottledPeriods uint64 `json:"throttledPeriods,omitempty"`
	ThrottledTime    uint64 `json:"throttledTime,omitempty"`
}

type cpuUsage struct {
	// Units: nanoseconds.
	Total  uint64   `json:"total,omitempty"`
	Percpu []uint64 `json:"percpu,omitempty"`
	Kernel uint64   `json:"kernel"`
	User   uint64   `json:"user"`
}

type cpu struct {
	Usage      cpuUsage   `json:"usage,omitempty"`
	Throttling throttling `json:"throttling,omitempty"`
}

type memoryEntry struct {
	Limit   uint64 `json:"limit"`
	Usage   uint64 `json:"usage,omitempty"`
	Max     uint64 `json:"max,omitempty"`
	Failcnt uint64 `json:"failcnt"`
}

type memory struct {
	Cache     uint64            `json:"cache,omitempty"`
	Usage     memoryEntry       `json:"usage,omitempty"`
	Swap      memoryEntry       `json:"swap,omitempty"`
	Kernel    memoryEntry       `json:"kernel,omitempty"`
	KernelTCP memoryEntry       `json:"kernelTCP,omitempty"`
	Raw       map[string]uint64 `json:"raw,omitempty"`
}

var eventsCommand = cli.Command{
	Name:  "events",
	Usage: "display container events such as OOM notifications, cpu, memory, and IO usage statistics",
	ArgsUsage: `<container-id>

Where "<container-id>" is the name for the instance of the container.`,
	Description: `The events command displays information about the container. By default the
information is displayed once every 5 seconds.`,
	Flags: []cli.Flag{
		cli.DurationFlag{Name: "interval", Value: 5 * time.Second, Usage: "set the stats collection interval"},
		cli.BoolFlag{Name: "stats", Usage: "display the container's stats then exit"},
	},
	Action: func(context *cli.Context) {
		container, err := getContainer(context)
		if err != nil {
			fatal(err)
		}
		status, err := container.Status()
		if err != nil {
			fatal(err)
		}
		if status == libcontainer.Destroyed {
			fatalf("container with id %s is not running", container.ID())
		}
		var (
			stats  = make(chan *libcontainer.Stats, 1)
			events = make(chan *event, 1024)
			group  = &sync.WaitGroup{}
		)
		group.Add(1)
		go func() {
			defer group.Done()
			enc := json.NewEncoder(os.Stdout)
			for e := range events {
				if err := enc.Encode(e); err != nil {
					logrus.Error(err)
				}
			}
		}()
		if context.Bool("stats") {
			s, err := container.Stats()
			if err != nil {
				fatal(err)
			}
			events <- &event{Type: "stats", ID: container.ID(), Data: convertLibcontainerStats(s)}
			close(events)
			group.Wait()
			return
		}
		go func() {
			for range time.Tick(context.Duration("interval")) {
				s, err := container.Stats()
				if err != nil {
					logrus.Error(err)
					continue
				}
				stats <- s
			}
		}()
		n, err := container.NotifyOOM()
		if err != nil {
			fatal(err)
		}
		for {
			select {
			case _, ok := <-n:
				if ok {
					// this means an oom event was received, if it is !ok then
					// the channel was closed because the container stopped and
					// the cgroups no longer exist.
					events <- &event{Type: "oom", ID: container.ID()}
				} else {
					n = nil
				}
			case s := <-stats:
				events <- &event{Type: "stats", ID: container.ID(), Data: convertLibcontainerStats(s)}
			}
			if n == nil {
				close(events)
				break
			}
		}
		group.Wait()
	},
}

func convertLibcontainerStats(ls *libcontainer.Stats) *stats {
	cg := ls.CgroupStats
	if cg == nil {
		return nil
	}
	var s stats
	s.Pids.Current = cg.PidsStats.Current
	s.Pids.Limit = cg.PidsStats.Limit

	s.Cpu.Usage.Kernel = cg.CpuStats.CpuUsage.UsageInKernelmode
	s.Cpu.Usage.User = cg.CpuStats.CpuUsage.UsageInUsermode
	s.Cpu.Usage.Total = cg.CpuStats.CpuUsage.TotalUsage
	s.Cpu.Usage.Percpu = cg.CpuStats.CpuUsage.PercpuUsage
	s.Cpu.Throttling.Periods = cg.CpuStats.ThrottlingData.Periods
	s.Cpu.Throttling.ThrottledPeriods = cg.CpuStats.ThrottlingData.ThrottledPeriods
	s.Cpu.Throttling.ThrottledTime = cg.CpuStats.ThrottlingData.ThrottledTime

	s.Memory.Cache = cg.MemoryStats.Cache
	s.Memory.Kernel = convertMemoryEntry(cg.MemoryStats.KernelUsage)
	s.Memory.KernelTCP = convertMemoryEntry(cg.MemoryStats.KernelTCPUsage)
	s.Memory.Swap = convertMemoryEntry(cg.MemoryStats.SwapUsage)
	s.Memory.Usage = convertMemoryEntry(cg.MemoryStats.Usage)
	s.Memory.Raw = cg.MemoryStats.Stats

	s.Blkio.IoServiceBytesRecursive = convertBlkioEntry(cg.BlkioStats.IoServiceBytesRecursive)
	s.Blkio.IoServicedRecursive = convertBlkioEntry(cg.BlkioStats.IoServicedRecursive)
	s.Blkio.IoQueuedRecursive = convertBlkioEntry(cg.BlkioStats.IoQueuedRecursive)
	s.Blkio.IoServiceTimeRecursive = convertBlkioEntry(cg.BlkioStats.IoServiceTimeRecursive)
	s.Blkio.IoWaitTimeRecursive = convertBlkioEntry(cg.BlkioStats.IoWaitTimeRecursive)
	s.Blkio.IoMergedRecursive = convertBlkioEntry(cg.BlkioStats.IoMergedRecursive)
	s.Blkio.IoTimeRecursive = convertBlkioEntry(cg.BlkioStats.IoTimeRecursive)
	s.Blkio.SectorsRecursive = convertBlkioEntry(cg.BlkioStats.SectorsRecursive)

	s.Hugetlb = make(map[string]hugetlb)
	for k, v := range cg.HugetlbStats {
		s.Hugetlb[k] = convertHugtlb(v)
	}
	return &s
}

func convertHugtlb(c cgroups.HugetlbStats) hugetlb {
	return hugetlb{
		Usage:   c.Usage,
		Max:     c.MaxUsage,
		Failcnt: c.Failcnt,
	}
}

func convertMemoryEntry(c cgroups.MemoryData) memoryEntry {
	return memoryEntry{
		Limit:   c.Limit,
		Usage:   c.Usage,
		Max:     c.MaxUsage,
		Failcnt: c.Failcnt,
	}
}

func convertBlkioEntry(c []cgroups.BlkioStatEntry) []blkioEntry {
	var out []blkioEntry
	for _, e := range c {
		out = append(out, blkioEntry{
			Major: e.Major,
			Minor: e.Minor,
			Op:    e.Op,
			Value: e.Value,
		})
	}
	return out
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync_test

import (
	"fmt"
	"net/http"
	"sync"
)

// This example fetches several URLs concurrently,
// using a WaitGroup to block until all the fetches are complete.
func ExampleWaitGroup() {
	var wg sync.WaitGroup
	var urls = []string{
		"http://www.golang.org/",
		"http://www.google.com/",
		"http://www.somestupidname.com/",
	}
	for _, url := range urls {
		// Increment the WaitGroup counter.
		wg.Add(1)
		// Launch a goroutine to fetch the URL.
		go func(url string) {
			// Decrement the counter when the goroutine completes.
			defer wg.Done()
			// Fetch the URL.
			http.Get(url)
		}(url)
	}
	// Wait for all HTTP fetches to complete.
	wg.Wait()
}

func ExampleOnce() {
	var once sync.Once
	onceBody := func() {
		fmt.Println("Only once")
	}
	done := make(chan bool)
	for i := 0; i < 10; i++ {
		go func() {
			once.Do(onceBody)
			done <- true
		}()
	}
	for i := 0; i < 10; i++ {
		<-done
	}
	// Output:
	// Only once
}
<commit_msg>sync/atomic: remove test dependency on net/http<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync_test

import (
	"fmt"
	"sync"
)

type httpPkg struct{}

func (httpPkg) Get(url string) {}

var http httpPkg

// This example fetches several URLs concurrently,
// using a WaitGroup to block until all the fetches are complete.
func ExampleWaitGroup() {
	var wg sync.WaitGroup
	var urls = []string{
		"http://www.golang.org/",
		"http://www.google.com/",
		"http://www.somestupidname.com/",
	}
	for _, url := range urls {
		// Increment the WaitGroup counter.
		wg.Add(1)
		// Launch a goroutine to fetch the URL.
		go func(url string) {
			// Decrement the counter when the goroutine completes.
			defer wg.Done()
			// Fetch the URL.
			http.Get(url)
		}(url)
	}
	// Wait for all HTTP fetches to complete.
	wg.Wait()
}

func ExampleOnce() {
	var once sync.Once
	onceBody := func() {
		fmt.Println("Only once")
	}
	done := make(chan bool)
	for i := 0; i < 10; i++ {
		go func() {
			once.Do(onceBody)
			done <- true
		}()
	}
	for i := 0; i < 10; i++ {
		<-done
	}
	// Output:
	// Only once
}
<|endoftext|>


<commit_before>// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package stat

import (
	"github.com/gonum/matrix/mat64"
)

// CovarianceMatrix calculates a covariance matrix (also known as a
// variance-covariance matrix) from a matrix of data.
func CovarianceMatrix(x mat64.Matrix) *mat64.Dense {

	// matrix version of the two pass algorithm.  This doesn't use
	// the correction found in the Covariance and Variance functions.

	r, _ := x.Dims()

	// determine the mean of each of the columns
	b := ones(1, r)
	b.Mul(b, x)
	b.Scale(1/float64(r), b)
	mu := b.RowView(0)

	// subtract the mean from the data
	xc := mat64.DenseCopyOf(x)
	for i := 0; i < r; i++ {
		rv := xc.RowView(i)
		for j, mean := range mu {
			rv[j] -= mean
		}
	}

	// todo: avoid matrix copy?
	var xt mat64.Dense
	xt.TCopy(xc)

	// It would be nice if we could indicate that this was a symmetric
	// matrix.
	var ss mat64.Dense
	ss.Mul(&xt, xc)
	ss.Scale(1/float64(r-1), &ss)
	return &ss
}

// ones is a matrix of all ones.
func ones(r, c int) *mat64.Dense {
	x := make([]float64, r*c)
	for i := range x {
		x[i] = 1
	}
	return mat64.NewDense(r, c, x)
}
<commit_msg>example implementation of pure go covariancematrix<commit_after>// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package stat

import (
	"sync"
//	"runtime"

	"github.com/gonum/matrix/mat64"
)

type covMatElem struct {
	i, j int
	v    float64
}

type covMatSlice struct {
	i, j int
	x, y []float64
}

// CovarianceMatrix calculates a covariance matrix (also known as a
// variance-covariance matrix) from a matrix of data.
func CovarianceMatrix(x mat64.Matrix) *mat64.Dense {

	// matrix version of the two pass algorithm.  This doesn't use
	// the correction found in the Covariance and Variance functions.
	r, c := x.Dims()

	if x, ok := x.(mat64.Vectorer); ok {
		cols := make([][]float64, c)
		// perform the covariance or variance as required
		blockSize := 1024
		if blockSize > c {
			blockSize = c
		}
		var wg sync.WaitGroup
		wg.Add(c)
		for j := 0; j < c; j++ {
			go func(j int) {
				// pull the columns out and subtract the means
				cols[j] = make([]float64, r)
				x.Col(cols[j], j)
				mean := Mean(cols[j], nil)
				for i := range cols[j] {
					cols[j][i] -= mean
				}
				wg.Done()
			}(j)
		}
		wg.Wait()

		colCh := make(chan covMatSlice, blockSize)
		resCh := make(chan covMatElem, blockSize)

		wg.Add(blockSize)
		go func() {
			wg.Wait()
			close(resCh)
		}()

		for i := 0; i < blockSize; i++ {
			go func(in <-chan covMatSlice, out chan<- covMatElem) {
				for {
					xy, more := <-in
					if !more {
						wg.Done()
						return
					}

					if xy.i == xy.j {
						out <- covMatElem{
							i: xy.i,
							j: xy.j,
							v: centeredVariance(xy.x),
						}
						continue
					}
					out <- covMatElem{
						i: xy.i,
						j: xy.j,
						v: centeredCovariance(xy.x, xy.y),
					}
				}
			}(colCh, resCh)
		}
		go func(out chan<- covMatSlice) {
			for i := 0; i < c; i++ {
				for j := 0; j <= i; j++ {
					out <- covMatSlice{
						i: i,
						j: j,
						x: cols[i],
						y: cols[j],
					}
				}
			}
			close(out)
		}(colCh)
		// create the output matrix
		m := mat64.NewDense(c, c, nil)
		for {
			c, more := <-resCh
			if !more {
				return m
			}
			m.Set(c.i, c.j, c.v)
			if c.i != c.j {
				m.Set(c.j, c.i, c.v)
			}
		}
	}
	// determine the mean of each of the columns
	b := ones(1, r)
	b.Mul(b, x)
	b.Scale(1/float64(r), b)
	mu := b.RowView(0)

	// subtract the mean from the data
	xc := mat64.DenseCopyOf(x)
	for i := 0; i < r; i++ {
		rv := xc.RowView(i)
		for j, mean := range mu {
			rv[j] -= mean
		}
	}

	// todo: avoid matrix copy?
	var xt mat64.Dense
	xt.TCopy(xc)

	// It would be nice if we could indicate that this was a symmetric
	// matrix.
	var ss mat64.Dense
	ss.Mul(&xt, xc)
	ss.Scale(1/float64(r-1), &ss)
	return &ss
}

// ones is a matrix of all ones.
func ones(r, c int) *mat64.Dense {
	x := make([]float64, r*c)
	for i := range x {
		x[i] = 1
	}
	return mat64.NewDense(r, c, x)
}

func centeredVariance(x []float64) float64 {
	var ss float64
	for _, xv := range x {
		ss += xv * xv
	}
	return ss / float64(len(x)-1)
}

func centeredCovariance(x, y []float64) float64 {
	var ss float64
	for i, xv := range x {
		ss += xv * y[i]
	}
	return ss / float64(len(x)-1)
}
<|endoftext|>


<commit_before>package crawler

import (
	"bytes"
	"errors"
	"io/ioutil"
	"net/http"
	neturl "net/url"
	"regexp"
	"strings"

	"github.com/daohoangson/go-sitemirror/cacher"
	cssScanner "github.com/gorilla/css/scanner"
	"golang.org/x/net/html"
	htmlAtom "golang.org/x/net/html/atom"
)

var (
	cssURIRegexp = regexp.MustCompile(`^(url\(['"]?)([^'"]+)(['"]?\))$`)
)

const (
	htmlAttrAction        = "action"
	htmlAttrHref          = "href"
	htmlAttrRel           = "rel"
	htmlAttrRelStylesheet = "stylesheet"
	htmlAttrSrc           = "src"
)

// Download returns parsed data after downloading the specified url.
func Download(input *Input) *Downloaded {
	result := &Downloaded{
		Input: input,

		BaseURL:         input.URL,
		LinksAssets:     make(map[string]Link),
		LinksDiscovered: make(map[string]Link),
	}

	if input.Client == nil {
		result.Error = errors.New(".Client cannot be nil")
		return result
	}

	if input.URL == nil {
		result.Error = errors.New(".URL cannot be nil")
		return result
	}

	if !input.URL.IsAbs() {
		result.Error = errors.New(".URL must be absolute")
		return result
	}

	if !strings.HasPrefix(input.URL.Scheme, cacher.SchemeDefault) {
		result.Error = errors.New(".URL.Scheme must be http/https")
		return result
	}

	httpClient := *input.Client
	// http://stackoverflow.com/questions/23297520/how-can-i-make-the-go-http-client-not-follow-redirects-automatically
	httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		// do not follow redirects
		return http.ErrUseLastResponse
	}

	req, err := http.NewRequest("GET", input.URL.String(), nil)
	if err != nil {
		result.Error = err
		return result
	}

	if input.Header != nil {
		for headerKey, headerValues := range input.Header {
			for _, headerValue := range headerValues {
				req.Header.Add(headerKey, headerValue)
			}
		}
	}

	resp, err := httpClient.Do(req)
	if err != nil {
		result.Error = err
		return result
	}
	defer resp.Body.Close()

	result.StatusCode = resp.StatusCode
	if result.StatusCode >= 200 && result.StatusCode <= 299 {
		result.Error = parseBody(resp, result)
	} else if result.StatusCode >= 300 && result.StatusCode <= 399 {
		result.Error = parseRedirect(resp, result)
	}

	return result
}

func parseBody(resp *http.Response, result *Downloaded) error {
	respHeaderCacheControl := resp.Header.Get(cacher.HeaderCacheControl)
	if len(respHeaderCacheControl) > 0 {
		result.AddHeader(cacher.HeaderCacheControl, respHeaderCacheControl)
	}

	respHeaderExpires := resp.Header.Get(cacher.HeaderExpires)
	if len(respHeaderExpires) > 0 {
		result.AddHeader(cacher.HeaderExpires, respHeaderExpires)
	}

	respHeaderContentType := resp.Header.Get(cacher.HeaderContentType)
	if len(respHeaderContentType) > 0 {
		result.AddHeader(cacher.HeaderContentType, respHeaderContentType)

		parts := strings.Split(respHeaderContentType, ";")
		contentType := parts[0]

		switch contentType {
		case "text/css":
			return parseBodyCSS(resp, result)
		case "text/html":
			return parseBodyHTML(resp, result)
		}
	}

	return parseBodyRaw(resp, result)
}

func parseBodyCSS(resp *http.Response, result *Downloaded) error {
	body, _ := ioutil.ReadAll(resp.Body)

	var buffer bytes.Buffer
	defer buffer.Reset()
	result.buffer = &buffer

	err := parseBodyCSSString(string(body), result)

	result.Body = buffer.String()
	result.buffer = nil

	return err
}

func parseBodyCSSString(css string, result *Downloaded) error {
	scanner := cssScanner.New(css)
	for {
		token := scanner.Next()
		if token.Type == cssScanner.TokenEOF || token.Type == cssScanner.TokenError {
			break
		}

		if token.Type == cssScanner.TokenURI {
			if m := cssURIRegexp.FindStringSubmatch(token.Value); m != nil {
				before, url, after := m[1], m[2], m[3]
				processedURL, err := result.ProcessURL(CSSUri, url)
				if err == nil && processedURL != url {
					result.buffer.WriteString(before)
					result.buffer.WriteString(processedURL)
					result.buffer.WriteString(after)
					continue
				}
			}
		}

		result.buffer.WriteString(token.Value)
	}

	return nil
}

func parseBodyHTML(resp *http.Response, result *Downloaded) error {
	var buffer bytes.Buffer
	defer buffer.Reset()
	result.buffer = &buffer

	tokenizer := html.NewTokenizer(resp.Body)
	for {
		if parseBodyHTMLToken(tokenizer, result) {
			break
		}
	}

	result.Body = buffer.String()
	result.buffer = nil

	return nil
}

func parseBodyHTMLToken(tokenizer *html.Tokenizer, result *Downloaded) bool {
	tokenType := tokenizer.Next()
	if tokenType == html.ErrorToken {
		return true
	}

	switch tokenType {
	case html.StartTagToken:
		token := tokenizer.Token()

		switch token.DataAtom {
		case htmlAtom.A:
			if parseBodyHTMLTagA(&token, result) {
				return false
			}
		case htmlAtom.Form:
			if parseBodyHTMLTagForm(&token, result) {
				return false
			}
		case htmlAtom.Img:
			if parseBodyHTMLTagImg(&token, result) {
				return false
			}
		case htmlAtom.Link:
			if parseBodyHTMLTagLink(&token, result) {
				return false
			}
		case htmlAtom.Script:
			if parseBodyHTMLTagScript(tokenizer, &token, result) {
				return false
			}
		case htmlAtom.Style:
			if parseBodyHTMLTagStyle(tokenizer, result) {
				return false
			}
		}

		rewriteTokenAttr(&token, result)
		return false
	case html.SelfClosingTagToken:
		token := tokenizer.Token()

		switch token.DataAtom {
		case htmlAtom.Base:
			if parseBodyHTMLTagBase(&token, result) {
				return false
			}
		case htmlAtom.Img:
			if parseBodyHTMLTagImg(&token, result) {
				return false
			}
		case htmlAtom.Link:
			if parseBodyHTMLTagLink(&token, result) {
				return false
			}
		}

		rewriteTokenAttr(&token, result)
		return false
	}

	result.buffer.Write(tokenizer.Raw())
	return false
}

func parseBodyHTMLTagA(token *html.Token, result *Downloaded) bool {
	for i, attr := range token.Attr {
		if attr.Key == htmlAttrHref {
			processedURL, err := result.ProcessURL(HTMLTagA, attr.Val)
			if err == nil && processedURL != attr.Val {
				token.Attr[i].Val = processedURL
				return rewriteTokenAttr(token, result)
			}
		}
	}

	return false
}

func parseBodyHTMLTagForm(token *html.Token, result *Downloaded) bool {
	for i, attr := range token.Attr {
		if attr.Key == htmlAttrAction {
			processedURL, err := result.ProcessURL(HTMLTagForm, attr.Val)
			if err == nil && processedURL != attr.Val {
				token.Attr[i].Val = processedURL
				return rewriteTokenAttr(token, result)
			}
		}
	}

	return false
}

func parseBodyHTMLTagBase(token *html.Token, result *Downloaded) bool {
	for _, attr := range token.Attr {
		if attr.Key == htmlAttrHref {
			if url, err := neturl.Parse(attr.Val); err == nil {
				result.BaseURL = result.BaseURL.ResolveReference(url)
				return true
			}
		}
	}

	return false
}

func parseBodyHTMLTagImg(token *html.Token, result *Downloaded) bool {
	needRewrite := false

	for i, attr := range token.Attr {
		if attr.Key != htmlAttrSrc && !strings.HasPrefix(attr.Key, "data-") {
			// process src attribute and any data-* attribute that contains url
			// some website uses those for lazy loading / high resolution quality / etc.
			continue
		}

		url := attr.Val
		parsedURL, err := neturl.Parse(url)
		if err != nil {
			continue
		}

		if attr.Key != htmlAttrSrc && !parsedURL.IsAbs() {
			// data-* attribute url must be absolute
			continue
		}

		processedURL, err := result.ProcessURL(HTMLTagImg, url)
		if err == nil && processedURL != attr.Val {
			token.Attr[i].Val = processedURL
			needRewrite = true
		}
	}

	if needRewrite {
		return rewriteTokenAttr(token, result)
	}

	return false
}

func parseBodyHTMLTagLink(token *html.Token, result *Downloaded) bool {
	var linkHref string
	var linkHrefAttrIndex int
	var linkRel string

	for i, attr := range token.Attr {
		switch attr.Key {
		case htmlAttrHref:
			linkHref = attr.Val
			linkHrefAttrIndex = i
		case htmlAttrRel:
			linkRel = attr.Val
		}
	}

	if len(linkHref) > 0 {
		switch linkRel {
		case htmlAttrRelStylesheet:
			processedURL, err := result.ProcessURL(HTMLTagLinkStylesheet, linkHref)
			if err == nil && processedURL != linkHref {
				token.Attr[linkHrefAttrIndex].Val = processedURL
				return rewriteTokenAttr(token, result)
			}
		}
	}

	return false
}

func parseBodyHTMLTagScript(tokenizer *html.Tokenizer, token *html.Token, result *Downloaded) bool {
	for i, attr := range token.Attr {
		if attr.Key == htmlAttrSrc {
			processedURL, err := result.ProcessURL(HTMLTagScript, attr.Val)
			if err == nil && processedURL != attr.Val {
				token.Attr[i].Val = processedURL
				return rewriteTokenAttr(token, result)
			}
		}
	}

	// handle inline script
	result.buffer.Write(tokenizer.Raw())
	for {
		tokenType := tokenizer.Next()
		raw := tokenizer.Raw()

		switch tokenType {
		case html.EndTagToken:
			result.buffer.Write(raw)
			return true
		case html.TextToken:
			parseBodyJsString(string(raw), result)
		}
	}
}

func parseBodyHTMLTagStyle(tokenizer *html.Tokenizer, result *Downloaded) bool {
	result.buffer.Write(tokenizer.Raw())

	for {
		tokenType := tokenizer.Next()
		raw := tokenizer.Raw()

		switch tokenType {
		case html.EndTagToken:
			result.buffer.Write(raw)
			return true
		case html.TextToken:
			parseBodyCSSString(string(raw), result)
		}
	}
}

func parseBodyJsString(js string, result *Downloaded) {
	if strings.Index(js, "getElementsByTagName('base')") > -1 {
		// skip inline js that deals with <base />
		return
	}

	result.buffer.WriteString(js)
}

func rewriteTokenAttr(token *html.Token, result *Downloaded) bool {
	result.buffer.WriteString("<")
	result.buffer.WriteString(token.Data)

	for _, attr := range token.Attr {
		result.buffer.WriteString(" ")
		result.buffer.WriteString(attr.Key)
		result.buffer.WriteString("=\"")

		if attr.Key == "style" {
			parseBodyCSSString(attr.Val, result)
		} else {
			result.buffer.WriteString(html.EscapeString(attr.Val))
		}

		result.buffer.WriteString("\"")
	}

	if token.Type == html.SelfClosingTagToken {
		result.buffer.WriteString(" /")
	}

	result.buffer.WriteString(">")

	return true
}

func parseBodyRaw(resp *http.Response, result *Downloaded) error {
	body, err := ioutil.ReadAll(resp.Body)
	result.Body = string(body)
	return err
}

func parseRedirect(resp *http.Response, result *Downloaded) error {
	location := resp.Header.Get(cacher.HeaderLocation)
	processedURL, err := result.ProcessURL(HTTP3xxLocation, location)
	if err == nil {
		result.AddHeader(cacher.HeaderLocation, processedURL)
	}

	return err
}
<commit_msg>Simplify crawler parseBodyHTMLToken func<commit_after>package crawler

import (
	"bytes"
	"errors"
	"io/ioutil"
	"net/http"
	neturl "net/url"
	"regexp"
	"strings"

	"github.com/daohoangson/go-sitemirror/cacher"
	cssScanner "github.com/gorilla/css/scanner"
	"golang.org/x/net/html"
	htmlAtom "golang.org/x/net/html/atom"
)

var (
	cssURIRegexp = regexp.MustCompile(`^(url\(['"]?)([^'"]+)(['"]?\))$`)
)

const (
	htmlAttrAction        = "action"
	htmlAttrHref          = "href"
	htmlAttrRel           = "rel"
	htmlAttrRelStylesheet = "stylesheet"
	htmlAttrSrc           = "src"
)

// Download returns parsed data after downloading the specified url.
func Download(input *Input) *Downloaded {
	result := &Downloaded{
		Input: input,

		BaseURL:         input.URL,
		LinksAssets:     make(map[string]Link),
		LinksDiscovered: make(map[string]Link),
	}

	if input.Client == nil {
		result.Error = errors.New(".Client cannot be nil")
		return result
	}

	if input.URL == nil {
		result.Error = errors.New(".URL cannot be nil")
		return result
	}

	if !input.URL.IsAbs() {
		result.Error = errors.New(".URL must be absolute")
		return result
	}

	if !strings.HasPrefix(input.URL.Scheme, cacher.SchemeDefault) {
		result.Error = errors.New(".URL.Scheme must be http/https")
		return result
	}

	httpClient := *input.Client
	// http://stackoverflow.com/questions/23297520/how-can-i-make-the-go-http-client-not-follow-redirects-automatically
	httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		// do not follow redirects
		return http.ErrUseLastResponse
	}

	req, err := http.NewRequest("GET", input.URL.String(), nil)
	if err != nil {
		result.Error = err
		return result
	}

	if input.Header != nil {
		for headerKey, headerValues := range input.Header {
			for _, headerValue := range headerValues {
				req.Header.Add(headerKey, headerValue)
			}
		}
	}

	resp, err := httpClient.Do(req)
	if err != nil {
		result.Error = err
		return result
	}
	defer resp.Body.Close()

	result.StatusCode = resp.StatusCode
	if result.StatusCode >= 200 && result.StatusCode <= 299 {
		result.Error = parseBody(resp, result)
	} else if result.StatusCode >= 300 && result.StatusCode <= 399 {
		result.Error = parseRedirect(resp, result)
	}

	return result
}

func parseBody(resp *http.Response, result *Downloaded) error {
	respHeaderCacheControl := resp.Header.Get(cacher.HeaderCacheControl)
	if len(respHeaderCacheControl) > 0 {
		result.AddHeader(cacher.HeaderCacheControl, respHeaderCacheControl)
	}

	respHeaderExpires := resp.Header.Get(cacher.HeaderExpires)
	if len(respHeaderExpires) > 0 {
		result.AddHeader(cacher.HeaderExpires, respHeaderExpires)
	}

	respHeaderContentType := resp.Header.Get(cacher.HeaderContentType)
	if len(respHeaderContentType) > 0 {
		result.AddHeader(cacher.HeaderContentType, respHeaderContentType)

		parts := strings.Split(respHeaderContentType, ";")
		contentType := parts[0]

		switch contentType {
		case "text/css":
			return parseBodyCSS(resp, result)
		case "text/html":
			return parseBodyHTML(resp, result)
		}
	}

	return parseBodyRaw(resp, result)
}

func parseBodyCSS(resp *http.Response, result *Downloaded) error {
	body, _ := ioutil.ReadAll(resp.Body)

	var buffer bytes.Buffer
	defer buffer.Reset()
	result.buffer = &buffer

	err := parseBodyCSSString(string(body), result)

	result.Body = buffer.String()
	result.buffer = nil

	return err
}

func parseBodyCSSString(css string, result *Downloaded) error {
	scanner := cssScanner.New(css)
	for {
		token := scanner.Next()
		if token.Type == cssScanner.TokenEOF || token.Type == cssScanner.TokenError {
			break
		}

		if token.Type == cssScanner.TokenURI {
			if m := cssURIRegexp.FindStringSubmatch(token.Value); m != nil {
				before, url, after := m[1], m[2], m[3]
				processedURL, err := result.ProcessURL(CSSUri, url)
				if err == nil && processedURL != url {
					result.buffer.WriteString(before)
					result.buffer.WriteString(processedURL)
					result.buffer.WriteString(after)
					continue
				}
			}
		}

		result.buffer.WriteString(token.Value)
	}

	return nil
}

func parseBodyHTML(resp *http.Response, result *Downloaded) error {
	var buffer bytes.Buffer
	defer buffer.Reset()
	result.buffer = &buffer

	tokenizer := html.NewTokenizer(resp.Body)
	for {
		if parseBodyHTMLToken(tokenizer, result) {
			break
		}
	}

	result.Body = buffer.String()
	result.buffer = nil

	return nil
}

func parseBodyHTMLToken(tokenizer *html.Tokenizer, result *Downloaded) bool {
	tokenType := tokenizer.Next()
	if tokenType == html.ErrorToken {
		return true
	}

	done := false
	switch tokenType {
	case html.StartTagToken:
		token := tokenizer.Token()

		switch token.DataAtom {
		case htmlAtom.A:
			done = parseBodyHTMLTagA(&token, result)
		case htmlAtom.Form:
			done = parseBodyHTMLTagForm(&token, result)
		case htmlAtom.Img:
			done = parseBodyHTMLTagImg(&token, result)
		case htmlAtom.Link:
			done = parseBodyHTMLTagLink(&token, result)
		case htmlAtom.Script:
			done = parseBodyHTMLTagScript(tokenizer, &token, result)
		case htmlAtom.Style:
			done = parseBodyHTMLTagStyle(tokenizer, result)
		}

		if !done {
			done = rewriteTokenAttr(&token, result)
		}
	case html.SelfClosingTagToken:
		token := tokenizer.Token()

		switch token.DataAtom {
		case htmlAtom.Base:
			done = parseBodyHTMLTagBase(&token, result)
		case htmlAtom.Img:
			done = parseBodyHTMLTagImg(&token, result)
		case htmlAtom.Link:
			done = parseBodyHTMLTagLink(&token, result)
		}

		if !done {
			done = rewriteTokenAttr(&token, result)
		}
	}

	if !done {
		result.buffer.Write(tokenizer.Raw())
	}

	return false
}

func parseBodyHTMLTagA(token *html.Token, result *Downloaded) bool {
	for i, attr := range token.Attr {
		if attr.Key == htmlAttrHref {
			processedURL, err := result.ProcessURL(HTMLTagA, attr.Val)
			if err == nil && processedURL != attr.Val {
				token.Attr[i].Val = processedURL
				return rewriteTokenAttr(token, result)
			}
		}
	}

	return false
}

func parseBodyHTMLTagForm(token *html.Token, result *Downloaded) bool {
	for i, attr := range token.Attr {
		if attr.Key == htmlAttrAction {
			processedURL, err := result.ProcessURL(HTMLTagForm, attr.Val)
			if err == nil && processedURL != attr.Val {
				token.Attr[i].Val = processedURL
				return rewriteTokenAttr(token, result)
			}
		}
	}

	return false
}

func parseBodyHTMLTagBase(token *html.Token, result *Downloaded) bool {
	for _, attr := range token.Attr {
		if attr.Key == htmlAttrHref {
			if url, err := neturl.Parse(attr.Val); err == nil {
				result.BaseURL = result.BaseURL.ResolveReference(url)
				return true
			}
		}
	}

	return false
}

func parseBodyHTMLTagImg(token *html.Token, result *Downloaded) bool {
	needRewrite := false

	for i, attr := range token.Attr {
		if attr.Key != htmlAttrSrc && !strings.HasPrefix(attr.Key, "data-") {
			// process src attribute and any data-* attribute that contains url
			// some website uses those for lazy loading / high resolution quality / etc.
			continue
		}

		url := attr.Val
		parsedURL, err := neturl.Parse(url)
		if err != nil {
			continue
		}

		if attr.Key != htmlAttrSrc && !parsedURL.IsAbs() {
			// data-* attribute url must be absolute
			continue
		}

		processedURL, err := result.ProcessURL(HTMLTagImg, url)
		if err == nil && processedURL != attr.Val {
			token.Attr[i].Val = processedURL
			needRewrite = true
		}
	}

	if needRewrite {
		return rewriteTokenAttr(token, result)
	}

	return false
}

func parseBodyHTMLTagLink(token *html.Token, result *Downloaded) bool {
	var linkHref string
	var linkHrefAttrIndex int
	var linkRel string

	for i, attr := range token.Attr {
		switch attr.Key {
		case htmlAttrHref:
			linkHref = attr.Val
			linkHrefAttrIndex = i
		case htmlAttrRel:
			linkRel = attr.Val
		}
	}

	if len(linkHref) > 0 {
		switch linkRel {
		case htmlAttrRelStylesheet:
			processedURL, err := result.ProcessURL(HTMLTagLinkStylesheet, linkHref)
			if err == nil && processedURL != linkHref {
				token.Attr[linkHrefAttrIndex].Val = processedURL
				return rewriteTokenAttr(token, result)
			}
		}
	}

	return false
}

func parseBodyHTMLTagScript(tokenizer *html.Tokenizer, token *html.Token, result *Downloaded) bool {
	for i, attr := range token.Attr {
		if attr.Key == htmlAttrSrc {
			processedURL, err := result.ProcessURL(HTMLTagScript, attr.Val)
			if err == nil && processedURL != attr.Val {
				token.Attr[i].Val = processedURL
				return rewriteTokenAttr(token, result)
			}
		}
	}

	// handle inline script
	result.buffer.Write(tokenizer.Raw())
	for {
		tokenType := tokenizer.Next()
		raw := tokenizer.Raw()

		switch tokenType {
		case html.EndTagToken:
			result.buffer.Write(raw)
			return true
		case html.TextToken:
			parseBodyJsString(string(raw), result)
		}
	}
}

func parseBodyHTMLTagStyle(tokenizer *html.Tokenizer, result *Downloaded) bool {
	result.buffer.Write(tokenizer.Raw())

	for {
		tokenType := tokenizer.Next()
		raw := tokenizer.Raw()

		switch tokenType {
		case html.EndTagToken:
			result.buffer.Write(raw)
			return true
		case html.TextToken:
			parseBodyCSSString(string(raw), result)
		}
	}
}

func parseBodyJsString(js string, result *Downloaded) {
	if strings.Index(js, "getElementsByTagName('base')") > -1 {
		// skip inline js that deals with <base />
		return
	}

	result.buffer.WriteString(js)
}

func rewriteTokenAttr(token *html.Token, result *Downloaded) bool {
	result.buffer.WriteString("<")
	result.buffer.WriteString(token.Data)

	for _, attr := range token.Attr {
		result.buffer.WriteString(" ")
		result.buffer.WriteString(attr.Key)
		result.buffer.WriteString("=\"")

		if attr.Key == "style" {
			parseBodyCSSString(attr.Val, result)
		} else {
			result.buffer.WriteString(html.EscapeString(attr.Val))
		}

		result.buffer.WriteString("\"")
	}

	if token.Type == html.SelfClosingTagToken {
		result.buffer.WriteString(" /")
	}

	result.buffer.WriteString(">")

	return true
}

func parseBodyRaw(resp *http.Response, result *Downloaded) error {
	body, err := ioutil.ReadAll(resp.Body)
	result.Body = string(body)
	return err
}

func parseRedirect(resp *http.Response, result *Downloaded) error {
	location := resp.Header.Get(cacher.HeaderLocation)
	processedURL, err := result.ProcessURL(HTTP3xxLocation, location)
	if err == nil {
		result.AddHeader(cacher.HeaderLocation, processedURL)
	}

	return err
}
<|endoftext|>


<commit_before>package client

import (
	"net/http"
	"os"

	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"

	"bosh-google-cpi/google/config"

	"context"
	"crypto/tls"
	"golang.org/x/oauth2"
	oauthgoogle "golang.org/x/oauth2/google"
	computebeta "google.golang.org/api/compute/v0.beta"
	"google.golang.org/api/compute/v1"
	"google.golang.org/api/storage/v1"
)

const (
	computeScope = compute.ComputeScope
	storageScope = storage.DevstorageFullControlScope
	// Metadata Host needs to be IP address, rather than FQDN, in case the system
	// is set up to use public DNS servers, which would not resolve correctly.
	metadataHost = "169.254.169.254"
)

type GoogleClient struct {
	config          config.Config
	computeService  *compute.Service
	computeServiceB *computebeta.Service
	storageService  *storage.Service
	logger          boshlog.Logger
}

func NewGoogleClient(
	config config.Config,
	logger boshlog.Logger,
) (GoogleClient, error) {
	var err error
	var computeClient, storageClient *http.Client
	userAgent := config.GetUserAgent()

	if config.JSONKey != "" {
		computeJwtConf, err := oauthgoogle.JWTConfigFromJSON([]byte(config.JSONKey), computeScope)
		if err != nil {
			return GoogleClient{}, bosherr.WrapError(err, "Reading Google JSON Key")
		}
		computeClient = computeJwtConf.Client(oauth2.NoContext)

		storageJwtConf, err := oauthgoogle.JWTConfigFromJSON([]byte(config.JSONKey), storageScope)
		if err != nil {
			return GoogleClient{}, bosherr.WrapError(err, "Reading Google JSON Key")
		}
		storageClient = storageJwtConf.Client(oauth2.NoContext)
	} else {
		if v := os.Getenv("GCE_METADATA_HOST"); v == "" {
			os.Setenv("GCE_METADATA_HOST", metadataHost)
		}
		computeClient, err = oauthgoogle.DefaultClient(oauth2.NoContext, computeScope)
		if err != nil {
			return GoogleClient{}, bosherr.WrapError(err, "Creating a Google default client")
		}

		storageClient, err = oauthgoogle.DefaultClient(oauth2.NoContext, storageScope)
		if err != nil {
			return GoogleClient{}, bosherr.WrapError(err, "Creating a Google default client")
		}
	}

	// Custom RoundTripper for retries
	computeRetrier := &RetryTransport{
		Base:       computeClient.Transport,
		MaxRetries: 3,
	}
	computeClient.Transport = computeRetrier
	computeService, err := compute.New(computeClient)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Creating a Google Compute Service client")
	}
	computeService.UserAgent = userAgent

	computeServiceB, err := computebeta.New(computeClient)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Creating a Google Compute Service client")
	}
	computeServiceB.UserAgent = userAgent

	// Custom RoundTripper for retries
	storageRetrier := &RetryTransport{
		Base:       storageClient.Transport,
		MaxRetries: 3,
	}
	storageClient.Transport = storageRetrier
	storageService, err := storage.New(storageClient)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Creating a Google Storage Service client")
	}
	storageService.UserAgent = userAgent

	return GoogleClient{
		config:          config,
		computeService:  computeService,
		computeServiceB: computeServiceB,
		storageService:  storageService,
		logger:          logger,
	}, nil
}

func (c GoogleClient) Project() string {
	return c.config.Project
}

func (c GoogleClient) DefaultRootDiskSizeGb() int {
	return c.config.DefaultRootDiskSizeGb
}

func (c GoogleClient) DefaultRootDiskType() string {
	return c.config.DefaultRootDiskType
}

func (c GoogleClient) ComputeService() *compute.Service {
	return c.computeService
}

func (c GoogleClient) ComputeBetaService() *computebeta.Service {
	return c.computeServiceB
}

func (c GoogleClient) StorageService() *storage.Service {
	return c.storageService
}
<commit_msg>workaround: use vendored context package<commit_after>package client

import (
	"net/http"
	"os"

	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"

	"bosh-google-cpi/google/config"

	"crypto/tls"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	oauthgoogle "golang.org/x/oauth2/google"
	computebeta "google.golang.org/api/compute/v0.beta"
	"google.golang.org/api/compute/v1"
	"google.golang.org/api/storage/v1"
)

const (
	computeScope = compute.ComputeScope
	storageScope = storage.DevstorageFullControlScope
	// Metadata Host needs to be IP address, rather than FQDN, in case the system
	// is set up to use public DNS servers, which would not resolve correctly.
	metadataHost = "169.254.169.254"
)

type GoogleClient struct {
	config          config.Config
	computeService  *compute.Service
	computeServiceB *computebeta.Service
	storageService  *storage.Service
	logger          boshlog.Logger
}

func NewGoogleClient(
	config config.Config,
	logger boshlog.Logger,
) (GoogleClient, error) {
	var err error
	var computeClient, storageClient *http.Client
	userAgent := config.GetUserAgent()

	if config.JSONKey != "" {
		computeJwtConf, err := oauthgoogle.JWTConfigFromJSON([]byte(config.JSONKey), computeScope)
		if err != nil {
			return GoogleClient{}, bosherr.WrapError(err, "Reading Google JSON Key")
		}
		computeClient = computeJwtConf.Client(oauth2.NoContext)

		storageJwtConf, err := oauthgoogle.JWTConfigFromJSON([]byte(config.JSONKey), storageScope)
		if err != nil {
			return GoogleClient{}, bosherr.WrapError(err, "Reading Google JSON Key")
		}
		storageClient = storageJwtConf.Client(oauth2.NoContext)
	} else {
		if v := os.Getenv("GCE_METADATA_HOST"); v == "" {
			os.Setenv("GCE_METADATA_HOST", metadataHost)
		}
		computeClient, err = oauthgoogle.DefaultClient(oauth2.NoContext, computeScope)
		if err != nil {
			return GoogleClient{}, bosherr.WrapError(err, "Creating a Google default client")
		}

		storageClient, err = oauthgoogle.DefaultClient(oauth2.NoContext, storageScope)
		if err != nil {
			return GoogleClient{}, bosherr.WrapError(err, "Creating a Google default client")
		}
	}

	// Custom RoundTripper for retries
	computeRetrier := &RetryTransport{
		Base:       computeClient.Transport,
		MaxRetries: 3,
	}
	computeClient.Transport = computeRetrier
	computeService, err := compute.New(computeClient)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Creating a Google Compute Service client")
	}
	computeService.UserAgent = userAgent

	computeServiceB, err := computebeta.New(computeClient)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Creating a Google Compute Service client")
	}
	computeServiceB.UserAgent = userAgent

	// Custom RoundTripper for retries
	storageRetrier := &RetryTransport{
		Base:       storageClient.Transport,
		MaxRetries: 3,
	}
	storageClient.Transport = storageRetrier
	storageService, err := storage.New(storageClient)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Creating a Google Storage Service client")
	}
	storageService.UserAgent = userAgent

	return GoogleClient{
		config:          config,
		computeService:  computeService,
		computeServiceB: computeServiceB,
		storageService:  storageService,
		logger:          logger,
	}, nil
}

func (c GoogleClient) Project() string {
	return c.config.Project
}

func (c GoogleClient) DefaultRootDiskSizeGb() int {
	return c.config.DefaultRootDiskSizeGb
}

func (c GoogleClient) DefaultRootDiskType() string {
	return c.config.DefaultRootDiskType
}

func (c GoogleClient) ComputeService() *compute.Service {
	return c.computeService
}

func (c GoogleClient) ComputeBetaService() *computebeta.Service {
	return c.computeServiceB
}

func (c GoogleClient) StorageService() *storage.Service {
	return c.storageService
}
<|endoftext|>


<commit_before>package moredis

import (
	"errors"
	"io/ioutil"

	"gopkg.in/v1/yaml"
)

// Config holds the config loaded from config.yml
type Config struct {
	Caches []CacheConfig `yaml:"caches"`
}

// CacheConfig is the config for a specific cache
type CacheConfig struct {
	Name        string             `yaml:"name"`
	Collections []CollectionConfig `yaml:"collections"`
}

// CollectionConfig is the config for a specific collection
type CollectionConfig struct {
	Collection string      `yaml:"collection"`
	Query      string      `yaml:"query"`
	Maps       []MapConfig `yaml:"maps"`
}

// MapConfig is the config for a specific map.
type MapConfig struct {
	Name    string `yaml:"name"`
	Key     string `yaml:"key"`
	Value   string `yaml:"val"`
	HashKey string
}

// LoadConfig takes a path to a config yaml file and loads it into the appropriate structs.
func LoadConfig(path string) (Config, error) {
	raw, err := ioutil.ReadFile(path)
	if err != nil {
		return Config{}, err
	}

	var conf Config
	if err := yaml.Unmarshal(raw, &conf); err != nil {
		return Config{}, err
	}

	return conf, nil
}

// GetCache returns the config for a specific cache inside the moredis config.
func (c Config) GetCache(cacheName string) (CacheConfig, error) {
	for _, cache := range c.Caches {
		if cache.Name == cacheName {
			return cache, nil
		}
	}
	return CacheConfig{}, errors.New("cache not found in config")
}
<commit_msg>use v2 of yaml pkg<commit_after>package moredis

import (
	"errors"
	"io/ioutil"

	"gopkg.in/v2/yaml"
)

// Config holds the config loaded from config.yml
type Config struct {
	Caches []CacheConfig `yaml:"caches"`
}

// CacheConfig is the config for a specific cache
type CacheConfig struct {
	Name        string             `yaml:"name"`
	Collections []CollectionConfig `yaml:"collections"`
}

// CollectionConfig is the config for a specific collection
type CollectionConfig struct {
	Collection string      `yaml:"collection"`
	Query      string      `yaml:"query"`
	Maps       []MapConfig `yaml:"maps"`
}

// MapConfig is the config for a specific map.
type MapConfig struct {
	Name    string `yaml:"name"`
	Key     string `yaml:"key"`
	Value   string `yaml:"val"`
	HashKey string
}

// LoadConfig takes a path to a config yaml file and loads it into the appropriate structs.
func LoadConfig(path string) (Config, error) {
	raw, err := ioutil.ReadFile(path)
	if err != nil {
		return Config{}, err
	}

	var conf Config
	if err := yaml.Unmarshal(raw, &conf); err != nil {
		return Config{}, err
	}

	return conf, nil
}

// GetCache returns the config for a specific cache inside the moredis config.
func (c Config) GetCache(cacheName string) (CacheConfig, error) {
	for _, cache := range c.Caches {
		if cache.Name == cacheName {
			return cache, nil
		}
	}
	return CacheConfig{}, errors.New("cache not found in config")
}
<|endoftext|>