<commit_before>package penname

import (
	"fmt"
	"net/http"
)

// PenName contains the state of the mock for performing its designated actions
// and capturing details for assertions
type PenName struct {
	Closed         bool
	Written        []byte
	WrittenHeaders []byte
	returnError    error
}

// New returns an initialized PenName for use in tests
func New() *PenName {
	return &PenName{}
}

// Close implements the closer interface, returning an error if returnError is
// set.  Whether or not Close is called is stored in Closed for inspection
// later.
func (p *PenName) Close() error {
	if p.returnError != nil {
		return p.returnError
	}

	p.Closed = true
	return nil
}

// Header implements the ResponseWriter interface, returning an empty set of
// headers to meet the interface requirements
func (p *PenName) Header() http.Header {
	return http.Header{}
}

// Reset is a convencinece method for reseting the state of the mock
func (p *PenName) Reset() {
	p.Closed = false
	p.Written = []byte{}
}

// ReturnError sets the error that will be returned when actions are attempted
func (p *PenName) ReturnError(err error) {
	p.returnError = err
}

// Write implements the Writer interface, returning an error if returnError is
// set.  The contents of what is written is stored in Written for inspection
// later.
func (p *PenName) Write(b []byte) (n int, err error) {
	if p.returnError != nil {
		return 0, p.returnError
	}

	p.Written = b
	return len(p.Written), nil
}

// WriteHeader implements the ResponseWriter interface, capturing headers to the
// same written buffer
func (p *PenName) WriteHeader(i int) {
	b := []byte(fmt.Sprintf("Header: %v", i))
	p.WrittenHeaders = b
	p.Write(b)
}
<commit_msg>updating package doc<commit_after>// Package penname provides a mock that implements the Closer & Writer
// interfaces for testing
package penname

import (
	"fmt"
	"net/http"
)

// PenName contains the state of the mock for performing its designated actions
// and capturing details for assertions
type PenName struct {
	Closed         bool
	Written        []byte
	WrittenHeaders []byte
	returnError    error
}

// New returns an initialized PenName for use in tests
func New() *PenName {
	return &PenName{}
}

// Close implements the closer interface, returning an error if returnError is
// set.  Whether or not Close is called is stored in Closed for inspection
// later.
func (p *PenName) Close() error {
	if p.returnError != nil {
		return p.returnError
	}

	p.Closed = true
	return nil
}

// Header implements the ResponseWriter interface, returning an empty set of
// headers to meet the interface requirements
func (p *PenName) Header() http.Header {
	return http.Header{}
}

// Reset is a convencinece method for reseting the state of the mock
func (p *PenName) Reset() {
	p.Closed = false
	p.Written = []byte{}
}

// ReturnError sets the error that will be returned when actions are attempted
func (p *PenName) ReturnError(err error) {
	p.returnError = err
}

// Write implements the Writer interface, returning an error if returnError is
// set.  The contents of what is written is stored in Written for inspection
// later.
func (p *PenName) Write(b []byte) (n int, err error) {
	if p.returnError != nil {
		return 0, p.returnError
	}

	p.Written = b
	return len(p.Written), nil
}

// WriteHeader implements the ResponseWriter interface, capturing headers to the
// same written buffer
func (p *PenName) WriteHeader(i int) {
	b := []byte(fmt.Sprintf("Header: %v", i))
	p.WrittenHeaders = b
	p.Write(b)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"io"
	"fmt"
	"database/sql"
	_ "github.com/ziutek/mymysql/godrv"
	"io/ioutil"
	"os"
	"strings"
)

const (
	nil_timestamp = "0000-00-00 00:00:00"
	mysql_datetime_format = "2006-01-02 15:04:05"
)

var (
	db *sql.DB
	db_connected = false
)

// escapeString and escapeQuotes copied from github.com/ziutek/mymysql/native/codecs.go
func escapeString(txt string) string {
	var (
		esc string
		buf bytes.Buffer
	)
	last := 0
	for ii, bb := range txt {
		switch bb {
		case 0:
			esc = `\0`
		case '\n':
			esc = `\n`
		case '\r':
			esc = `\r`
		case '\\':
			esc = `\\`
		case '\'':
			esc = `\'`
		case '"':
			esc = `\"`
		case '\032':
			esc = `\Z`
		default:
			continue
		}
		io.WriteString(&buf, txt[last:ii])
		io.WriteString(&buf, esc)
		last = ii + 1
	}
	io.WriteString(&buf, txt[last:])
	return buf.String()
}

func escapeQuotes(txt string) string {
	var buf bytes.Buffer
	last := 0
	for ii, bb := range txt {
		if bb == '\'' {
			io.WriteString(&buf, txt[last:ii])
			io.WriteString(&buf, `''`)
			last = ii + 1
		}
	}
	io.WriteString(&buf, txt[last:])
	return buf.String()
}

func connectToSQLServer() {
	// does the original initialsetupdb.sql (as opposed to .bak.sql) exist?
	var err error
	_, err1 := os.Stat("initialsetupdb.sql")
	_, err2 := os.Stat("initialsetupdb.bak.sql")
	if err2 == nil {
		// the .bak.sql file exists
		os.Remove("initialsetupdb.sql")
		fmt.Println("complete.")
		needs_initial_setup = false
		db, err = sql.Open("mymysql", config.DBhost + "*" + config.DBname + "/"+config.DBusername+"/"+config.DBpassword)
		if err != nil {
			fmt.Println("Failed to connect to the database, see log for details.")
			error_log.Fatal(err.Error())
		}
		return
	} else {
		if err1 != nil {
			// neither one exists
			fmt.Println("failed...initial setup file doesn't exist, please reinstall gochan.")
			error_log.Fatal("Initial setup file doesn't exist, exiting.")
			return
		}
	}
	err1 = nil
	err2 = nil

	db, err = sql.Open("mymysql", config.DBhost + "*" + config.DBname + "/"+config.DBusername+"/"+config.DBpassword)
	if err != nil {
		fmt.Println("Failed to connect to the database, see log for details.")
		error_log.Fatal(err.Error())
	}
	// read the initial setup sql file into a string
	initial_sql_bytes,err := ioutil.ReadFile("initialsetupdb.sql")
	if err != nil {
		fmt.Println("failed.")
		error_log.Fatal(err.Error())
	}
	initial_sql_str := string(initial_sql_bytes)
	initial_sql_bytes = nil
	fmt.Printf("Starting initial setup...")
	initial_sql_str = strings.Replace(initial_sql_str,"DBNAME",config.DBname, -1)
	initial_sql_str = strings.Replace(initial_sql_str,"DBPREFIX",config.DBprefix, -1)
	initial_sql_str += "\nINSERT INTO `"+config.DBname+"`.`"+config.DBprefix+"staff` (`username`, `password_checksum`, `salt`, `rank`) VALUES ('admin', '"+bcrypt_sum("password")+"', 'abc', 3);"
	initial_sql_arr := strings.Split(initial_sql_str, ";")
	initial_sql_str = ""

	for _,statement := range initial_sql_arr {
		if statement != "" {
			_,err := db.Exec(statement+";")
			if err != nil {
				fmt.Println("failed.")
				error_log.Fatal(err.Error())
				return
			}
		}
	}
	initial_sql_arr = nil
	// rename initialsetupdb.sql to initialsetup.bak.sql
	err = ioutil.WriteFile("initialsetupdb.bak.sql", initial_sql_bytes, 0777)
	if err != nil {
		fmt.Println("failed")
		error_log.Fatal(err.Error())
		return
	}

	err = os.Remove("initialsetupdb.bak.sql")
	if err != nil {
		fmt.Println("failed.")
		error_log.Fatal(err.Error())
		return
	}
	fmt.Println("complete.")

	needs_initial_setup = false
	db_connected = true
}<commit_msg>improve/streamlineMySQL connection code<commit_after>package main

import (
	"bytes"
	"io"
	"fmt"
	"database/sql"
	_ "github.com/ziutek/mymysql/godrv"
	"io/ioutil"
	"os"
	"strings"
)

const (
	nil_timestamp = "0000-00-00 00:00:00"
	mysql_datetime_format = "2006-01-02 15:04:05"
)

var (
	db *sql.DB
	db_connected = false
)

// escapeString and escapeQuotes copied from github.com/ziutek/mymysql/native/codecs.go
func escapeString(txt string) string {
	var (
		esc string
		buf bytes.Buffer
	)
	last := 0
	for ii, bb := range txt {
		switch bb {
		case 0:
			esc = `\0`
		case '\n':
			esc = `\n`
		case '\r':
			esc = `\r`
		case '\\':
			esc = `\\`
		case '\'':
			esc = `\'`
		case '"':
			esc = `\"`
		case '\032':
			esc = `\Z`
		default:
			continue
		}
		io.WriteString(&buf, txt[last:ii])
		io.WriteString(&buf, esc)
		last = ii + 1
	}
	io.WriteString(&buf, txt[last:])
	return buf.String()
}

func escapeQuotes(txt string) string {
	var buf bytes.Buffer
	last := 0
	for ii, bb := range txt {
		if bb == '\'' {
			io.WriteString(&buf, txt[last:ii])
			io.WriteString(&buf, `''`)
			last = ii + 1
		}
	}
	io.WriteString(&buf, txt[last:])
	return buf.String()
}

func connectToSQLServer() {
	var err error

	db, err = sql.Open("mymysql", config.DBhost + "*" + config.DBname + "/"+config.DBusername+"/"+config.DBpassword)
	if err != nil {
		fmt.Println("Failed to connect to the database, see log for details.")
		error_log.Fatal(err.Error())
	}

	// get the number of tables in the database. If the number > 1, we can assume that initial setup has already been run
	var num_rows int
	err = db.QueryRow("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '" + config.DBname + "';").Scan(&num_rows)
	if err == sql.ErrNoRows {
		num_rows = 0
	} else if err != nil {
		fmt.Println("Failed retrieving list of tables in database.")
		error_log.Fatal(err.Error())
	}
	if num_rows > 0 {
		// the initial setup has already been run
		needs_initial_setup = false
		db_connected = true
		fmt.Println("complete.")
		return
	} else {
		// does the  initialsetupdb.sql exist?
		_, err := os.Stat("initialsetupdb.sql")
		if err != nil {
			fmt.Println("Initial setup file (initialsetupdb.sql) missing. Please reinstall gochan")
			error_log.Fatal("Initial setup file (initialsetupdb.sql) missing. Please reinstall gochan")
		}

		// read the initial setup sql file into a string
		initial_sql_bytes,err := ioutil.ReadFile("initialsetupdb.sql")
		if err != nil {
			fmt.Println("failed, see log for details.")
			error_log.Fatal(err.Error())
		}
		initial_sql_str := string(initial_sql_bytes)
		initial_sql_bytes = nil
		fmt.Printf("Starting initial setup...")
		initial_sql_str = strings.Replace(initial_sql_str,"DBNAME",config.DBname, -1)
		initial_sql_str = strings.Replace(initial_sql_str,"DBPREFIX",config.DBprefix, -1)
		initial_sql_str += "\nINSERT INTO `"+config.DBname+"`.`"+config.DBprefix+"staff` (`username`, `password_checksum`, `salt`, `rank`) VALUES ('admin', '"+bcrypt_sum("password")+"', 'abc', 3);"
		initial_sql_arr := strings.Split(initial_sql_str, ";")
		initial_sql_str = ""

		for _,statement := range initial_sql_arr {
			if statement != "" {
				_,err := db.Exec(statement)
				if err != nil {
					fmt.Println("failed, see log for details.")
					error_log.Fatal(err.Error())
					return
				}
			}
		}
		fmt.Println("complete.")
		needs_initial_setup = false
		db_connected = true
	}
}<|endoftext|>


<commit_before>// Copyright 2014-2015 The DevMine authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// srctool is a command line tool to manage source code parsers. It is able to
// download parsers from a web server, install them and run them. In short, it is a
// manager for source code parsers.
package main

import (
	"fmt"
	"os"

	"github.com/codegangsta/cli"

	"github.com/DevMine/srctool/cmd"
	"github.com/DevMine/srctool/log"
)

func main() {
	app := cli.NewApp()
	app.Name = "srctool"
	app.Usage = "tool for parsing source code"
	app.Version = "1.0.0"
	app.Author = "The DevMine authors"
	app.Email = "contact@devmine.ch"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "d",
			Usage: "enable debug mode",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:      "install",
			ShortName: "i",
			Usage:     "install a language parser",
			Action: func(c *cli.Context) {
				log.SetDebugMode(c.GlobalBool("d"))
				cmd.Install(c)
			},
		},
		{
			Name:      "uninstall",
			ShortName: "u",
			Usage:     "uninstall a language parser",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "dry",
					Usage: "dry mode",
				},
			},
			Action: func(c *cli.Context) {
				log.SetDebugMode(c.GlobalBool("d"))
				cmd.Uninstall(c)
			},
		},
		{
			Name:      "update",
			ShortName: "u",
			Usage:     "update a language parser",
			Action: func(c *cli.Context) {
				log.SetDebugMode(c.GlobalBool("d"))
				cmd.Update(c)
			},
		},
		{
			Name:      "list",
			ShortName: "l",
			Usage:     "list all installed parsers",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "r",
					Usage: "list remote available parsers",
				},
			},
			Action: func(c *cli.Context) {
				log.SetDebugMode(c.GlobalBool("d"))
				cmd.List(c)
			},
		},
		{
			Name:      "parse",
			ShortName: "p",
			Usage:     "parse a project",
			Action: func(c *cli.Context) {
				log.SetDebugMode(c.GlobalBool("d"))
				cmd.Parse(c)
			},
		},
		{
			Name:      "config",
			ShortName: "c",
			Usage:     "create config file",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "server-url",
					Usage: "get/set the download server URL",
				},
			},
			Action: func(c *cli.Context) {
				log.SetDebugMode(c.GlobalBool("d"))
				cmd.Config(c)
			},
		},
		{
			Name:      "version",
			ShortName: "v",
			Usage:     "print program version",
			Action: func(c *cli.Context) {
				fmt.Printf("%s - v%s\n", app.Name, app.Version)
			},
		},
	}

	app.Run(os.Args)
}
<commit_msg>commands: change shortcut for 'upgrade' from 'u' to 'up'<commit_after>// Copyright 2014-2015 The DevMine authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// srctool is a command line tool to manage source code parsers. It is able to
// download parsers from a web server, install them and run them. In short, it is a
// manager for source code parsers.
package main

import (
	"fmt"
	"os"

	"github.com/codegangsta/cli"

	"github.com/DevMine/srctool/cmd"
	"github.com/DevMine/srctool/log"
)

func main() {
	app := cli.NewApp()
	app.Name = "srctool"
	app.Usage = "tool for parsing source code"
	app.Version = "1.0.0"
	app.Author = "The DevMine authors"
	app.Email = "contact@devmine.ch"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "d",
			Usage: "enable debug mode",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:      "install",
			ShortName: "i",
			Usage:     "install a language parser",
			Action: func(c *cli.Context) {
				log.SetDebugMode(c.GlobalBool("d"))
				cmd.Install(c)
			},
		},
		{
			Name:      "uninstall",
			ShortName: "u",
			Usage:     "uninstall a language parser",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "dry",
					Usage: "dry mode",
				},
			},
			Action: func(c *cli.Context) {
				log.SetDebugMode(c.GlobalBool("d"))
				cmd.Uninstall(c)
			},
		},
		{
			Name:      "update",
			ShortName: "up",
			Usage:     "update a language parser",
			Action: func(c *cli.Context) {
				log.SetDebugMode(c.GlobalBool("d"))
				cmd.Update(c)
			},
		},
		{
			Name:      "list",
			ShortName: "l",
			Usage:     "list all installed parsers",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "r",
					Usage: "list remote available parsers",
				},
			},
			Action: func(c *cli.Context) {
				log.SetDebugMode(c.GlobalBool("d"))
				cmd.List(c)
			},
		},
		{
			Name:      "parse",
			ShortName: "p",
			Usage:     "parse a project",
			Action: func(c *cli.Context) {
				log.SetDebugMode(c.GlobalBool("d"))
				cmd.Parse(c)
			},
		},
		{
			Name:      "config",
			ShortName: "c",
			Usage:     "create config file",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "server-url",
					Usage: "get/set the download server URL",
				},
			},
			Action: func(c *cli.Context) {
				log.SetDebugMode(c.GlobalBool("d"))
				cmd.Config(c)
			},
		},
		{
			Name:      "version",
			ShortName: "v",
			Usage:     "print program version",
			Action: func(c *cli.Context) {
				fmt.Printf("%s - v%s\n", app.Name, app.Version)
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ssh

import (
	"errors"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	gossh "github.com/coreos/fleet/Godeps/_workspace/src/golang.org/x/crypto/ssh"
	gosshagent "github.com/coreos/fleet/Godeps/_workspace/src/golang.org/x/crypto/ssh/agent"
	"github.com/coreos/fleet/Godeps/_workspace/src/golang.org/x/crypto/ssh/terminal"
)

type SSHForwardingClient struct {
	agentForwarding bool
	*gossh.Client
}

func (s *SSHForwardingClient) ForwardAgentAuthentication(session *gossh.Session) error {
	if s.agentForwarding {
		return gosshagent.RequestAgentForwarding(session)
	}
	return nil
}

func newSSHForwardingClient(client *gossh.Client, agentForwarding bool) (*SSHForwardingClient, error) {
	a, err := SSHAgentClient()
	if err != nil {
		return nil, err
	}

	err = gosshagent.ForwardToAgent(client, a)
	if err != nil {
		return nil, err
	}

	return &SSHForwardingClient{agentForwarding, client}, nil
}

// makeSession initializes a gossh.Session connected to the invoking process's stdout/stderr/stdout.
// If the invoking session is a terminal, a TTY will be requested for the SSH session.
// It returns a gossh.Session, a finalizing function used to clean up after the session terminates,
// and any error encountered in setting up the session.
func makeSession(client *SSHForwardingClient) (session *gossh.Session, finalize func(), err error) {
	session, err = client.NewSession()
	if err != nil {
		return
	}
	if err = client.ForwardAgentAuthentication(session); err != nil {
		return
	}

	session.Stdout = os.Stdout
	session.Stderr = os.Stderr
	session.Stdin = os.Stdin

	modes := gossh.TerminalModes{
		gossh.ECHO:          1,     // enable echoing
		gossh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
		gossh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	}

	fd := int(os.Stdin.Fd())
	if terminal.IsTerminal(fd) {

		var termWidth, termHeight int
		var oldState *terminal.State

		oldState, err = terminal.MakeRaw(fd)
		if err != nil {
			return
		}

		finalize = func() {
			session.Close()
			terminal.Restore(fd, oldState)
		}

		termWidth, termHeight, err = terminal.GetSize(fd)

		if err != nil {
			return
		}
		err = session.RequestPty("xterm-256color", termHeight, termWidth, modes)
	} else {
		finalize = func() {
			session.Close()
		}
	}

	return
}

// Execute runs the given command on the given client with stdin/stdout/stderr
// connected to the controlling terminal. It returns any error encountered in
// the SSH session, and the exit status of the remote command.
func Execute(client *SSHForwardingClient, cmd string) (error, int) {
	session, finalize, err := makeSession(client)
	if err != nil {
		return err, -1
	}

	defer finalize()

	session.Start(cmd)

	err = session.Wait()
	// the command ran and exited successfully
	if err == nil {
		return nil, 0
	}
	// if the session terminated normally, err should be ExitError; in that
	// case, return nil error and actual exit status of command
	if werr, ok := err.(*gossh.ExitError); ok {
		return nil, werr.ExitStatus()
	}
	// otherwise, we had an actual SSH error
	return err, -1
}

// Shell launches an interactive shell on the given client. It returns any
// error encountered in setting up the SSH session.
func Shell(client *SSHForwardingClient) error {
	session, finalize, err := makeSession(client)
	if err != nil {
		return err
	}

	defer finalize()

	if err = session.Shell(); err != nil {
		return err
	}

	session.Wait()
	return nil
}

// SSHAgentClient returns an Agent that talks to the local ssh-agent
func SSHAgentClient() (gosshagent.Agent, error) {
	sock := os.Getenv("SSH_AUTH_SOCK")
	if sock == "" {
		return nil, errors.New("SSH_AUTH_SOCK environment variable is not set. Verify ssh-agent is running. See https://github.com/coreos/fleet/blob/master/Documentation/using-the-client.md for help.")
	}

	agent, err := net.Dial("unix", sock)
	if err != nil {
		return nil, err
	}

	return gosshagent.NewClient(agent), nil
}

func sshClientConfig(user string, checker *HostKeyChecker) (*gossh.ClientConfig, error) {
	agentClient, err := SSHAgentClient()
	if err != nil {
		return nil, err
	}

	signers, err := agentClient.Signers()
	if err != nil {
		return nil, err
	}

	cfg := gossh.ClientConfig{
		User: user,
		Auth: []gossh.AuthMethod{
			gossh.PublicKeys(signers...),
		},
	}

	if checker != nil {
		cfg.HostKeyCallback = checker.Check
	}

	return &cfg, nil
}

func maybeAddDefaultPort(addr string) string {
	if strings.Contains(addr, ":") {
		return addr
	}
	return net.JoinHostPort(addr, strconv.Itoa(sshDefaultPort))
}

func NewSSHClient(user, addr string, checker *HostKeyChecker, agentForwarding bool, timeout time.Duration) (*SSHForwardingClient, error) {
	clientConfig, err := sshClientConfig(user, checker)
	if err != nil {
		return nil, err
	}

	addr = maybeAddDefaultPort(addr)

	var client *gossh.Client
	dialFunc := func(echan chan error) {
		var err error
		client, err = gossh.Dial("tcp", addr, clientConfig)
		echan <- err
	}
	err = timeoutSSHDial(dialFunc, timeout)
	if err != nil {
		return nil, err
	}

	return newSSHForwardingClient(client, agentForwarding)
}

func NewTunnelledSSHClient(user, tunaddr, tgtaddr string, checker *HostKeyChecker, agentForwarding bool, timeout time.Duration) (*SSHForwardingClient, error) {
	clientConfig, err := sshClientConfig(user, checker)
	if err != nil {
		return nil, err
	}

	tunaddr = maybeAddDefaultPort(tunaddr)
	tgtaddr = maybeAddDefaultPort(tgtaddr)

	var tunnelClient *gossh.Client
	dialFunc := func(echan chan error) {
		var err error
		tunnelClient, err = gossh.Dial("tcp", tunaddr, clientConfig)
		echan <- err
	}
	err = timeoutSSHDial(dialFunc, timeout)
	if err != nil {
		return nil, err
	}

	var targetConn net.Conn
	dialFunc = func(echan chan error) {
		tgtTCPAddr, err := net.ResolveTCPAddr("tcp", tgtaddr)
		if err != nil {
			echan <- err
			return
		}
		targetConn, err = tunnelClient.DialTCP("tcp", nil, tgtTCPAddr)
		echan <- err
	}
	err = timeoutSSHDial(dialFunc, timeout)
	if err != nil {
		return nil, err
	}

	c, chans, reqs, err := gossh.NewClientConn(targetConn, tgtaddr, clientConfig)
	if err != nil {
		return nil, err
	}
	return newSSHForwardingClient(gossh.NewClient(c, chans, reqs), agentForwarding)
}

func timeoutSSHDial(dial func(chan error), timeout time.Duration) error {
	var err error

	echan := make(chan error)
	go dial(echan)

	select {
	case <-time.After(timeout):
		return errors.New("timed out while initiating SSH connection")
	case err = <-echan:
		return err
	}
}
<commit_msg>fleetctl: fixed incorrect "auth-agent-req@openssh.com" behavior<commit_after>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ssh

import (
	"errors"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	gossh "github.com/coreos/fleet/Godeps/_workspace/src/golang.org/x/crypto/ssh"
	gosshagent "github.com/coreos/fleet/Godeps/_workspace/src/golang.org/x/crypto/ssh/agent"
	"github.com/coreos/fleet/Godeps/_workspace/src/golang.org/x/crypto/ssh/terminal"
)

type SSHForwardingClient struct {
	agentForwarding bool
	*gossh.Client
	agentForwardingEnabled bool
}

func (s *SSHForwardingClient) ForwardAgentAuthentication(session *gossh.Session) error {
	if s.agentForwarding && !s.agentForwardingEnabled {
		// We are allowed to send "auth-agent-req@openssh.com" request only once per channel
		// otherwise ssh daemon replies with the "SSH2_MSG_CHANNEL_FAILURE 100"
		s.agentForwardingEnabled = true
		return gosshagent.RequestAgentForwarding(session)
	}
	return nil
}

func newSSHForwardingClient(client *gossh.Client, agentForwarding bool) (*SSHForwardingClient, error) {
	a, err := SSHAgentClient()
	if err != nil {
		return nil, err
	}

	err = gosshagent.ForwardToAgent(client, a)
	if err != nil {
		return nil, err
	}

	return &SSHForwardingClient{agentForwarding, client, false}, nil
}

// makeSession initializes a gossh.Session connected to the invoking process's stdout/stderr/stdout.
// If the invoking session is a terminal, a TTY will be requested for the SSH session.
// It returns a gossh.Session, a finalizing function used to clean up after the session terminates,
// and any error encountered in setting up the session.
func makeSession(client *SSHForwardingClient) (session *gossh.Session, finalize func(), err error) {
	session, err = client.NewSession()
	if err != nil {
		return
	}
	if err = client.ForwardAgentAuthentication(session); err != nil {
		return
	}

	session.Stdout = os.Stdout
	session.Stderr = os.Stderr
	session.Stdin = os.Stdin

	modes := gossh.TerminalModes{
		gossh.ECHO:          1,     // enable echoing
		gossh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
		gossh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	}

	fd := int(os.Stdin.Fd())
	if terminal.IsTerminal(fd) {

		var termWidth, termHeight int
		var oldState *terminal.State

		oldState, err = terminal.MakeRaw(fd)
		if err != nil {
			return
		}

		finalize = func() {
			session.Close()
			terminal.Restore(fd, oldState)
		}

		termWidth, termHeight, err = terminal.GetSize(fd)

		if err != nil {
			return
		}
		err = session.RequestPty("xterm-256color", termHeight, termWidth, modes)
	} else {
		finalize = func() {
			session.Close()
		}
	}

	return
}

// Execute runs the given command on the given client with stdin/stdout/stderr
// connected to the controlling terminal. It returns any error encountered in
// the SSH session, and the exit status of the remote command.
func Execute(client *SSHForwardingClient, cmd string) (error, int) {
	session, finalize, err := makeSession(client)
	if err != nil {
		return err, -1
	}

	defer finalize()

	session.Start(cmd)

	err = session.Wait()
	// the command ran and exited successfully
	if err == nil {
		return nil, 0
	}
	// if the session terminated normally, err should be ExitError; in that
	// case, return nil error and actual exit status of command
	if werr, ok := err.(*gossh.ExitError); ok {
		return nil, werr.ExitStatus()
	}
	// otherwise, we had an actual SSH error
	return err, -1
}

// Shell launches an interactive shell on the given client. It returns any
// error encountered in setting up the SSH session.
func Shell(client *SSHForwardingClient) error {
	session, finalize, err := makeSession(client)
	if err != nil {
		return err
	}

	defer finalize()

	if err = session.Shell(); err != nil {
		return err
	}

	session.Wait()
	return nil
}

// SSHAgentClient returns an Agent that talks to the local ssh-agent
func SSHAgentClient() (gosshagent.Agent, error) {
	sock := os.Getenv("SSH_AUTH_SOCK")
	if sock == "" {
		return nil, errors.New("SSH_AUTH_SOCK environment variable is not set. Verify ssh-agent is running. See https://github.com/coreos/fleet/blob/master/Documentation/using-the-client.md for help.")
	}

	agent, err := net.Dial("unix", sock)
	if err != nil {
		return nil, err
	}

	return gosshagent.NewClient(agent), nil
}

func sshClientConfig(user string, checker *HostKeyChecker) (*gossh.ClientConfig, error) {
	agentClient, err := SSHAgentClient()
	if err != nil {
		return nil, err
	}

	signers, err := agentClient.Signers()
	if err != nil {
		return nil, err
	}

	cfg := gossh.ClientConfig{
		User: user,
		Auth: []gossh.AuthMethod{
			gossh.PublicKeys(signers...),
		},
	}

	if checker != nil {
		cfg.HostKeyCallback = checker.Check
	}

	return &cfg, nil
}

func maybeAddDefaultPort(addr string) string {
	if strings.Contains(addr, ":") {
		return addr
	}
	return net.JoinHostPort(addr, strconv.Itoa(sshDefaultPort))
}

func NewSSHClient(user, addr string, checker *HostKeyChecker, agentForwarding bool, timeout time.Duration) (*SSHForwardingClient, error) {
	clientConfig, err := sshClientConfig(user, checker)
	if err != nil {
		return nil, err
	}

	addr = maybeAddDefaultPort(addr)

	var client *gossh.Client
	dialFunc := func(echan chan error) {
		var err error
		client, err = gossh.Dial("tcp", addr, clientConfig)
		echan <- err
	}
	err = timeoutSSHDial(dialFunc, timeout)
	if err != nil {
		return nil, err
	}

	return newSSHForwardingClient(client, agentForwarding)
}

func NewTunnelledSSHClient(user, tunaddr, tgtaddr string, checker *HostKeyChecker, agentForwarding bool, timeout time.Duration) (*SSHForwardingClient, error) {
	clientConfig, err := sshClientConfig(user, checker)
	if err != nil {
		return nil, err
	}

	tunaddr = maybeAddDefaultPort(tunaddr)
	tgtaddr = maybeAddDefaultPort(tgtaddr)

	var tunnelClient *gossh.Client
	dialFunc := func(echan chan error) {
		var err error
		tunnelClient, err = gossh.Dial("tcp", tunaddr, clientConfig)
		echan <- err
	}
	err = timeoutSSHDial(dialFunc, timeout)
	if err != nil {
		return nil, err
	}

	var targetConn net.Conn
	dialFunc = func(echan chan error) {
		tgtTCPAddr, err := net.ResolveTCPAddr("tcp", tgtaddr)
		if err != nil {
			echan <- err
			return
		}
		targetConn, err = tunnelClient.DialTCP("tcp", nil, tgtTCPAddr)
		echan <- err
	}
	err = timeoutSSHDial(dialFunc, timeout)
	if err != nil {
		return nil, err
	}

	c, chans, reqs, err := gossh.NewClientConn(targetConn, tgtaddr, clientConfig)
	if err != nil {
		return nil, err
	}
	return newSSHForwardingClient(gossh.NewClient(c, chans, reqs), agentForwarding)
}

func timeoutSSHDial(dial func(chan error), timeout time.Duration) error {
	var err error

	echan := make(chan error)
	go dial(echan)

	select {
	case <-time.After(timeout):
		return errors.New("timed out while initiating SSH connection")
	case err = <-echan:
		return err
	}
}
<|endoftext|>


<commit_before>package weibo

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"reflect"
	"strings"

	"github.com/google/go-querystring/query"
)

const (
	libraryVersion  = "0.1"
	weiboApiVersion = "2"
	defaultBaseURL  = "https://api.weibo.com/"
	userAgent       = "go-weibo/" + libraryVersion
)

// A Client manages communication with the Weibo API.
type Client struct {
	// HTTP client used to communcate with the API.
	client *http.Client

	// Base URL for API requests.
	BaseURL *url.URL

	// User agent used when communicating with the Weibo API.
	UserAgent string

	// Services used for talking to different parts of the Weibo API.
	Statuses *StatusesService
}

// ListOptions specifies the optional parameters to various List methods that
// support pagination.
type ListOptions struct {
	// For paginated result sets, page of results to retrieve.
	Page int `url:"page,omitempty"`

	// For paginated result sets, the number of results to include per page.
	PerPage int `url:"count,omitempty"`
}

// addOptions adds the parameters in opt as URL query parameters to string.
// opt must be a struct whose fields may contain "url" tags.
func addOptions(s string, opt interface{}) (string, error) {
	v := reflect.ValueOf(opt)
	if v.Kind() == reflect.Ptr && v.IsNil() {
		return s, nil
	}

	u, err := url.Parse(s)
	if err != nil {
		return s, err
	}

	qs, err := query.Values(opt)
	if err != nil {
		return s, err
	}

	u.RawQuery = qs.Encode()
	return u.String(), nil
}

// NewClient returns a new Weibo API client.  If a nil httpClient is
// provided, http.DefaultClient will be used. To use API methods which require
// authentication, provide an http.Client that will perform the authentication
// for you (such as that provided by goauth2 library).
func NewClient(httpClient *http.Client) *Client {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}
	baseURL, _ := url.Parse(defaultBaseURL)

	c := &Client{client: httpClient, BaseURL: baseURL, UserAgent: userAgent}
	c.Statuses = &StatusesService{client: c}

	return c
}

// NewRequest creates an API request.  A relative URL can be provided in urlStr,
// in which case it is resolved relative to the BaseURL of the Client.
// Relative URLs should always be specified without a preceding slash. If
// specified, the value pointed to by body is JSON encoded and included as the
// request body.
func (c *Client) NewRequest(method, urlString string, body interface{}) (*http.Request, error) {
	if strings.HasPrefix(urlString, "/") {
		urlString = weiboApiVersion + urlString
	} else {
		urlString = weiboApiVersion + "/" + urlString
	}

	rel, err := url.Parse(urlString)
	if err != nil {
		return nil, err
	}

	u := c.BaseURL.ResolveReference(rel)

	buf := new(bytes.Buffer)
	if body != nil {
		if err := json.NewEncoder(buf).Encode(body); err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	req.Header.Add("User-Agent", c.UserAgent)
	return req, nil
}

// Response is a Weibo API response.
// This wraps the standrad http.Response returned from Weibo.
// For future use, maybe.
type Response struct {
	*http.Response
}

// newResponse creates a new Response for the provided http.Response.
func newResponse(r *http.Response) *Response {
	response := &Response{Response: r}

	return response
}

// Do sends an API request and returns the API response.  The API response is
// JSON decoded and stored in the value pointed to by v, or returned as an
// error if an API error has occured.  If v implements the io.Writer
// interface, the raw response body will be written to v, without attempting
// to first decode it.
func (c *Client) Do(req *http.Request, v interface{}) (*Response, error) {
	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	response := newResponse(resp)

	if err := CheckResponse(resp); err != nil {
		// even though there was an error, we still return the response
		// in case the caller wants to inspect it further
		return response, err
	}

	if v != nil {
		if w, ok := v.(io.Writer); ok {
			io.Copy(w, resp.Body)
		} else {
			err = json.NewDecoder(resp.Body).Decode(v)
		}
	}

	return response, err
}

// An Error Response reports one or more errors caused by an API request.
//
// Weibo API docs: http://open.weibo.com/wiki/Error_code
type ErrorResponse struct {
	Response   *http.Response // HTTP response that caused this error
	RequestURL string         `json:"request"`    // request on which the error occured
	ErrorCode  int            `json:"error_code"` // error_code
	Message    string         `json:"error"`      // error message
}

func (r *ErrorResponse) Error() string {
	return fmt.Sprintf("%v %v: %d %v %v",
		r.Response.Request.Method, r.RequestURL,
		r.Response.StatusCode, r.ErrorCode, r.Message)
}

// CheckResponse checks the API response for errors, and returns them if
// present.  A response is considered an error if it has a status code outside
// the 200 range. API error responses are expected to have either no response
// body, or a JSON response body that maps to ErrorResponse.  Any other
// response body will be silently ignored.
func CheckResponse(r *http.Response) error {
	if c := r.StatusCode; 200 <= c && c <= 299 {
		return nil
	}
	errorResponse := &ErrorResponse{Response: r}
	data, err := ioutil.ReadAll(r.Body)
	if err == nil && data != nil {
		json.Unmarshal(data, errorResponse)
	}
	return errorResponse
}

// Bool is a helper routine that allocates a new bool value
// to store v and returns a pointer to it.
func Bool(v bool) *bool {
	p := new(bool)
	*p = v
	return p
}

// Int is a helper routine that allocates a new int32 value
// to store v and returns a pointer to it, but unlike Int32
// its argument value is an int.
func Int(v int) *int {
	p := new(int)
	*p = v
	return p
}

// String is a helper routine that allocates a new string value
// to store v and returns a pointer to it.
func String(v string) *string {
	p := new(string)
	*p = v
	return p
}
<commit_msg>Pass access_token when NewClient.<commit_after>package weibo

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"reflect"
	"strings"

	"github.com/google/go-querystring/query"
)

const (
	libraryVersion  = "0.1"
	weiboApiVersion = "2"
	defaultBaseURL  = "https://api.weibo.com/"
	userAgent       = "go-weibo/" + libraryVersion
)

// A Client manages communication with the Weibo API.
type Client struct {
	// HTTP client used to communcate with the API.
	client *http.Client

	// Access Token
	accessToken string

	// Base URL for API requests.
	BaseURL *url.URL

	// User agent used when communicating with the Weibo API.
	UserAgent string

	// Services used for talking to different parts of the Weibo API.
	Statuses *StatusesService
}

// ListOptions specifies the optional parameters to various List methods that
// support pagination.
type ListOptions struct {
	// For paginated result sets, page of results to retrieve.
	Page int `url:"page,omitempty"`

	// For paginated result sets, the number of results to include per page.
	PerPage int `url:"count,omitempty"`
}

// addOptions adds the parameters in opt as URL query parameters to string.
// opt must be a struct whose fields may contain "url" tags.
func addOptions(s string, opt interface{}) (string, error) {
	v := reflect.ValueOf(opt)
	if v.Kind() == reflect.Ptr && v.IsNil() {
		return s, nil
	}

	u, err := url.Parse(s)
	if err != nil {
		return s, err
	}

	qs, err := query.Values(opt)
	if err != nil {
		return s, err
	}

	u.RawQuery = qs.Encode()
	return u.String(), nil
}

// NewClient returns a new Weibo API client.
func NewClient(accessToken string) *Client {
	baseURL, _ := url.Parse(defaultBaseURL)

	c := &Client{client: http.DefaultClient, accessToken: accessToken, BaseURL: baseURL, UserAgent: userAgent}
	c.Statuses = &StatusesService{client: c}

	return c
}

// NewRequest creates an API request.  A relative URL can be provided in urlStr,
// in which case it is resolved relative to the BaseURL of the Client.
// Relative URLs should always be specified without a preceding slash. If
// specified, the value pointed to by body is JSON encoded and included as the
// request body.
func (c *Client) NewRequest(method, urlString string, body interface{}) (*http.Request, error) {
	if strings.HasPrefix(urlString, "/") {
		urlString = weiboApiVersion + urlString
	} else {
		urlString = weiboApiVersion + "/" + urlString
	}

	rel, err := url.Parse(urlString)
	if err != nil {
		return nil, err
	}

	u := c.BaseURL.ResolveReference(rel)

	buf := new(bytes.Buffer)
	if body != nil {
		if err := json.NewEncoder(buf).Encode(body); err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	req.Header.Add("User-Agent", c.UserAgent)
	return req, nil
}

// Response is a Weibo API response.
// This wraps the standrad http.Response returned from Weibo.
// For future use, maybe.
type Response struct {
	*http.Response
}

// newResponse creates a new Response for the provided http.Response.
func newResponse(r *http.Response) *Response {
	response := &Response{Response: r}

	return response
}

// Do sends an API request and returns the API response.  The API response is
// JSON decoded and stored in the value pointed to by v, or returned as an
// error if an API error has occured.  If v implements the io.Writer
// interface, the raw response body will be written to v, without attempting
// to first decode it.
func (c *Client) Do(req *http.Request, v interface{}) (*Response, error) {
	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	response := newResponse(resp)

	if err := CheckResponse(resp); err != nil {
		// even though there was an error, we still return the response
		// in case the caller wants to inspect it further
		return response, err
	}

	if v != nil {
		if w, ok := v.(io.Writer); ok {
			io.Copy(w, resp.Body)
		} else {
			err = json.NewDecoder(resp.Body).Decode(v)
		}
	}

	return response, err
}

// An Error Response reports one or more errors caused by an API request.
//
// Weibo API docs: http://open.weibo.com/wiki/Error_code
type ErrorResponse struct {
	Response   *http.Response // HTTP response that caused this error
	RequestURL string         `json:"request"`    // request on which the error occured
	ErrorCode  int            `json:"error_code"` // error_code
	Message    string         `json:"error"`      // error message
}

func (r *ErrorResponse) Error() string {
	return fmt.Sprintf("%v %v: %d %v %v",
		r.Response.Request.Method, r.RequestURL,
		r.Response.StatusCode, r.ErrorCode, r.Message)
}

// CheckResponse checks the API response for errors, and returns them if
// present.  A response is considered an error if it has a status code outside
// the 200 range. API error responses are expected to have either no response
// body, or a JSON response body that maps to ErrorResponse.  Any other
// response body will be silently ignored.
func CheckResponse(r *http.Response) error {
	if c := r.StatusCode; 200 <= c && c <= 299 {
		return nil
	}
	errorResponse := &ErrorResponse{Response: r}
	data, err := ioutil.ReadAll(r.Body)
	if err == nil && data != nil {
		json.Unmarshal(data, errorResponse)
	}
	return errorResponse
}

// Bool is a helper routine that allocates a new bool value
// to store v and returns a pointer to it.
func Bool(v bool) *bool {
	p := new(bool)
	*p = v
	return p
}

// Int is a helper routine that allocates a new int32 value
// to store v and returns a pointer to it, but unlike Int32
// its argument value is an int.
func Int(v int) *int {
	p := new(int)
	*p = v
	return p
}

// String is a helper routine that allocates a new string value
// to store v and returns a pointer to it.
func String(v string) *string {
	p := new(string)
	*p = v
	return p
}
<|endoftext|>


<commit_before>package main

import "net/http"
import "time"
import "github.com/PuerkitoBio/goquery"
import uuidl "github.com/satori/go.uuid"
import "strconv"
import "fmt"

func StyleGetTo(crawinstanceuuid, fireurl, uid string, pageno int, dbe *ExecTimeDbS) (int, error) {

	conf, err := dbe.LoadConfigure()

	if err != nil {
		return 0, err
	}

	var cookie, ua string

	err = dbe.Dbc.QueryRow("SELECT crawcookie,crawua FROM crawinstance WHERE crawinstanceuuid=?", crawinstanceuuid).Scan(&cookie, &ua)

	if err != nil {
		return 0, err
	}

	req, err := http.NewRequest("GET", fireurl, nil)

	if err != nil {
		return 0, err
	}

	req.Header.Add("Cookie", cookie)

	req.Header.Add("User-Agent", ua)

	client := &http.Client{}

	resp, err := client.Do(req)

	if err != nil {
		return 0, err
	}

	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		return 0, err
	}
	bds := string(body)
	_, err = dbe.Dbc.Exec("INSERT INTO crawbuffer(uid,time,crawinstanceuuid,firedurl,ctx,pageno) VALUES (?,?,?,?,?,?)", uid, time.Now().Unix(), crawinstanceuuid, fireurl, bds, pageno)

	if err != nil {
		return 0, err
	}

	return len(bds), nil

}

func StyleParseNextPage(crawinstanceuuid, fireurl, uid string, pageno int, dbe *ExecTimeDbS) (int, int, error) {

	var ctx string

	err := dbe.Dbc.QueryRow("SELECT ctx FROM crawbuffer WHERE crawinstanceuuid=? AND uid=? AND pageno=? AND firedurl=?", crawinstanceuuid, uid, pageno, fireurl).Scan(&ctx)

	if err != nil {
		return -1, err
	}

	gq := goquery.NewDocumentFromReader(strings.NewReader(ctx))

	s := gq.Find("div.pa#pagelist form div").First().Children().Last().Text()

	if s == "" {
		return -1, errors.New("Assert Fail:no page count")
	}

	pagerx, err := regexp.Compile(`(?:[\t\n\v\f\r ])*?(?P<currentpage>(?:\d)*)/(?P<maxpage>(?:\d)*)页`)
	//https://regex101.com/r/wQ6mY2/1

	if err != nil {
		return -1, -1, err
	}

	sf := pagerx.FindStringSubmatch(s)

	rs := strconv.Atoi(sf[1]) //maxpage
	cs := strconv.Atoi(sf[0]) //currentpage

	return cs, rs, nil
}

func StyleComputePageurl(uid string, pageno int) string {

	return fmt.Sprintf(`http://weibo.cn/%v?filter=1&page=%v&vt=4`, uid, pageno)

}

func StyleMkcrawinst(crawas, crawua, crawnote, crawcookie string, dbe *ExecTimeDbS) (string, error) {

	cluuid := string(uuidl.NewV4())

	_, err = dbe.Dbc.Exec("INSERT INTO crawinstance(crawinstanceuuid,initializedtime,crawas,crawcookie,crawnote,crawua) VALUES (?,?,?,?,?,?)", cluuid, time.Now().Unix(), crawas, crawcookie, crawnote, crawua)

	if err != nil {
		return nil, err
	}

	return cluuid, nil
}

func Docraw(uid, crawinstanceuuid string, dbe *ExecTimeDbS) (int, error) {

	var cup, maxp int
	cup = 1
	maxp = 100
	for cup != maxp {
		url := StyleComputePageurl(uid, cup)
		_, err := StyleGetTo(crawinstanceuuid, url, uid, cup, dbe)
		if err != nil {
			return cup, err
		}
		ifcup, maxp, err := StyleParseNextPage(crawinstanceuuid, url, uid, cup, dbe)

		if err != nil {
			fmt.Println(err)
			fmt.Println("Failed to obtain page information")
		}

		fmt.Printf("%v/%vpage@%v", ifcup, maxp, uid)

	}

}
func crawTaskExec(crawinstanceuuid string, dbe *ExecTimeDbS) {
	r, err := dbe.Dbc.Query("SELECT uid FROM weibocrawtarget")
	if err != nil {
		return err
	}
	for r.Next() {
		var uid string
		rows.Scan(&uid)
		Docraw(uid, crawinstanceuuid, dbe)
	}
}
<commit_msg>Add craw target<commit_after>package main

import "net/http"
import "time"
import "github.com/PuerkitoBio/goquery"
import uuidl "github.com/satori/go.uuid"
import "strconv"
import "fmt"

func StyleGetTo(crawinstanceuuid, fireurl, uid string, pageno int, dbe *ExecTimeDbS) (int, error) {

	conf, err := dbe.LoadConfigure()

	if err != nil {
		return 0, err
	}

	var cookie, ua string

	err = dbe.Dbc.QueryRow("SELECT crawcookie,crawua FROM crawinstance WHERE crawinstanceuuid=?", crawinstanceuuid).Scan(&cookie, &ua)

	if err != nil {
		return 0, err
	}

	req, err := http.NewRequest("GET", fireurl, nil)

	if err != nil {
		return 0, err
	}

	req.Header.Add("Cookie", cookie)

	req.Header.Add("User-Agent", ua)

	client := &http.Client{}

	resp, err := client.Do(req)

	if err != nil {
		return 0, err
	}

	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		return 0, err
	}
	bds := string(body)
	_, err = dbe.Dbc.Exec("INSERT INTO crawbuffer(uid,time,crawinstanceuuid,firedurl,ctx,pageno) VALUES (?,?,?,?,?,?)", uid, time.Now().Unix(), crawinstanceuuid, fireurl, bds, pageno)

	if err != nil {
		return 0, err
	}

	return len(bds), nil

}

func StyleParseNextPage(crawinstanceuuid, fireurl, uid string, pageno int, dbe *ExecTimeDbS) (int, int, error) {

	var ctx string

	err := dbe.Dbc.QueryRow("SELECT ctx FROM crawbuffer WHERE crawinstanceuuid=? AND uid=? AND pageno=? AND firedurl=?", crawinstanceuuid, uid, pageno, fireurl).Scan(&ctx)

	if err != nil {
		return -1, err
	}

	gq := goquery.NewDocumentFromReader(strings.NewReader(ctx))

	s := gq.Find("div.pa#pagelist form div").First().Children().Last().Text()

	if s == "" {
		return -1, errors.New("Assert Fail:no page count")
	}

	pagerx, err := regexp.Compile(`(?:[\t\n\v\f\r ])*?(?P<currentpage>(?:\d)*)/(?P<maxpage>(?:\d)*)页`)
	//https://regex101.com/r/wQ6mY2/1

	if err != nil {
		return -1, -1, err
	}

	sf := pagerx.FindStringSubmatch(s)

	rs := strconv.Atoi(sf[1]) //maxpage
	cs := strconv.Atoi(sf[0]) //currentpage

	return cs, rs, nil
}

func StyleComputePageurl(uid string, pageno int) string {

	return fmt.Sprintf(`http://weibo.cn/%v?filter=1&page=%v&vt=4`, uid, pageno)

}

func StyleMkcrawinst(crawas, crawua, crawnote, crawcookie string, dbe *ExecTimeDbS) (string, error) {

	cluuid := string(uuidl.NewV4())

	_, err = dbe.Dbc.Exec("INSERT INTO crawinstance(crawinstanceuuid,initializedtime,crawas,crawcookie,crawnote,crawua) VALUES (?,?,?,?,?,?)", cluuid, time.Now().Unix(), crawas, crawcookie, crawnote, crawua)

	if err != nil {
		return nil, err
	}

	return cluuid, nil
}

func Docraw(uid, crawinstanceuuid string, dbe *ExecTimeDbS) (int, error) {

	var cup, maxp int
	cup = 1
	maxp = 100
	for cup != maxp {
		url := StyleComputePageurl(uid, cup)
		_, err := StyleGetTo(crawinstanceuuid, url, uid, cup, dbe)
		if err != nil {
			return cup, err
		}
		ifcup, maxp, err := StyleParseNextPage(crawinstanceuuid, url, uid, cup, dbe)

		if err != nil {
			fmt.Println(err)
			fmt.Println("Failed to obtain page information")
		}

		fmt.Printf("%v/%vpage@%v", ifcup, maxp, uid)

	}

}
func crawTaskExec(crawinstanceuuid string, dbe *ExecTimeDbS) {
	r, err := dbe.Dbc.Query("SELECT uid FROM weibocrawtarget")
	if err != nil {
		return err
	}
	for r.Next() {
		var uid string
		rows.Scan(&uid)
		Docraw(uid, crawinstanceuuid, dbe)
	}
}

func Addcrawtarget(uid, dbe *ExecTimeDbS) {

	dbe.Dbc.Exec("INSERT INTO weibocrawtarget(uid) VALUES(?)", uid)

}

func Setcrawtargettype(uid, typeofowner, dbe *ExecTimeDbS) {

	dbe.Dbc.Exec("UPDATE weibocrawtarget SET typeofowner=? WHERE uid=?", typeofowner, uid)

}
<|endoftext|>


<commit_before>package coverage_fixture

func A() string {
	return "A"
}

func B() string {
	return "B"
}

func C() string {
	return "C"
}

func D() string {
	return "D"
}

func E() string {
	return "untested"
}
<commit_msg>Update fixtures to work using go1.10<commit_after>package coverage_fixture

import (
	_ "github.com/onsi/ginkgo/integration/_fixtures/coverage_fixture/external_coverage_fixture"
)

func A() string {
	return "A"
}

func B() string {
	return "B"
}

func C() string {
	return "C"
}

func D() string {
	return "D"
}

func E() string {
	return "untested"
}
<|endoftext|>


<commit_before>// Copyright (c) 2011 Dmitry Chestnykh
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package stemmer declares Stemmer interface.
package stemmer

type Stemmer interface{
	Stem(s string) string
}<commit_msg>stemmer: apply gofmt<commit_after>// Copyright (c) 2011 Dmitry Chestnykh
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package stemmer declares Stemmer interface.
package stemmer

type Stemmer interface {
	Stem(s string) string
}
<|endoftext|>


<commit_before>package resources

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/rebuy-de/aws-nuke/pkg/types"
)

func init() {
	register("Route53HealthCheck", ListRoute53HealthChecks)
}

func ListRoute53HealthChecks(sess *session.Session) ([]Resource, error) {
	svc := route53.New(sess)
	params := &route53.ListHealthChecksInput{}
	resources := make([]Resource, 0)

	resp, err := svc.ListHealthChecks(params)
	if err != nil {
		return nil, err
	}

	for _, check := range resp.HealthChecks {
		resources = append(resources, &Route53HealthCheck{
			svc:  svc,
			id:   check.Id,
		})
	}

	return resources, nil
}

type Route53HealthCheck struct {
	svc  *route53.Route53
	id   *string
}

func (hz *Route53HealthCheck) Remove() error {
	params := &route53.DeleteHealthCheckInput{
		HealthCheckId: hz.id,
	}

	_, err := hz.svc.DeleteHealthCheck(params)
	if err != nil {
		return err
	}

	return nil
}

func (hz *Route53HealthCheck) Properties() types.Properties {
	return types.NewProperties().
		Set("ID", hz.id)
}

func (hz *Route53HealthCheck) String() string {
	return fmt.Sprintf("%s", *hz.id)
}
<commit_msg>Handle pagination 101+ route53 healthchecks (#472)<commit_after>package resources

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/rebuy-de/aws-nuke/pkg/types"
	"github.com/aws/aws-sdk-go/aws"
)

func init() {
	register("Route53HealthCheck", ListRoute53HealthChecks)
}

func ListRoute53HealthChecks(sess *session.Session) ([]Resource, error) {
	svc := route53.New(sess)
	params := &route53.ListHealthChecksInput{}
	resources := make([]Resource, 0)

	for {
		resp, err := svc.ListHealthChecks(params)
		if err != nil {
			return nil, err
		}

		for _, check := range resp.HealthChecks {
			resources = append(resources, &Route53HealthCheck{
				svc: svc,
				id:  check.Id,
			})
		}
		if aws.BoolValue(resp.IsTruncated) != false {
			break
		}
		params.Marker = resp.NextMarker
	}

	return resources, nil
}

type Route53HealthCheck struct {
	svc *route53.Route53
	id  *string
}

func (hz *Route53HealthCheck) Remove() error {
	params := &route53.DeleteHealthCheckInput{
		HealthCheckId: hz.id,
	}

	_, err := hz.svc.DeleteHealthCheck(params)
	if err != nil {
		return err
	}

	return nil
}

func (hz *Route53HealthCheck) Properties() types.Properties {
	return types.NewProperties().
		Set("ID", hz.id)
}

func (hz *Route53HealthCheck) String() string {
	return fmt.Sprintf("%s", *hz.id)
}
<|endoftext|>


<commit_before>// +build envtest

/*
 Copyright 2021 Crunchy Data Solutions, Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package postgrescluster

import (
	"context"
	"io"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/onsi/gomega"
	"github.com/onsi/gomega/gstruct"
	"go.opentelemetry.io/otel"
	"gotest.tools/v3/assert"
	appsv1 "k8s.io/api/apps/v1"
	v1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/tools/record"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/envtest"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
	"sigs.k8s.io/yaml"

	"github.com/crunchydata/postgres-operator/internal/naming"
	"github.com/crunchydata/postgres-operator/internal/patroni"
	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1alpha1"
)

func TestReconcilerHandleDelete(t *testing.T) {
	if !strings.EqualFold(os.Getenv("USE_EXISTING_CLUSTER"), "true") {
		t.Skip("requires a running garbage collection controller")
	}
	t.Parallel()

	ctx := context.Background()
	env := &envtest.Environment{
		CRDDirectoryPaths: []string{
			filepath.Join("..", "..", "..", "config", "crd", "bases"),
		},
	}

	options := client.Options{}
	options.Scheme = runtime.NewScheme()
	assert.NilError(t, scheme.AddToScheme(options.Scheme))
	assert.NilError(t, v1alpha1.AddToScheme(options.Scheme))

	config, err := env.Start()
	assert.NilError(t, err)
	t.Cleanup(func() { assert.Check(t, env.Stop()) })

	cc, err := client.New(config, options)
	assert.NilError(t, err)

	ns := &v1.Namespace{}
	ns.GenerateName = "postgres-operator-test-"
	ns.Labels = labels.Set{"postgres-operator-test": t.Name()}
	assert.NilError(t, cc.Create(ctx, ns))
	t.Cleanup(func() { assert.Check(t, cc.Delete(ctx, ns)) })

	// set the operator namespace variable
	naming.PostgresOperatorNamespace = ns.Name

	// TODO(cbandy): namespace rbac
	assert.NilError(t, cc.Create(ctx, &v1.ServiceAccount{
		ObjectMeta: metav1.ObjectMeta{Namespace: ns.Name, Name: "postgres-operator"},
	}))
	assert.NilError(t, cc.Create(ctx, &rbacv1.RoleBinding{
		ObjectMeta: metav1.ObjectMeta{Namespace: ns.Name, Name: "postgres-operator"},
		RoleRef: rbacv1.RoleRef{
			Kind: "ClusterRole", Name: "postgres-operator-role",
		},
		Subjects: []rbacv1.Subject{{
			Kind: "ServiceAccount", Namespace: ns.Name, Name: "postgres-operator",
		}},
	}))

	reconciler := Reconciler{
		Client:   cc,
		Owner:    client.FieldOwner(t.Name()),
		Recorder: new(record.FakeRecorder),
		Tracer:   otel.Tracer(t.Name()),
	}

	reconciler.PodExec, err = newPodExecutor(config)
	assert.NilError(t, err)

	mustReconcile := func(t *testing.T, cluster *v1alpha1.PostgresCluster) reconcile.Result {
		t.Helper()
		key := client.ObjectKeyFromObject(cluster)
		request := reconcile.Request{NamespacedName: key}
		result, err := reconciler.Reconcile(ctx, request)
		assert.NilError(t, err, "%+v", err)
		return result
	}

	for _, test := range []struct {
		name         string
		beforeDelete func(*testing.T, *v1alpha1.PostgresCluster)
		propagation  metav1.DeletionPropagation
	}{
		{
			name:        "Background",
			propagation: metav1.DeletePropagationBackground,
		},
		// TODO(cbandy): metav1.DeletePropagationForeground
		{
			name:        "AfterFailover",
			propagation: metav1.DeletePropagationBackground,
			beforeDelete: func(t *testing.T, cluster *v1alpha1.PostgresCluster) {
				list := v1.PodList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/instance",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))

				var primary *v1.Pod
				var replica *v1.Pod
				for i := range list.Items {
					if list.Items[i].Labels["postgres-operator.crunchydata.com/role"] == "replica" {
						replica = &list.Items[i]
					} else {
						primary = &list.Items[i]
					}
				}
				assert.Assert(t, primary != nil, "expected to find a primary in %+v", list.Items)
				assert.Assert(t, replica != nil, "expected to find a replica in %+v", list.Items)

				assert.NilError(t, patroni.Executor(func(_ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string) error {
					return reconciler.PodExec(replica.Namespace, replica.Name, "database", stdin, stdout, stderr, command...)
				}).ChangePrimary(ctx, primary.Name, replica.Name))
			},
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			g := gomega.NewWithT(t)

			cluster := &v1alpha1.PostgresCluster{}
			assert.NilError(t, yaml.Unmarshal([]byte(`{
				spec: {
					postgresVersion: 12,
					instances: [
						{ replicas: 2 },
					],
				},
			}`), cluster))

			cluster.Namespace = ns.Name
			cluster.Name = strings.ToLower(test.name)
			assert.NilError(t, cc.Create(ctx, cluster))

			t.Cleanup(func() {
				// Remove finalizers, if any, so the namespace can terminate.
				assert.Check(t, client.IgnoreNotFound(
					cc.Patch(ctx, cluster, client.RawPatch(
						client.Merge.Type(), []byte(`{"metadata":{"finalizers":[]}}`)))))
			})

			// Start cluster.
			mustReconcile(t, cluster)

			assert.NilError(t,
				cc.Get(ctx, client.ObjectKeyFromObject(cluster), cluster))
			g.Expect(cluster.Finalizers).To(
				gomega.ContainElement("postgres-operator.crunchydata.com/finalizer"),
				"cluster should immediately have a finalizer")

			// Continue until instances are healthy.
			g.Eventually(func() (instances []appsv1.StatefulSet) {
				mustReconcile(t, cluster)

				list := appsv1.StatefulSetList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/instance",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))
				return list.Items
			},
				"30s", // timeout
				"1s",  // interval
			).Should(gstruct.MatchElements(
				func(interface{}) string { return "each" },
				gstruct.AllowDuplicates,
				gstruct.Elements{
					"each": gomega.WithTransform(
						func(sts appsv1.StatefulSet) int32 {
							return sts.Status.ReadyReplicas
						},
						gomega.BeEquivalentTo(1),
					),
				},
			))

			if test.beforeDelete != nil {
				test.beforeDelete(t, cluster)
			}

			switch test.propagation {
			case metav1.DeletePropagationBackground:
				// Background deletion is the default for custom resources.
				// - https://issue.k8s.io/81628
				assert.NilError(t, cc.Delete(ctx, cluster))
			default:
				assert.NilError(t, cc.Delete(ctx, cluster,
					client.PropagationPolicy(test.propagation)))
			}

			// Stop cluster.
			result := mustReconcile(t, cluster)

			// Replicas should stop first, leaving just the one primary.
			g.Eventually(func() (instances []v1.Pod) {
				if result.Requeue {
					result = mustReconcile(t, cluster)
				}

				list := v1.PodList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/instance",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))
				return list.Items
			},
				"30s", // timeout
				"1s",  // interval
			).Should(gomega.ConsistOf(
				gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
					"ObjectMeta": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
						"Labels": gstruct.MatchKeys(gstruct.IgnoreExtras, gstruct.Keys{
							// Patroni doesn't use "primary" to identify the primary.
							"postgres-operator.crunchydata.com/role": gomega.Equal("master"),
						}),
					}),
				}),
			), "expected one instance")

			// Patroni DCS objects should not be deleted yet.
			{
				list := v1.EndpointsList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/patroni",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))

				assert.Assert(t, len(list.Items) >= 2, // config + leader
					"expected Patroni DCS objects to remain, there are %v",
					len(list.Items))

				// Endpoints are deleted differently than other resources, and
				// Patroni might have recreated them to stay alive. Check that
				// they are all from before the cluster delete operation.
				// - https://issue.k8s.io/99407
				assert.NilError(t,
					cc.Get(ctx, client.ObjectKeyFromObject(cluster), cluster))
				g.Expect(list.Items).To(gstruct.MatchElements(
					func(interface{}) string { return "each" },
					gstruct.AllowDuplicates,
					gstruct.Elements{
						"each": gomega.WithTransform(
							func(ep v1.Endpoints) time.Time {
								return ep.CreationTimestamp.Time
							},
							gomega.BeTemporally("<", cluster.DeletionTimestamp.Time),
						),
					},
				))
			}

			// Continue until cluster is gone.
			g.Eventually(func() error {
				mustReconcile(t, cluster)

				return cc.Get(ctx, client.ObjectKeyFromObject(cluster), cluster)
			},
				"30s", // timeout
				"1s",  // interval
			).Should(
				gomega.SatisfyAll(
					// https://github.com/onsi/gomega/issues/420
					gomega.Not(gomega.BeNil()),
					gomega.WithTransform(apierrors.IsNotFound, gomega.BeTrue()),
				),
			)

			g.Eventually(func() []v1.Endpoints {
				list := v1.EndpointsList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/patroni",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))
				return list.Items
			},
				"10s", // timeout
				"1s",  // interval
			).Should(gomega.BeEmpty(), "Patroni DCS objects should be gone")
		})
	}
}
<commit_msg>Update postgrescluster spec<commit_after>// +build envtest

/*
 Copyright 2021 Crunchy Data Solutions, Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package postgrescluster

import (
	"context"
	"io"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/onsi/gomega"
	"github.com/onsi/gomega/gstruct"
	"go.opentelemetry.io/otel"
	"gotest.tools/v3/assert"
	appsv1 "k8s.io/api/apps/v1"
	v1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/tools/record"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/envtest"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"

	"github.com/crunchydata/postgres-operator/internal/naming"
	"github.com/crunchydata/postgres-operator/internal/patroni"
	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1alpha1"
)

func TestReconcilerHandleDelete(t *testing.T) {
	if !strings.EqualFold(os.Getenv("USE_EXISTING_CLUSTER"), "true") {
		t.Skip("requires a running garbage collection controller")
	}
	t.Parallel()

	ctx := context.Background()
	env := &envtest.Environment{
		CRDDirectoryPaths: []string{
			filepath.Join("..", "..", "..", "config", "crd", "bases"),
		},
	}

	options := client.Options{}
	options.Scheme = runtime.NewScheme()
	assert.NilError(t, scheme.AddToScheme(options.Scheme))
	assert.NilError(t, v1alpha1.AddToScheme(options.Scheme))

	config, err := env.Start()
	assert.NilError(t, err)
	t.Cleanup(func() { assert.Check(t, env.Stop()) })

	cc, err := client.New(config, options)
	assert.NilError(t, err)

	ns := &v1.Namespace{}
	ns.GenerateName = "postgres-operator-test-"
	ns.Labels = labels.Set{"postgres-operator-test": t.Name()}
	assert.NilError(t, cc.Create(ctx, ns))
	t.Cleanup(func() { assert.Check(t, cc.Delete(ctx, ns)) })

	// set the operator namespace variable
	naming.PostgresOperatorNamespace = ns.Name

	// TODO(cbandy): namespace rbac
	assert.NilError(t, cc.Create(ctx, &v1.ServiceAccount{
		ObjectMeta: metav1.ObjectMeta{Namespace: ns.Name, Name: "postgres-operator"},
	}))
	assert.NilError(t, cc.Create(ctx, &rbacv1.RoleBinding{
		ObjectMeta: metav1.ObjectMeta{Namespace: ns.Name, Name: "postgres-operator"},
		RoleRef: rbacv1.RoleRef{
			Kind: "ClusterRole", Name: "postgres-operator-role",
		},
		Subjects: []rbacv1.Subject{{
			Kind: "ServiceAccount", Namespace: ns.Name, Name: "postgres-operator",
		}},
	}))

	reconciler := Reconciler{
		Client:   cc,
		Owner:    client.FieldOwner(t.Name()),
		Recorder: new(record.FakeRecorder),
		Tracer:   otel.Tracer(t.Name()),
	}

	reconciler.PodExec, err = newPodExecutor(config)
	assert.NilError(t, err)

	mustReconcile := func(t *testing.T, cluster *v1alpha1.PostgresCluster) reconcile.Result {
		t.Helper()
		key := client.ObjectKeyFromObject(cluster)
		request := reconcile.Request{NamespacedName: key}
		result, err := reconciler.Reconcile(ctx, request)
		assert.NilError(t, err, "%+v", err)
		return result
	}

	for _, test := range []struct {
		name         string
		beforeDelete func(*testing.T, *v1alpha1.PostgresCluster)
		propagation  metav1.DeletionPropagation
	}{
		{
			name:        "Background",
			propagation: metav1.DeletePropagationBackground,
		},
		// TODO(cbandy): metav1.DeletePropagationForeground
		{
			name:        "AfterFailover",
			propagation: metav1.DeletePropagationBackground,
			beforeDelete: func(t *testing.T, cluster *v1alpha1.PostgresCluster) {
				list := v1.PodList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/instance",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))

				var primary *v1.Pod
				var replica *v1.Pod
				for i := range list.Items {
					if list.Items[i].Labels["postgres-operator.crunchydata.com/role"] == "replica" {
						replica = &list.Items[i]
					} else {
						primary = &list.Items[i]
					}
				}
				assert.Assert(t, primary != nil, "expected to find a primary in %+v", list.Items)
				assert.Assert(t, replica != nil, "expected to find a replica in %+v", list.Items)

				assert.NilError(t, patroni.Executor(func(_ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string) error {
					return reconciler.PodExec(replica.Namespace, replica.Name, "database", stdin, stdout, stderr, command...)
				}).ChangePrimary(ctx, primary.Name, replica.Name))
			},
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			g := gomega.NewWithT(t)

			replicas := int32(2)
			cluster := &v1alpha1.PostgresCluster{
				ObjectMeta: metav1.ObjectMeta{
					Name:      strings.ToLower(test.name),
					Namespace: ns.Name,
				},
				Spec: v1alpha1.PostgresClusterSpec{
					Image:           "gcr.io/crunchy-dev-test/crunchy-postgres-ha:centos8-12.6-multi.dev2",
					PostgresVersion: 12,
					InstanceSets: []v1alpha1.PostgresInstanceSetSpec{{
						Replicas: &replicas,
						VolumeClaimSpec: v1.PersistentVolumeClaimSpec{
							AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
							Resources: v1.ResourceRequirements{
								Requests: map[v1.ResourceName]resource.Quantity{
									v1.ResourceStorage: resource.MustParse("1Gi"),
								},
							},
						},
					}},
				},
			}

			assert.NilError(t, cc.Create(ctx, cluster))

			t.Cleanup(func() {
				// Remove finalizers, if any, so the namespace can terminate.
				assert.Check(t, client.IgnoreNotFound(
					cc.Patch(ctx, cluster, client.RawPatch(
						client.Merge.Type(), []byte(`{"metadata":{"finalizers":[]}}`)))))
			})

			// Start cluster.
			mustReconcile(t, cluster)

			assert.NilError(t,
				cc.Get(ctx, client.ObjectKeyFromObject(cluster), cluster))
			g.Expect(cluster.Finalizers).To(
				gomega.ContainElement("postgres-operator.crunchydata.com/finalizer"),
				"cluster should immediately have a finalizer")

			// Continue until instances are healthy.
			g.Eventually(func() (instances []appsv1.StatefulSet) {
				mustReconcile(t, cluster)

				list := appsv1.StatefulSetList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/instance",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))
				return list.Items
			},
				"60s", // timeout
				"1s",  // interval
			).Should(gstruct.MatchElements(
				func(interface{}) string { return "each" },
				gstruct.AllowDuplicates,
				gstruct.Elements{
					"each": gomega.WithTransform(
						func(sts appsv1.StatefulSet) int32 {
							return sts.Status.ReadyReplicas
						},
						gomega.BeEquivalentTo(1),
					),
				},
			))

			if test.beforeDelete != nil {
				test.beforeDelete(t, cluster)
			}

			switch test.propagation {
			case metav1.DeletePropagationBackground:
				// Background deletion is the default for custom resources.
				// - https://issue.k8s.io/81628
				assert.NilError(t, cc.Delete(ctx, cluster))
			default:
				assert.NilError(t, cc.Delete(ctx, cluster,
					client.PropagationPolicy(test.propagation)))
			}

			// Stop cluster.
			result := mustReconcile(t, cluster)

			// Replicas should stop first, leaving just the one primary.
			g.Eventually(func() (instances []v1.Pod) {
				if result.Requeue {
					result = mustReconcile(t, cluster)
				}

				list := v1.PodList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/instance",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))
				return list.Items
			},
				"60s", // timeout
				"1s",  // interval
			).Should(gomega.ConsistOf(
				gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
					"ObjectMeta": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
						"Labels": gstruct.MatchKeys(gstruct.IgnoreExtras, gstruct.Keys{
							// Patroni doesn't use "primary" to identify the primary.
							"postgres-operator.crunchydata.com/role": gomega.Equal("master"),
						}),
					}),
				}),
			), "expected one instance")

			// Patroni DCS objects should not be deleted yet.
			{
				list := v1.EndpointsList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/patroni",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))

				assert.Assert(t, len(list.Items) >= 2, // config + leader
					"expected Patroni DCS objects to remain, there are %v",
					len(list.Items))

				// Endpoints are deleted differently than other resources, and
				// Patroni might have recreated them to stay alive. Check that
				// they are all from before the cluster delete operation.
				// - https://issue.k8s.io/99407
				assert.NilError(t,
					cc.Get(ctx, client.ObjectKeyFromObject(cluster), cluster))
				g.Expect(list.Items).To(gstruct.MatchElements(
					func(interface{}) string { return "each" },
					gstruct.AllowDuplicates,
					gstruct.Elements{
						"each": gomega.WithTransform(
							func(ep v1.Endpoints) time.Time {
								return ep.CreationTimestamp.Time
							},
							gomega.BeTemporally("<", cluster.DeletionTimestamp.Time),
						),
					},
				))
			}

			// Continue until cluster is gone.
			g.Eventually(func() error {
				mustReconcile(t, cluster)

				return cc.Get(ctx, client.ObjectKeyFromObject(cluster), cluster)
			},
				"60s", // timeout
				"1s",  // interval
			).Should(
				gomega.SatisfyAll(
					// https://github.com/onsi/gomega/issues/420
					gomega.Not(gomega.BeNil()),
					gomega.WithTransform(apierrors.IsNotFound, gomega.BeTrue()),
				),
			)

			g.Eventually(func() []v1.Endpoints {
				list := v1.EndpointsList{}
				selector, err := labels.Parse(strings.Join([]string{
					"postgres-operator.crunchydata.com/cluster=" + cluster.Name,
					"postgres-operator.crunchydata.com/patroni",
				}, ","))
				assert.NilError(t, err)
				assert.NilError(t, cc.List(ctx, &list,
					client.InNamespace(cluster.Namespace),
					client.MatchingLabelsSelector{Selector: selector}))
				return list.Items
			},
				"20s", // timeout
				"1s",  // interval
			).Should(gomega.BeEmpty(), "Patroni DCS objects should be gone")
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-sqlite3"
	"net/http"
	"regexp"
	"strings"
	"time"
)

// regexp POSTGRES_PLACEHOLDERS matches strings '$1', '$2', '$3', ...
var POSTGRES_PLACEHOLDERS *regexp.Regexp = regexp.MustCompile("\\$\\d+")

// Storage is the interface to the SQL database.
type Storage struct {
	driver     string
	dataSource string
	db         *sql.DB
}

// TODO: move call to CREATE_SCHEMA_IF_NOT_EXISTS_SQL to NewServer
func NewStorage(driver string, dataSource string) (*Storage, error) {
	db, err := sql.Open(driver, dataSource)
	if err != nil {
		return nil, err
	}
	storage := &Storage{driver: driver, dataSource: dataSource, db: db}
	_, err = storage.db.Exec(storage.dialectifySchema(CREATE_SCHEMA_IF_NOT_EXISTS_SQL))
	return storage, err
}

// Storage.FindEndpoint returns an endpoint matching the given site and HTTP request.
func (storage *Storage) FindEndpoint(site string, req *http.Request) (endpoint *Endpoint, found bool, err error) {
	endpoints, err := storage.getEndpoints(site)
	if err != nil {
		return nil, false, err
	}
	for _, endpoint := range endpoints {
		if endpoint.Matches(req) {
			return endpoint, true, nil
		}
	}
	return nil, false, nil
}

func (storage *Storage) getEndpoints(site string) ([]*Endpoint, error) {
	endpoints := make([]*Endpoint, 0)
	rows, err := storage.db.Query(storage.dialectifyQuery(GET_SITE_ENDPOINTS_SQL), site)
	if err != nil {
		return endpoints, err
	}
	defer rows.Close()

	for rows.Next() {
		endpoint, err := makeEndpoint(rows)
		if err != nil {
			return endpoints, err
		}
		endpoints = append(endpoints, endpoint)
	}
	return endpoints, rows.Err()
}

func (storage *Storage) dialectifyQuery(sql string) string {
	if storage.isPostgres() {
		return sql
	}
	return POSTGRES_PLACEHOLDERS.ReplaceAllString(sql, "?")
}

func (storage *Storage) isPostgres() bool {
	return storage.driver == "postgres"
}

func (storage *Storage) dialectifySchema(sql string) string {
	if storage.isPostgres() {
		return sql
	}
	// it's not pretty, but it's covered by tests
	REPLACE_ALL := -1
	return strings.Replace(sql, " BYTEA,", " BLOB,", REPLACE_ALL)
}

func makeEndpoint(rows *sql.Rows) (*Endpoint, error) {
	endpoint := &Endpoint{}
	var headersJson string
	var delay int64
	err := rows.Scan(&endpoint.Site, &endpoint.Path, &endpoint.Method, &headersJson, &delay,
		&endpoint.StatusCode, &endpoint.Response)
	if err != nil {
		return endpoint, err
	}
	endpoint.Delay = time.Duration(delay)
	endpoint.Headers, err = jsonToStringMap(headersJson)
	return endpoint, err
}

func jsonToStringMap(js string) (map[string]string, error) {
	object := make(map[string]interface{})
	err := json.Unmarshal([]byte(js), &object)
	if err != nil {
		return nil, err
	}
	return objectToStringMap(object)
}

func objectToStringMap(object map[string]interface{}) (map[string]string, error) {
	m := make(map[string]string)
	for key, value := range object {
		switch value.(type) {
		case string:
			m[key] = value.(string)
		default:
			return nil, fmt.Errorf("Expecting string, got %+v", value)
		}
	}
	return m, nil
}

// Storage.SaveEndpoint upserts the given endpoint into a database.
func (storage *Storage) SaveEndpoint(endpoint *Endpoint) error {
	tx, err := storage.db.Begin()
	if err != nil {
		return err
	}
	// If tx is commited, then tx.Rollback() basically has no effect.
	// If there's some error and tx isn't commited, then we want to rollback.
	defer tx.Rollback()
	// upsert as delete-and-insert isn't correct in all cases
	// (e.g concurrent upserts of the same endpoint will lead to "duplicate key value violates unique constraint")
	// but is practical enough, because concurrent upserts of the same endpoint are going to be extremely rare
	_, err = tx.Exec(storage.dialectifyQuery(DELETE_ENDPOINT_SQL),
		endpoint.Site, endpoint.Path, endpoint.Method)
	if err != nil {
		return err
	}
	headersJson, err := stringMapToJson(endpoint.Headers)
	if err != nil {
		return err
	}
	_, err = tx.Exec(storage.dialectifyQuery(INSERT_ENDPOINT_SQL),
		endpoint.Site, endpoint.Path, endpoint.Method,
		headersJson, int64(endpoint.Delay), endpoint.StatusCode, endpoint.Response)
	if err != nil {
		return err
	}
	return tx.Commit()
}

func stringMapToJson(m map[string]string) (string, error) {
	jsonBytes, err := json.Marshal(m)
	return string(jsonBytes), err
}

// Storage.CreateSite returns an error if the given site already exists in a database.
func (storage *Storage) CreateSite(site string) error {
	_, err := storage.db.Exec(storage.dialectifyQuery(INSERT_SITE_SQL), site)
	return err
}

// TODO: generalize (e.g storage.HasResults(sql string))
func (storage *Storage) SiteExists(site string) (bool, error) {
	rows, err := storage.db.Query(storage.dialectifyQuery(GET_SITE_SQL), site)
	if err != nil {
		return false, err
	}
	defer rows.Close()
	siteExists := rows.Next()
	return siteExists, rows.Err()
}
<commit_msg>Add method Storage.HasResults<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-sqlite3"
	"net/http"
	"regexp"
	"strings"
	"time"
)

// regexp POSTGRES_PLACEHOLDERS matches strings '$1', '$2', '$3', ...
var POSTGRES_PLACEHOLDERS *regexp.Regexp = regexp.MustCompile("\\$\\d+")

// Storage is the interface to the SQL database.
type Storage struct {
	driver     string
	dataSource string
	db         *sql.DB
}

// TODO: move call to CREATE_SCHEMA_IF_NOT_EXISTS_SQL to NewServer
func NewStorage(driver string, dataSource string) (*Storage, error) {
	db, err := sql.Open(driver, dataSource)
	if err != nil {
		return nil, err
	}
	storage := &Storage{driver: driver, dataSource: dataSource, db: db}
	_, err = storage.db.Exec(storage.dialectifySchema(CREATE_SCHEMA_IF_NOT_EXISTS_SQL))
	return storage, err
}

// Storage.FindEndpoint returns an endpoint matching the given site and HTTP request.
func (storage *Storage) FindEndpoint(site string, req *http.Request) (endpoint *Endpoint, found bool, err error) {
	endpoints, err := storage.getEndpoints(site)
	if err != nil {
		return nil, false, err
	}
	for _, endpoint := range endpoints {
		if endpoint.Matches(req) {
			return endpoint, true, nil
		}
	}
	return nil, false, nil
}

func (storage *Storage) getEndpoints(site string) ([]*Endpoint, error) {
	endpoints := make([]*Endpoint, 0)
	rows, err := storage.db.Query(storage.dialectifyQuery(GET_SITE_ENDPOINTS_SQL), site)
	if err != nil {
		return endpoints, err
	}
	defer rows.Close()

	for rows.Next() {
		endpoint, err := makeEndpoint(rows)
		if err != nil {
			return endpoints, err
		}
		endpoints = append(endpoints, endpoint)
	}
	return endpoints, rows.Err()
}

func (storage *Storage) dialectifyQuery(sql string) string {
	if storage.isPostgres() {
		return sql
	}
	return POSTGRES_PLACEHOLDERS.ReplaceAllString(sql, "?")
}

func (storage *Storage) isPostgres() bool {
	return storage.driver == "postgres"
}

func (storage *Storage) dialectifySchema(sql string) string {
	if storage.isPostgres() {
		return sql
	}
	// it's not pretty, but it's covered by tests
	REPLACE_ALL := -1
	return strings.Replace(sql, " BYTEA,", " BLOB,", REPLACE_ALL)
}

func makeEndpoint(rows *sql.Rows) (*Endpoint, error) {
	endpoint := &Endpoint{}
	var headersJson string
	var delay int64
	err := rows.Scan(&endpoint.Site, &endpoint.Path, &endpoint.Method, &headersJson, &delay,
		&endpoint.StatusCode, &endpoint.Response)
	if err != nil {
		return endpoint, err
	}
	endpoint.Delay = time.Duration(delay)
	endpoint.Headers, err = jsonToStringMap(headersJson)
	return endpoint, err
}

func jsonToStringMap(js string) (map[string]string, error) {
	object := make(map[string]interface{})
	err := json.Unmarshal([]byte(js), &object)
	if err != nil {
		return nil, err
	}
	return objectToStringMap(object)
}

func objectToStringMap(object map[string]interface{}) (map[string]string, error) {
	m := make(map[string]string)
	for key, value := range object {
		switch value.(type) {
		case string:
			m[key] = value.(string)
		default:
			return nil, fmt.Errorf("Expecting string, got %+v", value)
		}
	}
	return m, nil
}

// Storage.SaveEndpoint upserts the given endpoint into a database.
func (storage *Storage) SaveEndpoint(endpoint *Endpoint) error {
	tx, err := storage.db.Begin()
	if err != nil {
		return err
	}
	// If tx is commited, then tx.Rollback() basically has no effect.
	// If there's some error and tx isn't commited, then we want to rollback.
	defer tx.Rollback()
	// upsert as delete-and-insert isn't correct in all cases
	// (e.g concurrent upserts of the same endpoint will lead to "duplicate key value violates unique constraint")
	// but is practical enough, because concurrent upserts of the same endpoint are going to be extremely rare
	_, err = tx.Exec(storage.dialectifyQuery(DELETE_ENDPOINT_SQL),
		endpoint.Site, endpoint.Path, endpoint.Method)
	if err != nil {
		return err
	}
	headersJson, err := stringMapToJson(endpoint.Headers)
	if err != nil {
		return err
	}
	_, err = tx.Exec(storage.dialectifyQuery(INSERT_ENDPOINT_SQL),
		endpoint.Site, endpoint.Path, endpoint.Method,
		headersJson, int64(endpoint.Delay), endpoint.StatusCode, endpoint.Response)
	if err != nil {
		return err
	}
	return tx.Commit()
}

func stringMapToJson(m map[string]string) (string, error) {
	jsonBytes, err := json.Marshal(m)
	return string(jsonBytes), err
}

// Storage.CreateSite returns an error if the given site already exists in a database.
func (storage *Storage) CreateSite(site string) error {
	_, err := storage.db.Exec(storage.dialectifyQuery(INSERT_SITE_SQL), site)
	return err
}

func (storage *Storage) SiteExists(site string) (bool, error) {
	return storage.HasResults(GET_SITE_SQL, site)
}

func (storage *Storage) HasResults(sql string, args ...interface{}) (bool, error) {
	rows, err := storage.db.Query(storage.dialectifyQuery(sql), args...)
	if err != nil {
		return false, err
	}
	defer rows.Close()
	hasResults := rows.Next()
	return hasResults, rows.Err()
}
<|endoftext|>


<commit_before>package main

import (
    "github.com/google/go-github/github"
)

func hasChanges(pr1 *github.PullRequest, pr2 *github.PullRequest) bool {
    return pr1.Head.SHA != pr2.Head.SHA || pr1.Base.SHA != pr2.Base.SHA
}

type Storage struct {
    All map[int]*github.PullRequest
}

func (self Storage) Has(key int) bool {
    _, ok := self.All[key];

    return ok
}

func (self *Storage) Filter(pullRequests []*github.PullRequest) []*github.PullRequest {
    var out []*github.PullRequest

    for _, pr := range pullRequests {
        if self.Has(*pr.Number) != true {
            self.All[*pr.Number] = pr
            out = append(out, pr)
        } else {
            if hasChanges(self.All[*pr.Number], pr) == true {
                self.All[*pr.Number] = pr
                out = append(out, pr)
            }
        }
    }

    return out
}
<commit_msg>Compare values instead references to check SHA<commit_after>package main

import (
    "github.com/google/go-github/github"
)

func hasChanges(pr1 *github.PullRequest, pr2 *github.PullRequest) bool {
    return *pr1.Head.SHA != *pr2.Head.SHA || *pr1.Base.SHA != *pr2.Base.SHA
}

type Storage struct {
    All map[int]*github.PullRequest
}

func (self Storage) Has(key int) bool {
    _, ok := self.All[key];

    return ok
}

func (self *Storage) Filter(pullRequests []*github.PullRequest) []*github.PullRequest {
    var out []*github.PullRequest

    for _, pr := range pullRequests {
        if self.Has(*pr.Number) != true {
            self.All[*pr.Number] = pr
            out = append(out, pr)
        } else {
            if hasChanges(self.All[*pr.Number], pr) == true {
                self.All[*pr.Number] = pr
                out = append(out, pr)
            }
        }
    }

    return out
}
<|endoftext|>