<commit_before>package main

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"image"
	"image/gif"
	"image/jpeg"
	"image/png"
	"io"
	"io/ioutil"
	"net/http"
	"regexp"

	"github.com/pkg/errors"

	log "github.com/Sirupsen/logrus"
	"github.com/google/go-github/github"
	"github.com/lelenanam/downsize"
	"github.com/lelenanam/screenshot"
)

//StartCutiePullReq is the number of pull request
//where new template (with a picture of a cute animal) was added
const StartCutiePullReq = 20514

//Repo is repository with docker cuties
const Repo = "docker"

//Owner of repository
const Owner = "docker"

//SearchIssuesLimit is github limit for Search.Issues results
const SearchIssuesLimit = 1000

//PerPage number of pull requests per page
const PerPage = 50

//TwitterUser is account for docker cuties in twitter
const TwitterUser = "DockerCuties"

//ProjectOwner is a twitter account for notifications
const ProjectOwner = "lelenanam"

//TwitterUploadLimit is limit for media upload in bytes
const TwitterUploadLimit = 3145728

// GetURLFromPull parses the body of the pull request and return last image URL if found
// try to find flickr first
func GetURLFromPull(pull *github.Issue) string {
	str := *pull.Body
	res := ""

	// Example:
	// [![kitteh](https://c2.staticflickr.com/4/3147/2567501805_17ee8fd947_z.jpg)](https://flic.kr/p/4UT7Qv)
	flickre := regexp.MustCompile(`\[!\[.*\]\((.*)\)\]\(.*\)`)
	flickResult := flickre.FindAllStringSubmatch(str, -1)

	if len(flickResult) > 0 {
		lastres := flickResult[len(flickResult)-1]
		if len(lastres) > 1 {
			res := lastres[len(lastres)-1]
			return res
		}
	}

	// Example:
	// ![image](https://cloud.githubusercontent.com/assets/2367858/23283487/02bb756e-f9db-11e6-9aa8-5f3e1bb80df3.png)
	imagere := regexp.MustCompile(`!\[.*\]\((.*)\)`)
	imageResult := imagere.FindAllStringSubmatch(str, -1)

	if len(imageResult) > 0 {
		lastres := imageResult[len(imageResult)-1]
		if len(lastres) > 1 {
			res := lastres[len(lastres)-1]
			return res
		}
	}
	return res
}

// GetImageFromURL downloads an image from url and returns image img, its size, format and error
// for animated gif format returns data in gifByte slice
func GetImageFromURL(url string) (img image.Image, format string, size int, gifByte []byte, err error) {
	log.WithFields(log.Fields{"URL": url}).Debug("Download")
	res, err := http.Get(url)
	if err != nil {
		log.WithFields(log.Fields{"URL": url}).WithError(err).Error("Cannot download")
		return nil, "", 0, nil, err
	}
	log.WithFields(log.Fields{"Content Length": res.ContentLength, "Status Code": res.StatusCode}).Debug("Got")

	defer func() {
		if err := res.Body.Close(); err != nil {
			log.WithFields(log.Fields{"URL": url}).WithError(err).Error("Cannot close body")
		}
	}()

	var reader io.Reader = res.Body

	if res.Header["Content-Type"][0] == "image/gif" {
		gifByte, err = ioutil.ReadAll(res.Body)
		if err != nil {
			log.WithFields(log.Fields{"URL": url}).WithError(err).Error("Cannot read body")
			return nil, "gif", 0, nil, err
		}
		reader = bytes.NewReader(gifByte)
	}

	img, format, err = image.Decode(reader)
	if err != nil {
		log.WithFields(log.Fields{"URL": url}).WithError(err).Error("Cannot decode image")
		return nil, "", 0, nil, err
	}

	log.WithFields(log.Fields{"format": format, "size": int(res.ContentLength)}).Debug("Image decoded")
	return img, format, int(res.ContentLength), gifByte, nil
}

// GetStringFromImage returns string of cutie image img or error
func GetStringFromImage(img image.Image, format string, size int, gifByte []byte) (string, error) {
	b := bytes.NewBuffer(nil)
	encoder := base64.NewEncoder(base64.StdEncoding, b)

	// Need to resize image
	if size >= TwitterUploadLimit || size < 0 {
		log.WithFields(log.Fields{"Twitter upload limit": TwitterUploadLimit}).Debug("Downsize image")
		opts := &downsize.Options{Size: TwitterUploadLimit, Format: format}
		err := downsize.Encode(encoder, img, opts)
		if err != nil {
			log.WithFields(log.Fields{"format": format}).WithError(err).Error("Cannot downsize image")
			return "", err
		}
	} else {
		if format == "gif" {
			gifReader := bytes.NewReader(gifByte)
			_, err := io.Copy(encoder, gifReader)
			if err != nil {
				log.WithFields(log.Fields{"format": format}).WithError(err).Error("Cannot copy gif image")
				return "", err
			}
		} else {
			// no need to resize, just encode
			if err := ImageEncode(encoder, img, format); err != nil {
				log.WithFields(log.Fields{"format": format}).WithError(err).Error("Cannot encode image")
				return "", err
			}
		}
	}

	if err := encoder.Close(); err != nil {
		log.WithError(err).Error("Cannot close encoder")
		return "", err
	}

	if b.Len() == 0 {
		log.Warn("Empty image data")
		return "", nil
	}
	return b.String(), nil
}

var errIsScreenshot = errors.New("picture is screenshot")
var errImageNotFound = errors.New("picture not found")

// GetCutieFromPull returns string of cutie image from pull request pull
func GetCutieFromPull(pull *github.Issue) (string, error) {
	url := GetURLFromPull(pull)
	if url == "" {
		return "", errImageNotFound
	}
	img, format, size, gifByte, err := GetImageFromURL(url)
	if err != nil {
		return "", errors.Wrap(err, "cannot get image from URL")
	}
	if screenshot.Detect(img) {
		return "", errIsScreenshot
	}
	str, err := GetStringFromImage(img, format, size, gifByte)
	if err != nil {
		return "", errors.Wrap(err, "cannot get string for image")
	}
	return str, nil
}

// ImageEncode encodes image m with format to writer w
func ImageEncode(w io.Writer, m image.Image, format string) error {
	switch format {
	case "jpeg":
		return jpeg.Encode(w, m, &jpeg.Options{Quality: 95})
	case "png":
		return png.Encode(w, m)
	case "gif":
		return gif.Encode(w, m, nil)
	default:
		return fmt.Errorf("Unknown image format %q", format)
	}
}
<commit_msg>Rename docker to moby<commit_after>package main

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"image"
	"image/gif"
	"image/jpeg"
	"image/png"
	"io"
	"io/ioutil"
	"net/http"
	"regexp"

	"github.com/pkg/errors"

	log "github.com/Sirupsen/logrus"
	"github.com/google/go-github/github"
	"github.com/lelenanam/downsize"
	"github.com/lelenanam/screenshot"
)

//StartCutiePullReq is the number of pull request
//where new template (with a picture of a cute animal) was added
const StartCutiePullReq = 20514

//Repo is repository with docker cuties
const Repo = "moby"

//Owner of repository
const Owner = "moby"

//SearchIssuesLimit is github limit for Search.Issues results
const SearchIssuesLimit = 1000

//PerPage number of pull requests per page
const PerPage = 50

//TwitterUser is account for docker cuties in twitter
const TwitterUser = "DockerCuties"

//ProjectOwner is a twitter account for notifications
const ProjectOwner = "lelenanam"

//TwitterUploadLimit is limit for media upload in bytes
const TwitterUploadLimit = 3145728

// GetURLFromPull parses the body of the pull request and return last image URL if found
// try to find flickr first
func GetURLFromPull(pull *github.Issue) string {
	str := *pull.Body
	res := ""

	// Example:
	// [![kitteh](https://c2.staticflickr.com/4/3147/2567501805_17ee8fd947_z.jpg)](https://flic.kr/p/4UT7Qv)
	flickre := regexp.MustCompile(`\[!\[.*\]\((.*)\)\]\(.*\)`)
	flickResult := flickre.FindAllStringSubmatch(str, -1)

	if len(flickResult) > 0 {
		lastres := flickResult[len(flickResult)-1]
		if len(lastres) > 1 {
			res := lastres[len(lastres)-1]
			return res
		}
	}

	// Example:
	// ![image](https://cloud.githubusercontent.com/assets/2367858/23283487/02bb756e-f9db-11e6-9aa8-5f3e1bb80df3.png)
	imagere := regexp.MustCompile(`!\[.*\]\((.*)\)`)
	imageResult := imagere.FindAllStringSubmatch(str, -1)

	if len(imageResult) > 0 {
		lastres := imageResult[len(imageResult)-1]
		if len(lastres) > 1 {
			res := lastres[len(lastres)-1]
			return res
		}
	}
	return res
}

// GetImageFromURL downloads an image from url and returns image img, its size, format and error
// for animated gif format returns data in gifByte slice
func GetImageFromURL(url string) (img image.Image, format string, size int, gifByte []byte, err error) {
	log.WithFields(log.Fields{"URL": url}).Debug("Download")
	res, err := http.Get(url)
	if err != nil {
		log.WithFields(log.Fields{"URL": url}).WithError(err).Error("Cannot download")
		return nil, "", 0, nil, err
	}
	log.WithFields(log.Fields{"Content Length": res.ContentLength, "Status Code": res.StatusCode}).Debug("Got")

	defer func() {
		if err := res.Body.Close(); err != nil {
			log.WithFields(log.Fields{"URL": url}).WithError(err).Error("Cannot close body")
		}
	}()

	var reader io.Reader = res.Body

	if res.Header["Content-Type"][0] == "image/gif" {
		gifByte, err = ioutil.ReadAll(res.Body)
		if err != nil {
			log.WithFields(log.Fields{"URL": url}).WithError(err).Error("Cannot read body")
			return nil, "gif", 0, nil, err
		}
		reader = bytes.NewReader(gifByte)
	}

	img, format, err = image.Decode(reader)
	if err != nil {
		log.WithFields(log.Fields{"URL": url}).WithError(err).Error("Cannot decode image")
		return nil, "", 0, nil, err
	}

	log.WithFields(log.Fields{"format": format, "size": int(res.ContentLength)}).Debug("Image decoded")
	return img, format, int(res.ContentLength), gifByte, nil
}

// GetStringFromImage returns string of cutie image img or error
func GetStringFromImage(img image.Image, format string, size int, gifByte []byte) (string, error) {
	b := bytes.NewBuffer(nil)
	encoder := base64.NewEncoder(base64.StdEncoding, b)

	// Need to resize image
	if size >= TwitterUploadLimit || size < 0 {
		log.WithFields(log.Fields{"Twitter upload limit": TwitterUploadLimit}).Debug("Downsize image")
		opts := &downsize.Options{Size: TwitterUploadLimit, Format: format}
		err := downsize.Encode(encoder, img, opts)
		if err != nil {
			log.WithFields(log.Fields{"format": format}).WithError(err).Error("Cannot downsize image")
			return "", err
		}
	} else {
		if format == "gif" {
			gifReader := bytes.NewReader(gifByte)
			_, err := io.Copy(encoder, gifReader)
			if err != nil {
				log.WithFields(log.Fields{"format": format}).WithError(err).Error("Cannot copy gif image")
				return "", err
			}
		} else {
			// no need to resize, just encode
			if err := ImageEncode(encoder, img, format); err != nil {
				log.WithFields(log.Fields{"format": format}).WithError(err).Error("Cannot encode image")
				return "", err
			}
		}
	}

	if err := encoder.Close(); err != nil {
		log.WithError(err).Error("Cannot close encoder")
		return "", err
	}

	if b.Len() == 0 {
		log.Warn("Empty image data")
		return "", nil
	}
	return b.String(), nil
}

var errIsScreenshot = errors.New("picture is screenshot")
var errImageNotFound = errors.New("picture not found")

// GetCutieFromPull returns string of cutie image from pull request pull
func GetCutieFromPull(pull *github.Issue) (string, error) {
	url := GetURLFromPull(pull)
	if url == "" {
		return "", errImageNotFound
	}
	img, format, size, gifByte, err := GetImageFromURL(url)
	if err != nil {
		return "", errors.Wrap(err, "cannot get image from URL")
	}
	if screenshot.Detect(img) {
		return "", errIsScreenshot
	}
	str, err := GetStringFromImage(img, format, size, gifByte)
	if err != nil {
		return "", errors.Wrap(err, "cannot get string for image")
	}
	return str, nil
}

// ImageEncode encodes image m with format to writer w
func ImageEncode(w io.Writer, m image.Image, format string) error {
	switch format {
	case "jpeg":
		return jpeg.Encode(w, m, &jpeg.Options{Quality: 95})
	case "png":
		return png.Encode(w, m)
	case "gif":
		return gif.Encode(w, m, nil)
	default:
		return fmt.Errorf("Unknown image format %q", format)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/xml"
	"flag"
	"fmt"
	"github.com/scottdware/go-requestor"
	"os"
	"strings"
)

type CVE struct {
	Title       string `xml:"title"`
	Link        string `xml:"link"`
	Description string `xml:"description"`
	Date        string `xml:"date"`
}

type Vulnerabilities struct {
	XMLName xml.Name `xml:"RDF"`
	CVEs    []CVE    `xml:"item"`
}

var (
	rssNormal   = "https://nvd.nist.gov/download/nvd-rss.xml"
	rssAnalyzed = "https://nvd.nist.gov/download/nvd-rss-analyzed.xml"
	vulns       Vulnerabilities
	list        bool
	cve         string
	analyzed    bool
)

func init() {
	flag.Usage = func() {
		fmt.Println("cveget - List recent CVE's and view information about them.\n")
		fmt.Println("Usage: cveget [OPTIONS]\n")
		flag.PrintDefaults()
		os.Exit(0)
	}
	flag.BoolVar(&list, "list", false, "List all CVE's within the previous eight days.")
	flag.BoolVar(&analyzed, "analyzed", false, "Provides only vulnerabilities which have been analyzed within the previous eight days.")
	flag.StringVar(&cve, "cve", "", "Specify a CVE to view information on.")
	flag.Parse()

	if !list && cve == "" {
		flag.Usage()
	}
}

func main() {
	feedData := requestor.Send(rssNormal, nil)
	if feedData.Error != nil {
		fmt.Println(feedData.Error)
	}

	if analyzed {
		feedData = requestor.Send(rssAnalyzed, nil)
		if feedData.Error != nil {
			fmt.Println(feedData.Error)
		}
	}

	if err := xml.Unmarshal(feedData.Body, &vulns); err != nil {
		fmt.Println(err)
	}

	for _, c := range vulns.CVEs {
		if list {
			fmt.Println(c.Title)
		}

		if !list {
			if strings.Contains(c.Title, cve) {
				fmt.Printf("%s\n\n", c.Title)
				fmt.Printf("Published: %s\n", c.Date)
				fmt.Printf("%s\n\n", c.Link)
				fmt.Printf("%s\n\n", c.Description)
			}
		}
	}
}
<commit_msg>Renamed package<commit_after>package main

import (
	"encoding/xml"
	"flag"
	"fmt"
	"github.com/scottdware/go-rested"
	"os"
	"strings"
)

type CVE struct {
	Title       string `xml:"title"`
	Link        string `xml:"link"`
	Description string `xml:"description"`
	Date        string `xml:"date"`
}

type Vulnerabilities struct {
	XMLName xml.Name `xml:"RDF"`
	CVEs    []CVE    `xml:"item"`
}

var (
	rssNormal   = "https://nvd.nist.gov/download/nvd-rss.xml"
	rssAnalyzed = "https://nvd.nist.gov/download/nvd-rss-analyzed.xml"
	vulns       Vulnerabilities
	list        bool
	cve         string
	analyzed    bool
)

func init() {
	flag.Usage = func() {
		fmt.Println("cveget - List recent CVE's and view information about them.\n")
		fmt.Println("Usage: cveget [OPTIONS]\n")
		flag.PrintDefaults()
		os.Exit(0)
	}
	flag.BoolVar(&list, "list", false, "List all CVE's within the previous eight days.")
	flag.BoolVar(&analyzed, "analyzed", false, "Provides only vulnerabilities which have been analyzed within the previous eight days.")
	flag.StringVar(&cve, "cve", "", "Specify a CVE to view information on.")
	flag.Parse()

	if !list && cve == "" {
		flag.Usage()
	}
}

func main() {
	feedData := rested.Send(rssNormal, nil)
	if feedData.Error != nil {
		fmt.Println(feedData.Error)
	}

	if analyzed {
		feedData = rested.Send(rssAnalyzed, nil)
		if feedData.Error != nil {
			fmt.Println(feedData.Error)
		}
	}

	if err := xml.Unmarshal(feedData.Body, &vulns); err != nil {
		fmt.Println(err)
	}

	for _, c := range vulns.CVEs {
		if list {
			fmt.Println(c.Title)
		}

		if !list {
			if strings.Contains(c.Title, cve) {
				fmt.Printf("%s\n\n", c.Title)
				fmt.Printf("Published: %s\n", c.Date)
				fmt.Printf("%s\n\n", c.Link)
				fmt.Printf("%s\n\n", c.Description)
			}
		}
	}
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"io"
	"sync"

	"github.com/gogo/protobuf/types"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/admin"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/log"
)

type apiServer struct {
	log.Logger
	address        string
	pachClient     *client.APIClient
	pachClientOnce sync.Once
}

func (a *apiServer) Extract(request *admin.ExtractRequest, extractServer admin.API_ExtractServer) error {
	pachClient, err := a.getPachClient()
	if err != nil {
		return err
	}
	ris, err := pachClient.ListRepo(nil)
	if err != nil {
		return err
	}
	for _, ri := range ris {
		if len(ri.Provenance) > 0 {
			continue
		}
		if err := extractServer.Send(&admin.Op{
			Repo: &pfs.CreateRepoRequest{
				Repo:        ri.Repo,
				Provenance:  ri.Provenance,
				Description: ri.Description,
			},
		}); err != nil {
			return err
		}
		cis, err := pachClient.ListCommit(ri.Repo.Name, "", "", 0)
		if err != nil {
			return err
		}
		for _, ci := range sortCommitInfos(cis) {
			// Even without a parent, ParentCommit is used to indicate which
			// repo to make the commit in.
			if ci.ParentCommit == nil {
				ci.ParentCommit = client.NewCommit(ri.Repo.Name, "")
			}
			if err := extractServer.Send(&admin.Op{
				Commit: &pfs.BuildCommitRequest{
					Parent: ci.ParentCommit,
					Tree:   ci.Tree,
					ID:     ci.Commit.ID,
				},
			}); err != nil {
				return err
			}
		}
		bis, err := pachClient.ListBranch(ri.Repo.Name)
		if err != nil {
			return err
		}
		for _, bi := range bis {
			if err := extractServer.Send(&admin.Op{
				Branch: &pfs.SetBranchRequest{
					Commit: bi.Head,
					Branch: bi.Name,
				},
			},
			); err != nil {
				return err
			}
		}
	}
	pis, err := pachClient.ListPipeline()
	if err != nil {
		return err
	}
	for _, pi := range pis {
		if err := extractServer.Send(&admin.Op{
			Pipeline: &pps.CreatePipelineRequest{
				Pipeline:           pi.Pipeline,
				Transform:          pi.Transform,
				ParallelismSpec:    pi.ParallelismSpec,
				Egress:             pi.Egress,
				OutputBranch:       pi.OutputBranch,
				ScaleDownThreshold: pi.ScaleDownThreshold,
				ResourceRequests:   pi.ResourceRequests,
				ResourceLimits:     pi.ResourceLimits,
				Input:              pi.Input,
				Description:        pi.Description,
				Incremental:        pi.Incremental,
				CacheSize:          pi.CacheSize,
				EnableStats:        pi.EnableStats,
				Batch:              pi.Batch,
				MaxQueueSize:       pi.MaxQueueSize,
				Service:            pi.Service,
				ChunkSpec:          pi.ChunkSpec,
				DatumTimeout:       pi.DatumTimeout,
				JobTimeout:         pi.JobTimeout,
				Salt:               pi.Salt,
			},
		}); err != nil {
			return err
		}
	}
	return nil
}

func sortCommitInfos(cis []*pfs.CommitInfo) []*pfs.CommitInfo {
	commitMap := make(map[string]*pfs.CommitInfo)
	for _, ci := range cis {
		commitMap[ci.Commit.ID] = ci
	}
	var result []*pfs.CommitInfo
	for _, ci := range cis {
		if commitMap[ci.Commit.ID] == nil {
			continue
		}
		var localResult []*pfs.CommitInfo
		for ci != nil {
			localResult = append(localResult, ci)
			delete(commitMap, ci.Commit.ID)
			if ci.ParentCommit != nil {
				ci = commitMap[ci.ParentCommit.ID]
			} else {
				ci = nil
			}
		}
		for i := range localResult {
			result = append(result, localResult[len(localResult)-i-1])
		}
	}
	return result
}

func (a *apiServer) Restore(restoreServer admin.API_RestoreServer) (retErr error) {
	ctx := restoreServer.Context()
	pachClient, err := a.getPachClient()
	if err != nil {
		return err
	}
	defer func() {
		for {
			_, err := restoreServer.Recv()
			if err != nil {
				break
			}
		}
		if err := restoreServer.SendAndClose(&types.Empty{}); err != nil && retErr == nil {
			retErr = err
		}
	}()
	for {
		req, err := restoreServer.Recv()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		op := req.Op
		switch {
		case op.Object != nil:
		case op.Repo != nil:
			if _, err := pachClient.PfsAPIClient.CreateRepo(ctx, op.Repo); err != nil {
				return fmt.Errorf("error creating repo: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Commit != nil:
			if _, err := pachClient.PfsAPIClient.BuildCommit(ctx, op.Commit); err != nil {
				return fmt.Errorf("error creating commit: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Branch != nil:
			if _, err := pachClient.PfsAPIClient.SetBranch(ctx, op.Branch); err != nil {
				return fmt.Errorf("error creating branch: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Pipeline != nil:
			if _, err := pachClient.PpsAPIClient.CreatePipeline(ctx, op.Pipeline); err != nil {
				return fmt.Errorf("error creating pipeline: %v", grpcutil.ScrubGRPC(err))
			}
		}
	}
	return nil
}

func (a *apiServer) getPachClient() (*client.APIClient, error) {
	if a.pachClient == nil {
		var onceErr error
		a.pachClientOnce.Do(func() {
			a.pachClient, onceErr = client.NewFromAddress(a.address)
		})
		if onceErr != nil {
			return nil, onceErr
		}
	}
	return a.pachClient, nil
}
<commit_msg>Extract a version along with other data.<commit_after>package server

import (
	"fmt"
	"io"
	"sync"

	"github.com/gogo/protobuf/types"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/admin"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/log"
)

type apiServer struct {
	log.Logger
	address        string
	pachClient     *client.APIClient
	pachClientOnce sync.Once
}

func (a *apiServer) Extract(request *admin.ExtractRequest, extractServer admin.API_ExtractServer) error {
	pachClient, err := a.getPachClient()
	if err != nil {
		return err
	}
	pachClient = pachClient.WithCtx(extractServer.Context())
	v, err := pachClient.VersionAPIClient.GetVersion(pachClient.Ctx(), &types.Empty{})
	if err != nil {
		return err
	}
	if err := extractServer.Send(&admin.Op{
		Version: v,
	}); err != nil {
		return err
	}
	ris, err := pachClient.ListRepo(nil)
	if err != nil {
		return err
	}
	for _, ri := range ris {
		if len(ri.Provenance) > 0 {
			continue
		}
		if err := extractServer.Send(&admin.Op{
			Repo: &pfs.CreateRepoRequest{
				Repo:        ri.Repo,
				Provenance:  ri.Provenance,
				Description: ri.Description,
			},
		}); err != nil {
			return err
		}
		cis, err := pachClient.ListCommit(ri.Repo.Name, "", "", 0)
		if err != nil {
			return err
		}
		for _, ci := range sortCommitInfos(cis) {
			// Even without a parent, ParentCommit is used to indicate which
			// repo to make the commit in.
			if ci.ParentCommit == nil {
				ci.ParentCommit = client.NewCommit(ri.Repo.Name, "")
			}
			if err := extractServer.Send(&admin.Op{
				Commit: &pfs.BuildCommitRequest{
					Parent: ci.ParentCommit,
					Tree:   ci.Tree,
					ID:     ci.Commit.ID,
				},
			}); err != nil {
				return err
			}
		}
		bis, err := pachClient.ListBranch(ri.Repo.Name)
		if err != nil {
			return err
		}
		for _, bi := range bis {
			if err := extractServer.Send(&admin.Op{
				Branch: &pfs.SetBranchRequest{
					Commit: bi.Head,
					Branch: bi.Name,
				},
			},
			); err != nil {
				return err
			}
		}
	}
	pis, err := pachClient.ListPipeline()
	if err != nil {
		return err
	}
	for _, pi := range pis {
		if err := extractServer.Send(&admin.Op{
			Pipeline: &pps.CreatePipelineRequest{
				Pipeline:           pi.Pipeline,
				Transform:          pi.Transform,
				ParallelismSpec:    pi.ParallelismSpec,
				Egress:             pi.Egress,
				OutputBranch:       pi.OutputBranch,
				ScaleDownThreshold: pi.ScaleDownThreshold,
				ResourceRequests:   pi.ResourceRequests,
				ResourceLimits:     pi.ResourceLimits,
				Input:              pi.Input,
				Description:        pi.Description,
				Incremental:        pi.Incremental,
				CacheSize:          pi.CacheSize,
				EnableStats:        pi.EnableStats,
				Batch:              pi.Batch,
				MaxQueueSize:       pi.MaxQueueSize,
				Service:            pi.Service,
				ChunkSpec:          pi.ChunkSpec,
				DatumTimeout:       pi.DatumTimeout,
				JobTimeout:         pi.JobTimeout,
				Salt:               pi.Salt,
			},
		}); err != nil {
			return err
		}
	}
	return nil
}

func sortCommitInfos(cis []*pfs.CommitInfo) []*pfs.CommitInfo {
	commitMap := make(map[string]*pfs.CommitInfo)
	for _, ci := range cis {
		commitMap[ci.Commit.ID] = ci
	}
	var result []*pfs.CommitInfo
	for _, ci := range cis {
		if commitMap[ci.Commit.ID] == nil {
			continue
		}
		var localResult []*pfs.CommitInfo
		for ci != nil {
			localResult = append(localResult, ci)
			delete(commitMap, ci.Commit.ID)
			if ci.ParentCommit != nil {
				ci = commitMap[ci.ParentCommit.ID]
			} else {
				ci = nil
			}
		}
		for i := range localResult {
			result = append(result, localResult[len(localResult)-i-1])
		}
	}
	return result
}

func (a *apiServer) Restore(restoreServer admin.API_RestoreServer) (retErr error) {
	ctx := restoreServer.Context()
	pachClient, err := a.getPachClient()
	if err != nil {
		return err
	}
	defer func() {
		for {
			_, err := restoreServer.Recv()
			if err != nil {
				break
			}
		}
		if err := restoreServer.SendAndClose(&types.Empty{}); err != nil && retErr == nil {
			retErr = err
		}
	}()
	for {
		req, err := restoreServer.Recv()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		op := req.Op
		switch {
		case op.Version != nil:
		case op.Object != nil:
		case op.Repo != nil:
			if _, err := pachClient.PfsAPIClient.CreateRepo(ctx, op.Repo); err != nil {
				return fmt.Errorf("error creating repo: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Commit != nil:
			if _, err := pachClient.PfsAPIClient.BuildCommit(ctx, op.Commit); err != nil {
				return fmt.Errorf("error creating commit: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Branch != nil:
			if _, err := pachClient.PfsAPIClient.SetBranch(ctx, op.Branch); err != nil {
				return fmt.Errorf("error creating branch: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Pipeline != nil:
			if _, err := pachClient.PpsAPIClient.CreatePipeline(ctx, op.Pipeline); err != nil {
				return fmt.Errorf("error creating pipeline: %v", grpcutil.ScrubGRPC(err))
			}
		}
	}
	return nil
}

func (a *apiServer) getPachClient() (*client.APIClient, error) {
	if a.pachClient == nil {
		var onceErr error
		a.pachClientOnce.Do(func() {
			a.pachClient, onceErr = client.NewFromAddress(a.address)
		})
		if onceErr != nil {
			return nil, onceErr
		}
	}
	return a.pachClient, nil
}
<|endoftext|>


<commit_before>package horizon

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"

	"github.com/PuerkitoBio/throttled"
	"github.com/stellar/horizon/render/problem"
	"github.com/stellar/horizon/test"
)

func NewTestApp() *App {
	app, err := NewApp(NewTestConfig())

	if err != nil {
		log.Panic(err)
	}

	return app
}

func NewTestConfig() Config {
	return Config{
		DatabaseUrl:            test.DatabaseUrl(),
		StellarCoreDatabaseUrl: test.StellarCoreDatabaseUrl(),
		RateLimit:              throttled.PerHour(1000),
	}
}

func NewRequestHelper(app *App) test.RequestHelper {
	return test.NewRequestHelper(app.web.router)
}

func ShouldBePageOf(actual interface{}, options ...interface{}) string {
	body := actual.(*bytes.Buffer)
	expected := options[0].(int)

	var result map[string]interface{}
	err := json.Unmarshal(body.Bytes(), &result)

	if err != nil {
		return fmt.Sprintf("Could not unmarshal json:\n%s\n", body.String())
	}

	embedded, ok := result["_embedded"]

	if !ok {
		return "No _embedded key in response"
	}

	records, ok := embedded.(map[string]interface{})["records"]

	if !ok {
		return "No records key in _embedded"
	}

	length := len(records.([]interface{}))

	if length != expected {
		return fmt.Sprintf("Expected %d records in page, got %d", expected, length)
	}

	return ""
}

func ShouldBeProblem(a interface{}, options ...interface{}) string {
	body := a.(*bytes.Buffer)
	expected := options[0].(problem.P)

	problem.Inflate(test.Context(), &expected)

	var actual problem.P
	err := json.Unmarshal(body.Bytes(), &actual)

	if err != nil {
		return fmt.Sprintf("Could not unmarshal json into problem struct:\n%s\n", body.String())
	}

	if expected.Type != "" && actual.Type != expected.Type {
		return fmt.Sprintf("Mismatched problem type: %s expected, got %s", expected.Type, actual.Type)
	}

	if expected.Status != 0 && actual.Status != expected.Status {
		return fmt.Sprintf("Mismatched problem status: %s expected, got %s", expected.Status, actual.Status)
	}

	return ""
}
<commit_msg>Set test loglevel<commit_after>package horizon

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"

	"github.com/PuerkitoBio/throttled"
	hlog "github.com/stellar/horizon/log"
	"github.com/stellar/horizon/render/problem"
	"github.com/stellar/horizon/test"
)

func NewTestApp() *App {
	app, err := NewApp(NewTestConfig())

	if err != nil {
		log.Panic(err)
	}

	return app
}

func NewTestConfig() Config {
	return Config{
		DatabaseUrl:            test.DatabaseUrl(),
		StellarCoreDatabaseUrl: test.StellarCoreDatabaseUrl(),
		RateLimit:              throttled.PerHour(1000),
		LogLevel:               hlog.InfoLevel,
	}
}

func NewRequestHelper(app *App) test.RequestHelper {
	return test.NewRequestHelper(app.web.router)
}

func ShouldBePageOf(actual interface{}, options ...interface{}) string {
	body := actual.(*bytes.Buffer)
	expected := options[0].(int)

	var result map[string]interface{}
	err := json.Unmarshal(body.Bytes(), &result)

	if err != nil {
		return fmt.Sprintf("Could not unmarshal json:\n%s\n", body.String())
	}

	embedded, ok := result["_embedded"]

	if !ok {
		return "No _embedded key in response"
	}

	records, ok := embedded.(map[string]interface{})["records"]

	if !ok {
		return "No records key in _embedded"
	}

	length := len(records.([]interface{}))

	if length != expected {
		return fmt.Sprintf("Expected %d records in page, got %d", expected, length)
	}

	return ""
}

func ShouldBeProblem(a interface{}, options ...interface{}) string {
	body := a.(*bytes.Buffer)
	expected := options[0].(problem.P)

	problem.Inflate(test.Context(), &expected)

	var actual problem.P
	err := json.Unmarshal(body.Bytes(), &actual)

	if err != nil {
		return fmt.Sprintf("Could not unmarshal json into problem struct:\n%s\n", body.String())
	}

	if expected.Type != "" && actual.Type != expected.Type {
		return fmt.Sprintf("Mismatched problem type: %s expected, got %s", expected.Type, actual.Type)
	}

	if expected.Status != 0 && actual.Status != expected.Status {
		return fmt.Sprintf("Mismatched problem status: %s expected, got %s", expected.Status, actual.Status)
	}

	return ""
}
<|endoftext|>


<commit_before>// Copyright 2015 tsuru-autoscale authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package action

import (
	"errors"
	"fmt"
	"net/http"
	"strings"

	"github.com/tsuru/tsuru-autoscale/db"
	"github.com/tsuru/tsuru-autoscale/log"
	"gopkg.in/mgo.v2/bson"
)

func logger() *log.Logger {
	return log.Log()
}

// Action represents an AutoScale action to increase or decrease the
// number of the units.
type Action struct {
	Name    string
	URL     string
	Method  string
	Body    string
	Headers map[string]string
}

// New creates a new action.
func New(a *Action) error {
	if a.URL == "" {
		return errors.New("action: url required")
	}
	if a.Method == "" {
		return errors.New("action: method required")
	}
	conn, err := db.Conn()
	if err != nil {
		logger().Error(err)
		return err
	}
	defer conn.Close()
	return conn.Actions().Insert(&a)
}

// FindByName finds action by name.
func FindByName(name string) (*Action, error) {
	conn, err := db.Conn()
	if err != nil {
		logger().Error(err)
		return nil, err
	}
	defer conn.Close()
	var action Action
	err = conn.Actions().Find(bson.M{"name": name}).One(&action)
	if err != nil {
		logger().Error(err)
		return nil, err
	}
	return &action, nil
}

// Remove removes an action.
func Remove(a *Action) error {
	conn, err := db.Conn()
	if err != nil {
		logger().Error(err)
		return err
	}
	defer conn.Close()
	return conn.Actions().Remove(bson.M{"name": a.Name})
}

func All() ([]Action, error) {
	conn, err := db.Conn()
	if err != nil {
		logger().Error(err)
		return nil, err
	}
	defer conn.Close()
	var actions []Action
	err = conn.Actions().Find(nil).All(&actions)
	if err != nil {
		logger().Error(err)
		return nil, err
	}
	return actions, nil
}

func (a *Action) Do(appName string, envs map[string]string) error {
	body := a.Body
	url := strings.Replace(a.URL, "{app}", appName, -1)
	for key, value := range envs {
		body = strings.Replace(body, fmt.Sprintf("{%s}", key), value, -1)
		url = strings.Replace(url, fmt.Sprintf("{%s}", key), value, -1)
	}
	logger().Printf("action %s - url: %s - body: %s - method: %s", a.Name, url, body, a.Method)
	req, err := http.NewRequest(a.Method, url, strings.NewReader(body))
	if err != nil {
		logger().Error(err)
		return err
	}
	for key, value := range a.Headers {
		req.Header.Add(key, value)
	}
	client := &http.Client{}
	_, err = client.Do(req)
	if err != nil {
		logger().Error(err)
		return err
	}
	return nil
}
<commit_msg>action: prevent connection leak on action.Do<commit_after>// Copyright 2015 tsuru-autoscale authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package action

import (
	"errors"
	"fmt"
	"net/http"
	"strings"

	"github.com/tsuru/tsuru-autoscale/db"
	"github.com/tsuru/tsuru-autoscale/log"
	"gopkg.in/mgo.v2/bson"
)

func logger() *log.Logger {
	return log.Log()
}

// Action represents an AutoScale action to increase or decrease the
// number of the units.
type Action struct {
	Name    string
	URL     string
	Method  string
	Body    string
	Headers map[string]string
}

// New creates a new action.
func New(a *Action) error {
	if a.URL == "" {
		return errors.New("action: url required")
	}
	if a.Method == "" {
		return errors.New("action: method required")
	}
	conn, err := db.Conn()
	if err != nil {
		logger().Error(err)
		return err
	}
	defer conn.Close()
	return conn.Actions().Insert(&a)
}

// FindByName finds action by name.
func FindByName(name string) (*Action, error) {
	conn, err := db.Conn()
	if err != nil {
		logger().Error(err)
		return nil, err
	}
	defer conn.Close()
	var action Action
	err = conn.Actions().Find(bson.M{"name": name}).One(&action)
	if err != nil {
		logger().Error(err)
		return nil, err
	}
	return &action, nil
}

// Remove removes an action.
func Remove(a *Action) error {
	conn, err := db.Conn()
	if err != nil {
		logger().Error(err)
		return err
	}
	defer conn.Close()
	return conn.Actions().Remove(bson.M{"name": a.Name})
}

func All() ([]Action, error) {
	conn, err := db.Conn()
	if err != nil {
		logger().Error(err)
		return nil, err
	}
	defer conn.Close()
	var actions []Action
	err = conn.Actions().Find(nil).All(&actions)
	if err != nil {
		logger().Error(err)
		return nil, err
	}
	return actions, nil
}

func (a *Action) Do(appName string, envs map[string]string) error {
	body := a.Body
	url := strings.Replace(a.URL, "{app}", appName, -1)
	for key, value := range envs {
		body = strings.Replace(body, fmt.Sprintf("{%s}", key), value, -1)
		url = strings.Replace(url, fmt.Sprintf("{%s}", key), value, -1)
	}
	logger().Printf("action %s - url: %s - body: %s - method: %s", a.Name, url, body, a.Method)
	req, err := http.NewRequest(a.Method, url, strings.NewReader(body))
	if err != nil {
		logger().Error(err)
		return err
	}
	for key, value := range a.Headers {
		req.Header.Add(key, value)
	}
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		logger().Error(err)
		return err
	}
	defer resp.Body.Close()
	return nil
}
<|endoftext|>


<commit_before>package actions

import (
	"bytes"
	"fmt"
	"gopkg.in/karlseguin/typed.v1"
	"log"
	"os/exec"
	"strings"
)

type Shell struct {
	command   string
	dir       string
	arguments []string
}

func (a *Shell) Run() error {
	var out bytes.Buffer
	cmd := exec.Command(a.command, a.arguments...)
	if len(a.dir) > 0 {
		cmd.Dir = a.dir
	}
	cmd.Stderr = &out
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("error running %s %s\n   %s", a.command, strings.Join(a.arguments, " "), out.String())
	}
	return nil
}

func NewShell(t typed.Typed) *Shell {
	command, ok := t.StringIf("command")
	if ok == false {
		log.Fatalf("shell action must have a command parameter: %s", t.MustBytes(""))
	}
	return &Shell{
		command:   command,
		dir:       t.String("dir"),
		arguments: t.Strings("arguments"),
	}
}
<commit_msg>grab action stdout<commit_after>package actions

import (
	"fmt"
	"gopkg.in/karlseguin/typed.v1"
	"log"
	"os/exec"
	"strings"
)

type Shell struct {
	command   string
	dir       string
	arguments []string
}

func (a *Shell) Run() error {
	cmd := exec.Command(a.command, a.arguments...)
	if len(a.dir) > 0 {
		cmd.Dir = a.dir
	}
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("error running %s %s\n   %s", a.command, strings.Join(a.arguments, " "), string(out))
	}
	return nil
}

func NewShell(t typed.Typed) *Shell {
	command, ok := t.StringIf("command")
	if ok == false {
		log.Fatalf("shell action must have a command parameter: %s", t.MustBytes(""))
	}
	return &Shell{
		command:   command,
		dir:       t.String("dir"),
		arguments: t.Strings("arguments"),
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build windows

package registry

import (
	"errors"
	"io"
	"syscall"
	"unicode/utf16"
	"unsafe"
)

const (
	// Registry value types.
	NONE                       = 0
	SZ                         = 1
	EXPAND_SZ                  = 2
	BINARY                     = 3
	DWORD                      = 4
	DWORD_BIG_ENDIAN           = 5
	LINK                       = 6
	MULTI_SZ                   = 7
	RESOURCE_LIST              = 8
	FULL_RESOURCE_DESCRIPTOR   = 9
	RESOURCE_REQUIREMENTS_LIST = 10
	QWORD                      = 11
)

var (
	// ErrShortBuffer is returned when the buffer was too short for the operation.
	ErrShortBuffer = syscall.ERROR_MORE_DATA

	// ErrNotExist is returned when a registry key or value does not exist.
	ErrNotExist = syscall.ERROR_FILE_NOT_FOUND

	// ErrUnexpectedType is returned by Get*Value when the value's type was unexpected.
	ErrUnexpectedType = errors.New("unexpected key value type")
)

// GetValue retrieves the type and data for the specified value associated
// with an open key k. It fills up buffer buf and returns the retrieved
// byte count n. If buf is too small to fit the stored value it returns
// ErrShortBuffer error along with the required buffer size n.
// If no buffer is provided, it returns true and actual buffer size n.
// If no buffer is provided, GetValue returns the value's type only.
// If the value does not exist, the error returned is ErrNotExist.
//
// GetValue is a low level function. If value's type is known, use the appropriate
// Get*Value function instead.
func (k Key) GetValue(name string, buf []byte) (n int, valtype uint32, err error) {
	pname, err := syscall.UTF16PtrFromString(name)
	if err != nil {
		return 0, 0, err
	}
	var pbuf *byte
	if len(buf) > 0 {
		pbuf = (*byte)(unsafe.Pointer(&buf[0]))
	}
	l := uint32(len(buf))
	err = syscall.RegQueryValueEx(syscall.Handle(k), pname, nil, &valtype, pbuf, &l)
	if err != nil {
		return int(l), valtype, err
	}
	return int(l), valtype, nil
}

func (k Key) getValue(name string, buf []byte) (date []byte, valtype uint32, err error) {
	p, err := syscall.UTF16PtrFromString(name)
	if err != nil {
		return nil, 0, err
	}
	var t uint32
	n := uint32(len(buf))
	for {
		err = syscall.RegQueryValueEx(syscall.Handle(k), p, nil, &t, (*byte)(unsafe.Pointer(&buf[0])), &n)
		if err == nil {
			return buf[:n], t, nil
		}
		if err != syscall.ERROR_MORE_DATA {
			return nil, 0, err
		}
		if n <= uint32(len(buf)) {
			return nil, 0, err
		}
		buf = make([]byte, n)
	}
}

// GetStringValue retrieves the string value for the specified
// value name associated with an open key k. It also returns the value's type.
// If value does not exist, GetStringValue returns ErrNotExist.
// If value is not SZ or EXPAND_SZ, it will return the correct value
// type and ErrUnexpectedType.
func (k Key) GetStringValue(name string) (val string, valtype uint32, err error) {
	data, typ, err2 := k.getValue(name, make([]byte, 64))
	if err2 != nil {
		return "", typ, err2
	}
	switch typ {
	case SZ, EXPAND_SZ:
	default:
		return "", typ, ErrUnexpectedType
	}
	if len(data) == 0 {
		return "", typ, nil
	}
	u := (*[1 << 10]uint16)(unsafe.Pointer(&data[0]))[:]
	return syscall.UTF16ToString(u), typ, nil
}

// ExpandString expands environment-variable strings and replaces
// them with the values defined for the current user.
// Use ExpandString to expand EXPAND_SZ strings.
func ExpandString(value string) (string, error) {
	if value == "" {
		return "", nil
	}
	p, err := syscall.UTF16PtrFromString(value)
	if err != nil {
		return "", err
	}
	r := make([]uint16, 100)
	for {
		n, err := expandEnvironmentStrings(p, &r[0], uint32(len(r)))
		if err != nil {
			return "", err
		}
		if n <= uint32(len(r)) {
			u := (*[1 << 10]uint16)(unsafe.Pointer(&r[0]))[:]
			return syscall.UTF16ToString(u), nil
		}
		r = make([]uint16, n)
	}
}

// GetStringsValue retrieves the []string value for the specified
// value name associated with an open key k. It also returns the value's type.
// If value does not exist, GetStringsValue returns ErrNotExist.
// If value is not MULTI_SZ, it will return the correct value
// type and ErrUnexpectedType.
func (k Key) GetStringsValue(name string) (val []string, valtype uint32, err error) {
	data, typ, err2 := k.getValue(name, make([]byte, 64))
	if err2 != nil {
		return nil, typ, err2
	}
	if typ != MULTI_SZ {
		return nil, typ, ErrUnexpectedType
	}
	val = make([]string, 0, 5)
	p := (*[1 << 24]uint16)(unsafe.Pointer(&data[0]))[:len(data)/2]
	p = p[:len(p)-1] // remove terminating nil
	from := 0
	for i, c := range p {
		if c == 0 {
			val = append(val, string(utf16.Decode(p[from:i])))
			from = i + 1
		}
	}
	return val, typ, nil
}

// GetIntegerValue retrieves the integer value for the specified
// value name associated with an open key k. It also returns the value's type.
// If value does not exist, GetIntegerValue returns ErrNotExist.
// If value is not DWORD or QWORD, it will return the correct value
// type and ErrUnexpectedType.
func (k Key) GetIntegerValue(name string) (val uint64, valtype uint32, err error) {
	data, typ, err2 := k.getValue(name, make([]byte, 8))
	if err2 != nil {
		return 0, typ, err2
	}
	switch typ {
	case DWORD:
		return uint64(*(*uint32)(unsafe.Pointer(&data[0]))), DWORD, nil
	case QWORD:
		return uint64(*(*uint64)(unsafe.Pointer(&data[0]))), QWORD, nil
	default:
		return 0, typ, ErrUnexpectedType
	}
}

// GetBinaryValue retrieves the binary value for the specified
// value name associated with an open key k. It also returns the value's type.
// If value does not exist, GetBinaryValue returns ErrNotExist.
// If value is not BINARY, it will return the correct value
// type and ErrUnexpectedType.
func (k Key) GetBinaryValue(name string) (val []byte, valtype uint32, err error) {
	data, typ, err2 := k.getValue(name, make([]byte, 64))
	if err2 != nil {
		return nil, typ, err2
	}
	if typ != BINARY {
		return nil, typ, ErrUnexpectedType
	}
	return data, typ, nil
}

func (k Key) setValue(name string, valtype uint32, data []byte) error {
	p, err := syscall.UTF16PtrFromString(name)
	if err != nil {
		return err
	}
	if len(data) == 0 {
		return regSetValueEx(syscall.Handle(k), p, 0, valtype, nil, 0)
	}
	return regSetValueEx(syscall.Handle(k), p, 0, valtype, &data[0], uint32(len(data)))
}

// SetDWordValue sets the data and type of a name value
// under key k to value and DWORD.
func (k Key) SetDWordValue(name string, value uint32) error {
	return k.setValue(name, DWORD, (*[4]byte)(unsafe.Pointer(&value))[:])
}

// SetQWordValue sets the data and type of a name value
// under key k to value and QWORD.
func (k Key) SetQWordValue(name string, value uint64) error {
	return k.setValue(name, QWORD, (*[8]byte)(unsafe.Pointer(&value))[:])
}

func (k Key) setStringValue(name string, valtype uint32, value string) error {
	v, err := syscall.UTF16FromString(value)
	if err != nil {
		return err
	}
	buf := (*[1 << 10]byte)(unsafe.Pointer(&v[0]))[:len(v)*2]
	return k.setValue(name, valtype, buf)
}

// SetStringValue sets the data and type of a name value
// under key k to value and SZ. The value must not contain a zero byte.
func (k Key) SetStringValue(name, value string) error {
	return k.setStringValue(name, SZ, value)
}

// SetExpandStringValue sets the data and type of a name value
// under key k to value and EXPAND_SZ. The value must not contain a zero byte.
func (k Key) SetExpandStringValue(name, value string) error {
	return k.setStringValue(name, EXPAND_SZ, value)
}

// SetStringsValue sets the data and type of a name value
// under key k to value and MULTI_SZ. The value strings
// must not contain a zero byte.
func (k Key) SetStringsValue(name string, value []string) error {
	ss := ""
	for _, s := range value {
		for i := 0; i < len(s); i++ {
			if s[i] == 0 {
				return errors.New("string cannot have 0 inside")
			}
		}
		ss += s + "\x00"
	}
	v := utf16.Encode([]rune(ss + "\x00"))
	buf := (*[1 << 10]byte)(unsafe.Pointer(&v[0]))[:len(v)*2]
	return k.setValue(name, MULTI_SZ, buf)
}

// SetBinaryValue sets the data and type of a name value
// under key k to value and BINARY.
func (k Key) SetBinaryValue(name string, value []byte) error {
	return k.setValue(name, BINARY, value)
}

// DeleteValue removes a named value from the key k.
func (k Key) DeleteValue(name string) error {
	return regDeleteValue(syscall.Handle(k), syscall.StringToUTF16Ptr(name))
}

// ReadValueNames returns the value names of key k.
// The parameter n controls the number of returned names,
// analogous to the way os.File.Readdirnames works.
func (k Key) ReadValueNames(n int) ([]string, error) {
	ki, err := k.Stat()
	if err != nil {
		return nil, err
	}
	names := make([]string, 0, ki.ValueCount)
	buf := make([]uint16, ki.MaxValueNameLen+1) // extra room for terminating null character
loopItems:
	for i := uint32(0); ; i++ {
		if n > 0 {
			if len(names) == n {
				return names, nil
			}
		}
		l := uint32(len(buf))
		for {
			err := regEnumValue(syscall.Handle(k), i, &buf[0], &l, nil, nil, nil, nil)
			if err == nil {
				break
			}
			if err == syscall.ERROR_MORE_DATA {
				println(len(buf), l)
				// Double buffer size and try again.
				l = uint32(2 * len(buf))
				buf = make([]uint16, l)
				continue
			}
			if err == _ERROR_NO_MORE_ITEMS {
				break loopItems
			}
			return names, err
		}
		names = append(names, syscall.UTF16ToString(buf[:l]))
	}
	if n > len(names) {
		return names, io.EOF
	}
	return names, nil
}
<commit_msg>internal/syscall/windows: increase registry.ExpandString buffer<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build windows

package registry

import (
	"errors"
	"io"
	"syscall"
	"unicode/utf16"
	"unsafe"
)

const (
	// Registry value types.
	NONE                       = 0
	SZ                         = 1
	EXPAND_SZ                  = 2
	BINARY                     = 3
	DWORD                      = 4
	DWORD_BIG_ENDIAN           = 5
	LINK                       = 6
	MULTI_SZ                   = 7
	RESOURCE_LIST              = 8
	FULL_RESOURCE_DESCRIPTOR   = 9
	RESOURCE_REQUIREMENTS_LIST = 10
	QWORD                      = 11
)

var (
	// ErrShortBuffer is returned when the buffer was too short for the operation.
	ErrShortBuffer = syscall.ERROR_MORE_DATA

	// ErrNotExist is returned when a registry key or value does not exist.
	ErrNotExist = syscall.ERROR_FILE_NOT_FOUND

	// ErrUnexpectedType is returned by Get*Value when the value's type was unexpected.
	ErrUnexpectedType = errors.New("unexpected key value type")
)

// GetValue retrieves the type and data for the specified value associated
// with an open key k. It fills up buffer buf and returns the retrieved
// byte count n. If buf is too small to fit the stored value it returns
// ErrShortBuffer error along with the required buffer size n.
// If no buffer is provided, it returns true and actual buffer size n.
// If no buffer is provided, GetValue returns the value's type only.
// If the value does not exist, the error returned is ErrNotExist.
//
// GetValue is a low level function. If value's type is known, use the appropriate
// Get*Value function instead.
func (k Key) GetValue(name string, buf []byte) (n int, valtype uint32, err error) {
	pname, err := syscall.UTF16PtrFromString(name)
	if err != nil {
		return 0, 0, err
	}
	var pbuf *byte
	if len(buf) > 0 {
		pbuf = (*byte)(unsafe.Pointer(&buf[0]))
	}
	l := uint32(len(buf))
	err = syscall.RegQueryValueEx(syscall.Handle(k), pname, nil, &valtype, pbuf, &l)
	if err != nil {
		return int(l), valtype, err
	}
	return int(l), valtype, nil
}

func (k Key) getValue(name string, buf []byte) (date []byte, valtype uint32, err error) {
	p, err := syscall.UTF16PtrFromString(name)
	if err != nil {
		return nil, 0, err
	}
	var t uint32
	n := uint32(len(buf))
	for {
		err = syscall.RegQueryValueEx(syscall.Handle(k), p, nil, &t, (*byte)(unsafe.Pointer(&buf[0])), &n)
		if err == nil {
			return buf[:n], t, nil
		}
		if err != syscall.ERROR_MORE_DATA {
			return nil, 0, err
		}
		if n <= uint32(len(buf)) {
			return nil, 0, err
		}
		buf = make([]byte, n)
	}
}

// GetStringValue retrieves the string value for the specified
// value name associated with an open key k. It also returns the value's type.
// If value does not exist, GetStringValue returns ErrNotExist.
// If value is not SZ or EXPAND_SZ, it will return the correct value
// type and ErrUnexpectedType.
func (k Key) GetStringValue(name string) (val string, valtype uint32, err error) {
	data, typ, err2 := k.getValue(name, make([]byte, 64))
	if err2 != nil {
		return "", typ, err2
	}
	switch typ {
	case SZ, EXPAND_SZ:
	default:
		return "", typ, ErrUnexpectedType
	}
	if len(data) == 0 {
		return "", typ, nil
	}
	u := (*[1 << 10]uint16)(unsafe.Pointer(&data[0]))[:]
	return syscall.UTF16ToString(u), typ, nil
}

// ExpandString expands environment-variable strings and replaces
// them with the values defined for the current user.
// Use ExpandString to expand EXPAND_SZ strings.
func ExpandString(value string) (string, error) {
	if value == "" {
		return "", nil
	}
	p, err := syscall.UTF16PtrFromString(value)
	if err != nil {
		return "", err
	}
	r := make([]uint16, 100)
	for {
		n, err := expandEnvironmentStrings(p, &r[0], uint32(len(r)))
		if err != nil {
			return "", err
		}
		if n <= uint32(len(r)) {
			u := (*[1 << 15]uint16)(unsafe.Pointer(&r[0]))[:]
			return syscall.UTF16ToString(u), nil
		}
		r = make([]uint16, n)
	}
}

// GetStringsValue retrieves the []string value for the specified
// value name associated with an open key k. It also returns the value's type.
// If value does not exist, GetStringsValue returns ErrNotExist.
// If value is not MULTI_SZ, it will return the correct value
// type and ErrUnexpectedType.
func (k Key) GetStringsValue(name string) (val []string, valtype uint32, err error) {
	data, typ, err2 := k.getValue(name, make([]byte, 64))
	if err2 != nil {
		return nil, typ, err2
	}
	if typ != MULTI_SZ {
		return nil, typ, ErrUnexpectedType
	}
	val = make([]string, 0, 5)
	p := (*[1 << 24]uint16)(unsafe.Pointer(&data[0]))[:len(data)/2]
	p = p[:len(p)-1] // remove terminating nil
	from := 0
	for i, c := range p {
		if c == 0 {
			val = append(val, string(utf16.Decode(p[from:i])))
			from = i + 1
		}
	}
	return val, typ, nil
}

// GetIntegerValue retrieves the integer value for the specified
// value name associated with an open key k. It also returns the value's type.
// If value does not exist, GetIntegerValue returns ErrNotExist.
// If value is not DWORD or QWORD, it will return the correct value
// type and ErrUnexpectedType.
func (k Key) GetIntegerValue(name string) (val uint64, valtype uint32, err error) {
	data, typ, err2 := k.getValue(name, make([]byte, 8))
	if err2 != nil {
		return 0, typ, err2
	}
	switch typ {
	case DWORD:
		return uint64(*(*uint32)(unsafe.Pointer(&data[0]))), DWORD, nil
	case QWORD:
		return uint64(*(*uint64)(unsafe.Pointer(&data[0]))), QWORD, nil
	default:
		return 0, typ, ErrUnexpectedType
	}
}

// GetBinaryValue retrieves the binary value for the specified
// value name associated with an open key k. It also returns the value's type.
// If value does not exist, GetBinaryValue returns ErrNotExist.
// If value is not BINARY, it will return the correct value
// type and ErrUnexpectedType.
func (k Key) GetBinaryValue(name string) (val []byte, valtype uint32, err error) {
	data, typ, err2 := k.getValue(name, make([]byte, 64))
	if err2 != nil {
		return nil, typ, err2
	}
	if typ != BINARY {
		return nil, typ, ErrUnexpectedType
	}
	return data, typ, nil
}

func (k Key) setValue(name string, valtype uint32, data []byte) error {
	p, err := syscall.UTF16PtrFromString(name)
	if err != nil {
		return err
	}
	if len(data) == 0 {
		return regSetValueEx(syscall.Handle(k), p, 0, valtype, nil, 0)
	}
	return regSetValueEx(syscall.Handle(k), p, 0, valtype, &data[0], uint32(len(data)))
}

// SetDWordValue sets the data and type of a name value
// under key k to value and DWORD.
func (k Key) SetDWordValue(name string, value uint32) error {
	return k.setValue(name, DWORD, (*[4]byte)(unsafe.Pointer(&value))[:])
}

// SetQWordValue sets the data and type of a name value
// under key k to value and QWORD.
func (k Key) SetQWordValue(name string, value uint64) error {
	return k.setValue(name, QWORD, (*[8]byte)(unsafe.Pointer(&value))[:])
}

func (k Key) setStringValue(name string, valtype uint32, value string) error {
	v, err := syscall.UTF16FromString(value)
	if err != nil {
		return err
	}
	buf := (*[1 << 10]byte)(unsafe.Pointer(&v[0]))[:len(v)*2]
	return k.setValue(name, valtype, buf)
}

// SetStringValue sets the data and type of a name value
// under key k to value and SZ. The value must not contain a zero byte.
func (k Key) SetStringValue(name, value string) error {
	return k.setStringValue(name, SZ, value)
}

// SetExpandStringValue sets the data and type of a name value
// under key k to value and EXPAND_SZ. The value must not contain a zero byte.
func (k Key) SetExpandStringValue(name, value string) error {
	return k.setStringValue(name, EXPAND_SZ, value)
}

// SetStringsValue sets the data and type of a name value
// under key k to value and MULTI_SZ. The value strings
// must not contain a zero byte.
func (k Key) SetStringsValue(name string, value []string) error {
	ss := ""
	for _, s := range value {
		for i := 0; i < len(s); i++ {
			if s[i] == 0 {
				return errors.New("string cannot have 0 inside")
			}
		}
		ss += s + "\x00"
	}
	v := utf16.Encode([]rune(ss + "\x00"))
	buf := (*[1 << 10]byte)(unsafe.Pointer(&v[0]))[:len(v)*2]
	return k.setValue(name, MULTI_SZ, buf)
}

// SetBinaryValue sets the data and type of a name value
// under key k to value and BINARY.
func (k Key) SetBinaryValue(name string, value []byte) error {
	return k.setValue(name, BINARY, value)
}

// DeleteValue removes a named value from the key k.
func (k Key) DeleteValue(name string) error {
	return regDeleteValue(syscall.Handle(k), syscall.StringToUTF16Ptr(name))
}

// ReadValueNames returns the value names of key k.
// The parameter n controls the number of returned names,
// analogous to the way os.File.Readdirnames works.
func (k Key) ReadValueNames(n int) ([]string, error) {
	ki, err := k.Stat()
	if err != nil {
		return nil, err
	}
	names := make([]string, 0, ki.ValueCount)
	buf := make([]uint16, ki.MaxValueNameLen+1) // extra room for terminating null character
loopItems:
	for i := uint32(0); ; i++ {
		if n > 0 {
			if len(names) == n {
				return names, nil
			}
		}
		l := uint32(len(buf))
		for {
			err := regEnumValue(syscall.Handle(k), i, &buf[0], &l, nil, nil, nil, nil)
			if err == nil {
				break
			}
			if err == syscall.ERROR_MORE_DATA {
				println(len(buf), l)
				// Double buffer size and try again.
				l = uint32(2 * len(buf))
				buf = make([]uint16, l)
				continue
			}
			if err == _ERROR_NO_MORE_ITEMS {
				break loopItems
			}
			return names, err
		}
		names = append(names, syscall.UTF16ToString(buf[:l]))
	}
	if n > len(names) {
		return names, io.EOF
	}
	return names, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"

	"github.com/bjwbell/renfish/auth"
	"github.com/bjwbell/renfish/conf"
	"github.com/bjwbell/renfish/db"
	"github.com/bjwbell/renfish/submit"
	"github.com/bjwbell/renroll/src/renroll"
)

func indexHandler(w http.ResponseWriter, r *http.Request) {
	log.Print("indexhandler - start")
	index := struct{ Conf conf.Configuration }{conf.Config()}
	t, e := template.ParseFiles("idx.html", "templates/header.html", "templates/topbar.html", "templates/bottombar.html")
	if e != nil {
		panic(e)
	}
	log.Print("indexhandler - execute")
	if e = t.Execute(w, index); e != nil {
		panic(e)
	}
}

func robotsHandler(w http.ResponseWriter, r *http.Request) {
	log.Print("robothandler - start")
	index := struct{ Conf conf.Configuration }{conf.Config()}
	t, e := template.ParseFiles("robots.txt")
	if e != nil {
		panic(e)
	}
	log.Print("robothandler - execute")
	if e = t.Execute(w, index); e != nil {
		panic(e)
	}
}

func googleAdwordsVerifyHandler(w http.ResponseWriter, r *http.Request) {
	log.Print("adwordsVerifyHandler - start")
	index := struct{ Conf conf.Configuration }{conf.Config()}
	t, e := template.ParseFiles("google41fd03a6c9348593.html")
	if e != nil {
		panic(e)
	}
	log.Print("adwordsVerifyHandler - execute")
	if e = t.Execute(w, index); e != nil {
		panic(e)
	}
}

func aboutHandler(w http.ResponseWriter, r *http.Request) {
	about := struct{ Conf conf.Configuration }{conf.Config()}
	t, _ := template.ParseFiles(
		"about.html",
		"templates/header.html",
		"templates/topbar.html",
		"templates/bottombar.html")
	t.Execute(w, about)
}

func unreleasedHandler(w http.ResponseWriter, r *http.Request) {
	conf := struct{ Conf renroll.Configuration }{renroll.Config()}
	conf.Conf.GPlusSigninCallback = "gSettings"
	conf.Conf.FacebookSigninCallback = "fbSettings"
	t, _ := template.ParseFiles(
		"unreleased.html",
		"templates/header.html",
		"templates/topbar.html",
		"templates/bottombar.html")
	t.Execute(w, conf)
}

func getNextIP() string {
	ips := db.DbGetIPs(db.DbName)
	return db.DbGetNextAvailableIP(ips)
}

func createSite(emailAddress, siteName string) {
	domain := siteName + "." + "renfish.com"
	// Add nginx conf file
	nginxConf := `server {
     listen 443 ssl;
    listen [::]:443 ssl;
    server_name  <site-name>;
    ssl_certificate     /etc/letsencrypt/live/<site-name>/cert.pem;
    ssl_certificate_key /etc/letsencrypt/live/<site-name>/privkey.pem;
    ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
    ssl_ciphers         HIGH:!aNULL:!MD5;
    location / {
             proxy_pass http://<ip-address>:8080;
            proxy_set_header Host $host;
    }
}
server {
    listen              80;
    server_name  <site-name>;
    location / {
            proxy_pass http://<ip-address>;
             proxy_set_header Host $host;
    }
}
`
	ipAddr := getNextIP()
	nginxConf = strings.Replace(nginxConf, "<site-name>", domain, -1)
	nginxConf = strings.Replace(nginxConf, "<ip-address>", ipAddr, -1)
	fileName := "/etc/nginx/sites-available/" + siteName + "." + "renfish.com"
	if err := ioutil.WriteFile(fileName, []byte(nginxConf), 0644); err != nil {
		auth.LogError(fmt.Sprintf("ERROR WRITING NGINX CONF FILE, sitename: %v, filename: %v, err: %v", siteName, fileName, err))
		return
	}

	// create certificate
	out, err := exec.Command("certbot", "certonly", "-n", "-q", "--standalone", "--pre-hook", "service nginx stop", "--post-hook", "service nginx start", "-d", domain).Output()
	if err != nil {
		auth.LogError(fmt.Sprintf("CERTBOT ERROR, err: %v, stdout: %v", err, string(out)))
		log.Fatal(err)
	}

	// Link nginx conf file to sites-enabled/
	symlink := "/etc/nginx/sites-enabled/" + siteName + "." + "renfish.com"
	if err := os.Symlink(fileName, symlink); err != nil {
		auth.LogError(fmt.Sprintf("ERROR CREATING NGINX CONF FILE SYMLINK, sitename: %v, filename: %v, symlink: %v, err: %v", siteName, fileName, symlink, err))
		return
	}

	// Reload nginx conf
	out, err = exec.Command("nginx", "-s", "reload").Output()
	if err != nil {
		auth.LogError(fmt.Sprintf("ERROR RELOADING NGINX CONF, err: %v, stdout: %v", err, string(out)))
		log.Fatal(err)
	}

	// start Gophish container
	out, err = exec.Command("docker", "run", "--net", "gophish", "--ip", ipAddr, "bjwbell/gophish-container", "/gophish/gophish").Output()
	if err != nil {
		auth.LogError(fmt.Sprintf("ERROR STARTING GOPHISH CONTAINER, err: %v, stdout: %v", err, string(out)))
		log.Fatal(err)
	}

	// Save details to database
	if _, success := db.SaveSite(emailAddress, siteName, ipAddr); !success {
		auth.LogError(fmt.Sprintf("ERROR SAVING SITE TO DB email (%s), sitename (%s), ip (%s)",
			emailAddress, siteName, ipAddr))
		log.Fatal(nil)
	}
	return
}

func createsiteHandler(w http.ResponseWriter, r *http.Request) {
	conf := struct {
		Conf     renroll.Configuration
		Email    string
		SiteName string
	}{renroll.Config(), "", ""}
	conf.Conf.GPlusSigninCallback = "gSettings"
	conf.Conf.FacebookSigninCallback = "fbSettings"
	if err := r.ParseForm(); err != nil {
		auth.LogError(fmt.Sprintf("ERROR PARSEFORM, ERR: %v", err))
		t, _ := template.ParseFiles(
			"setuperror.html",
			"templates/header.html",
			"templates/topbar.html",
			"templates/bottombar.html")
		if err := t.Execute(w, conf); err != nil {
			auth.LogError(fmt.Sprintf("ERROR t.EXECUTE, ERR: %v", err))
		}
	}
	email := r.Form.Get("email")
	siteName := r.Form.Get("sitename")
	conf.Email = email
	conf.SiteName = "https://" + siteName + "." + r.Host
	if email == "" || siteName == "" {
		auth.LogError(fmt.Sprintf("MiSSING EMAIL or SITENAME, email: %v, sitename: %v", email, siteName))
		t, _ := template.ParseFiles(
			"setuperror.html",
			"templates/header.html",
			"templates/topbar.html",
			"templates/bottombar.html")
		if err := t.Execute(w, conf); err != nil {
			auth.LogError(fmt.Sprintf("ERROR t.EXECUTE, ERR: %v", err))
		}
	} else {
		createSite(email, siteName)
		t, _ := template.ParseFiles(
			"setup.html",
			"templates/header.html",
			"templates/topbar.html",
			"templates/bottombar.html")
		if err := t.Execute(w, conf); err != nil {
			auth.LogError(fmt.Sprintf("ERROR t.EXECUTE, ERR: %v", err))
		}
	}
}

func settingsHandler(w http.ResponseWriter, r *http.Request) {
	conf := struct{ Conf conf.Configuration }{conf.Config()}
	conf.Conf.GPlusSigninCallback = "gSettings"
	conf.Conf.FacebookSigninCallback = "fbSettings"
	t, _ := template.ParseFiles(
		"settings.html",
		"templates/header.html",
		"templates/topbar.html",
		"templates/bottombar.html")
	if err := t.Execute(w, conf); err != nil {
		auth.LogError(fmt.Sprintf("ERROR t.EXECUTE, ERR: %v", err))
	}
}

func redir(w http.ResponseWriter, req *http.Request) {
	host := req.Host
	httpsPort := "443"
	if strings.Index(host, ":8080") != -1 {
		httpsPort = "8443"
	}
	host = strings.TrimSuffix(host, ":8080")
	host = strings.TrimSuffix(host, ":80")
	if httpsPort == "443" {
		http.Redirect(w, req, "https://"+host+req.RequestURI, http.StatusMovedPermanently)
	} else {
		http.Redirect(w, req, "https://"+host+":"+httpsPort+req.RequestURI, http.StatusMovedPermanently)
	}
}

func main() {
	http.HandleFunc("/about", aboutHandler)
	http.HandleFunc("/auth/getemail", auth.GetGPlusEmailHandler)
	http.HandleFunc("/createaccount", auth.CreateAccountHandler)
	http.HandleFunc("/index", indexHandler)
	http.HandleFunc("/logerror", auth.LogErrorHandler)
	http.HandleFunc("/oauth2callback", auth.Oauth2callback)
	http.HandleFunc("/settings", settingsHandler)
	http.HandleFunc("/signinform", auth.SigninFormHandler)
	http.HandleFunc("/submit", submit.SubmitHandler)
	http.HandleFunc("/unreleased", unreleasedHandler)
	http.HandleFunc("/createsite", createsiteHandler)

	http.HandleFunc("/index.html", indexHandler)
	http.HandleFunc("/robots.txt", robotsHandler)
	http.HandleFunc("/google41fd03a6c9348593.html", googleAdwordsVerifyHandler)

	http.Handle("/css/", http.StripPrefix("/css/", http.FileServer(http.Dir("./css"))))
	http.Handle("/font-awesome-4.7.0/", http.StripPrefix("/font-awesome-4.7.0/", http.FileServer(http.Dir("./font-awesome-4.7.0"))))
	http.Handle("/fonts/", http.StripPrefix("/fonts/", http.FileServer(http.Dir("./fonts"))))
	http.Handle("/images/", http.StripPrefix("/images/", http.FileServer(http.Dir("./images"))))
	http.Handle("/js/", http.StripPrefix("/js/", http.FileServer(http.Dir("./js"))))
	http.Handle("/screenshots/", http.StripPrefix("/screenshots/", http.FileServer(http.Dir("./screenshots"))))
	http.HandleFunc("/", indexHandler)

	// HTTP to HTTPS redirection
	// go func() {
	// 	err := http.ListenAndServe(":80", http.HandlerFunc(redir))
	// 	if err != nil {
	// 		log.Print("HTTP ListenAndServe :8080", err)
	// 		log.Print("Trying HTTP ListenAndServe :8080.")
	// 		panic(http.ListenAndServe(":8080", http.HandlerFunc(redir)))

	// 	}
	// }()

	if !db.Exists(db.DbName) {
		db.Create(db.DbName)
	}

	go func() {
		err := http.ListenAndServe(":80", nil)
		if err != nil {
			log.Print("HTTP ListenAndServe :80, ", err)
			log.Print("Trying HTTP ListenAndServe :8080.")
			if err != nil {
				panic(http.ListenAndServe(":8080", nil))
			}

		}
	}()

	cert := "/etc/letsencrypt/live/renfish.com/cert.pem"
	privkey := "/etc/letsencrypt/live/renfish.com/privkey.pem"
	if _, err := os.Stat(cert); os.IsNotExist(err) {
		log.Print("cert: ", err)
		cert = "./generate_cert/cert.pem"
	}
	if _, err := os.Stat(privkey); os.IsNotExist(err) {
		log.Print("cert: ", err)
		privkey = "./generate_cert/key.pem"
	}
	err := http.ListenAndServeTLS(":443", cert, privkey, nil)
	if err != nil {
		log.Print("HTTPS ListenAndServe :8443")
		err = http.ListenAndServeTLS(":8443", cert, privkey, nil)
		if err != nil {
			log.Print("HTTPS ListenAndServe: ", err)
		}
	}
}
<commit_msg>Add logging output<commit_after>package main

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"

	"github.com/bjwbell/renfish/auth"
	"github.com/bjwbell/renfish/conf"
	"github.com/bjwbell/renfish/db"
	"github.com/bjwbell/renfish/submit"
	"github.com/bjwbell/renroll/src/renroll"
)

func indexHandler(w http.ResponseWriter, r *http.Request) {
	log.Print("indexhandler - start")
	index := struct{ Conf conf.Configuration }{conf.Config()}
	t, e := template.ParseFiles("idx.html", "templates/header.html", "templates/topbar.html", "templates/bottombar.html")
	if e != nil {
		panic(e)
	}
	log.Print("indexhandler - execute")
	if e = t.Execute(w, index); e != nil {
		panic(e)
	}
}

func robotsHandler(w http.ResponseWriter, r *http.Request) {
	log.Print("robothandler - start")
	index := struct{ Conf conf.Configuration }{conf.Config()}
	t, e := template.ParseFiles("robots.txt")
	if e != nil {
		panic(e)
	}
	log.Print("robothandler - execute")
	if e = t.Execute(w, index); e != nil {
		panic(e)
	}
}

func googleAdwordsVerifyHandler(w http.ResponseWriter, r *http.Request) {
	log.Print("adwordsVerifyHandler - start")
	index := struct{ Conf conf.Configuration }{conf.Config()}
	t, e := template.ParseFiles("google41fd03a6c9348593.html")
	if e != nil {
		panic(e)
	}
	log.Print("adwordsVerifyHandler - execute")
	if e = t.Execute(w, index); e != nil {
		panic(e)
	}
}

func aboutHandler(w http.ResponseWriter, r *http.Request) {
	about := struct{ Conf conf.Configuration }{conf.Config()}
	t, _ := template.ParseFiles(
		"about.html",
		"templates/header.html",
		"templates/topbar.html",
		"templates/bottombar.html")
	t.Execute(w, about)
}

func unreleasedHandler(w http.ResponseWriter, r *http.Request) {
	conf := struct{ Conf renroll.Configuration }{renroll.Config()}
	conf.Conf.GPlusSigninCallback = "gSettings"
	conf.Conf.FacebookSigninCallback = "fbSettings"
	t, _ := template.ParseFiles(
		"unreleased.html",
		"templates/header.html",
		"templates/topbar.html",
		"templates/bottombar.html")
	t.Execute(w, conf)
}

func getNextIP() string {
	ips := db.DbGetIPs(db.DbName)
	return db.DbGetNextAvailableIP(ips)
}

func createSite(emailAddress, siteName string) {
	domain := siteName + "." + "renfish.com"
	// Add nginx conf file
	nginxConf := `server {
     listen 443 ssl;
    listen [::]:443 ssl;
    server_name  <site-name>;
    ssl_certificate     /etc/letsencrypt/live/<site-name>/cert.pem;
    ssl_certificate_key /etc/letsencrypt/live/<site-name>/privkey.pem;
    ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
    ssl_ciphers         HIGH:!aNULL:!MD5;
    location / {
             proxy_pass http://<ip-address>:8080;
            proxy_set_header Host $host;
    }
}
server {
    listen              80;
    server_name  <site-name>;
    location / {
            proxy_pass http://<ip-address>;
             proxy_set_header Host $host;
    }
}
`
	ipAddr := getNextIP()
	nginxConf = strings.Replace(nginxConf, "<site-name>", domain, -1)
	nginxConf = strings.Replace(nginxConf, "<ip-address>", ipAddr, -1)
	fileName := "/etc/nginx/sites-available/" + siteName + "." + "renfish.com"
	if err := ioutil.WriteFile(fileName, []byte(nginxConf), 0644); err != nil {
		auth.LogError(fmt.Sprintf("ERROR WRITING NGINX CONF FILE, sitename: %v, filename: %v, err: %v", siteName, fileName, err))
		return
	}

	// create certificate
	out, err := exec.Command("certbot", "certonly", "-n", "-q", "--standalone", "--pre-hook", "service nginx stop", "--post-hook", "service nginx start", "-d", domain).Output()
	if err != nil {
		auth.LogError(fmt.Sprintf("CERTBOT ERROR, err: %v, stdout: %v", err, string(out)))
		log.Fatal(err)
	} else {
		fmt.Println("CREATED CERTBOT CERTIFICATE")
	}

	// Link nginx conf file to sites-enabled/
	symlink := "/etc/nginx/sites-enabled/" + siteName + "." + "renfish.com"
	if err := os.Symlink(fileName, symlink); err != nil {
		auth.LogError(fmt.Sprintf("ERROR CREATING NGINX CONF FILE SYMLINK, sitename: %v, filename: %v, symlink: %v, err: %v", siteName, fileName, symlink, err))
		return
	} else {
		fmt.Println("CREATED NGINX CONF FILE")
	}

	// Reload nginx conf
	out, err = exec.Command("nginx", "-s", "reload").Output()
	if err != nil {
		auth.LogError(fmt.Sprintf("ERROR RELOADING NGINX CONF, err: %v, stdout: %v", err, string(out)))
		log.Fatal(err)
	} else {
		fmt.Println("RELOADED NGINX CONF")
	}

	// start Gophish container
	out, err = exec.Command("docker", "run", "--net", "gophish", "--ip", ipAddr, "bjwbell/gophish-container", "/gophish/gophish").Output()
	if err != nil {
		auth.LogError(fmt.Sprintf("ERROR STARTING GOPHISH CONTAINER, err: %v, stdout: %v", err, string(out)))
		log.Fatal(err)
	} else {
		fmt.Println("STARTED GOPHISH CONTAINER")
	}

	// Save details to database
	if _, success := db.SaveSite(emailAddress, siteName, ipAddr); !success {
		auth.LogError(fmt.Sprintf("ERROR SAVING SITE TO DB email (%s), sitename (%s), ip (%s)",
			emailAddress, siteName, ipAddr))
		log.Fatal(nil)
	} else {
		fmt.Println(fmt.Sprintf("SAVED SITE TO DB email (%s), sitename (%s), ip (%s)", emailAddress, siteName, ipAddr))
	}
	return
}

func createsiteHandler(w http.ResponseWriter, r *http.Request) {
	conf := struct {
		Conf     renroll.Configuration
		Email    string
		SiteName string
	}{renroll.Config(), "", ""}
	conf.Conf.GPlusSigninCallback = "gSettings"
	conf.Conf.FacebookSigninCallback = "fbSettings"
	if err := r.ParseForm(); err != nil {
		auth.LogError(fmt.Sprintf("ERROR PARSEFORM, ERR: %v", err))
		t, _ := template.ParseFiles(
			"setuperror.html",
			"templates/header.html",
			"templates/topbar.html",
			"templates/bottombar.html")
		if err := t.Execute(w, conf); err != nil {
			auth.LogError(fmt.Sprintf("ERROR t.EXECUTE, ERR: %v", err))
		}
	}
	email := r.Form.Get("email")
	siteName := r.Form.Get("sitename")
	conf.Email = email
	conf.SiteName = "https://" + siteName + "." + r.Host
	if email == "" || siteName == "" {
		auth.LogError(fmt.Sprintf("MiSSING EMAIL or SITENAME, email: %v, sitename: %v", email, siteName))
		t, _ := template.ParseFiles(
			"setuperror.html",
			"templates/header.html",
			"templates/topbar.html",
			"templates/bottombar.html")
		if err := t.Execute(w, conf); err != nil {
			auth.LogError(fmt.Sprintf("ERROR t.EXECUTE, ERR: %v", err))
		}
	} else {
		createSite(email, siteName)
		t, _ := template.ParseFiles(
			"setup.html",
			"templates/header.html",
			"templates/topbar.html",
			"templates/bottombar.html")
		if err := t.Execute(w, conf); err != nil {
			auth.LogError(fmt.Sprintf("ERROR t.EXECUTE, ERR: %v", err))
		}
	}
}

func settingsHandler(w http.ResponseWriter, r *http.Request) {
	conf := struct{ Conf conf.Configuration }{conf.Config()}
	conf.Conf.GPlusSigninCallback = "gSettings"
	conf.Conf.FacebookSigninCallback = "fbSettings"
	t, _ := template.ParseFiles(
		"settings.html",
		"templates/header.html",
		"templates/topbar.html",
		"templates/bottombar.html")
	if err := t.Execute(w, conf); err != nil {
		auth.LogError(fmt.Sprintf("ERROR t.EXECUTE, ERR: %v", err))
	}
}

func redir(w http.ResponseWriter, req *http.Request) {
	host := req.Host
	httpsPort := "443"
	if strings.Index(host, ":8080") != -1 {
		httpsPort = "8443"
	}
	host = strings.TrimSuffix(host, ":8080")
	host = strings.TrimSuffix(host, ":80")
	if httpsPort == "443" {
		http.Redirect(w, req, "https://"+host+req.RequestURI, http.StatusMovedPermanently)
	} else {
		http.Redirect(w, req, "https://"+host+":"+httpsPort+req.RequestURI, http.StatusMovedPermanently)
	}
}

func main() {
	http.HandleFunc("/about", aboutHandler)
	http.HandleFunc("/auth/getemail", auth.GetGPlusEmailHandler)
	http.HandleFunc("/createaccount", auth.CreateAccountHandler)
	http.HandleFunc("/index", indexHandler)
	http.HandleFunc("/logerror", auth.LogErrorHandler)
	http.HandleFunc("/oauth2callback", auth.Oauth2callback)
	http.HandleFunc("/settings", settingsHandler)
	http.HandleFunc("/signinform", auth.SigninFormHandler)
	http.HandleFunc("/submit", submit.SubmitHandler)
	http.HandleFunc("/unreleased", unreleasedHandler)
	http.HandleFunc("/createsite", createsiteHandler)

	http.HandleFunc("/index.html", indexHandler)
	http.HandleFunc("/robots.txt", robotsHandler)
	http.HandleFunc("/google41fd03a6c9348593.html", googleAdwordsVerifyHandler)

	http.Handle("/css/", http.StripPrefix("/css/", http.FileServer(http.Dir("./css"))))
	http.Handle("/font-awesome-4.7.0/", http.StripPrefix("/font-awesome-4.7.0/", http.FileServer(http.Dir("./font-awesome-4.7.0"))))
	http.Handle("/fonts/", http.StripPrefix("/fonts/", http.FileServer(http.Dir("./fonts"))))
	http.Handle("/images/", http.StripPrefix("/images/", http.FileServer(http.Dir("./images"))))
	http.Handle("/js/", http.StripPrefix("/js/", http.FileServer(http.Dir("./js"))))
	http.Handle("/screenshots/", http.StripPrefix("/screenshots/", http.FileServer(http.Dir("./screenshots"))))
	http.HandleFunc("/", indexHandler)

	// HTTP to HTTPS redirection
	// go func() {
	// 	err := http.ListenAndServe(":80", http.HandlerFunc(redir))
	// 	if err != nil {
	// 		log.Print("HTTP ListenAndServe :8080", err)
	// 		log.Print("Trying HTTP ListenAndServe :8080.")
	// 		panic(http.ListenAndServe(":8080", http.HandlerFunc(redir)))

	// 	}
	// }()

	if !db.Exists(db.DbName) {
		db.Create(db.DbName)
	}

	go func() {
		err := http.ListenAndServe(":80", nil)
		if err != nil {
			log.Print("HTTP ListenAndServe :80, ", err)
			log.Print("Trying HTTP ListenAndServe :8080.")
			if err != nil {
				panic(http.ListenAndServe(":8080", nil))
			}

		}
	}()

	cert := "/etc/letsencrypt/live/renfish.com/cert.pem"
	privkey := "/etc/letsencrypt/live/renfish.com/privkey.pem"
	if _, err := os.Stat(cert); os.IsNotExist(err) {
		log.Print("cert: ", err)
		cert = "./generate_cert/cert.pem"
	}
	if _, err := os.Stat(privkey); os.IsNotExist(err) {
		log.Print("cert: ", err)
		privkey = "./generate_cert/key.pem"
	}
	err := http.ListenAndServeTLS(":443", cert, privkey, nil)
	if err != nil {
		log.Print("HTTPS ListenAndServe :8443")
		err = http.ListenAndServeTLS(":8443", cert, privkey, nil)
		if err != nil {
			log.Print("HTTPS ListenAndServe: ", err)
		}
	}
}
<|endoftext|>


<commit_before>package goinsta

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

type reqOptions struct {
	Endpoint     string
	PostData     string
	IsLoggedIn   bool
	IgnoreStatus bool
	Query        map[string]string
}

func (insta *Instagram) sendSimpleRequest(endpoint string, a ...interface{}) (body []byte, err error) {
	return insta.sendRequest(&reqOptions{
		Endpoint: fmt.Sprintf(endpoint, a...),
	})
}

func (insta *Instagram) sendRequest(o *reqOptions) (body []byte, err error) {

	if !insta.IsLoggedIn && !o.IsLoggedIn {
		return nil, fmt.Errorf("not logged in")
	}

	method := "GET"
	if len(o.PostData) > 0 {
		method = "POST"
	}

	u, err := url.Parse(GOINSTA_API_URL + o.Endpoint)
	if err != nil {
		return nil, err
	}

	q := u.Query()
	for k, v := range o.Query {
		q.Add(k, v)
	}
	u.RawQuery = q.Encode()

	var req *http.Request
	req, err = http.NewRequest(method, u.String(), bytes.NewBuffer([]byte(o.PostData)))
	if err != nil {
		return
	}

	req.Header.Set("Connection", "close")
	req.Header.Set("Accept", "*/*")
	req.Header.Set("Content-type", "application/x-www-form-urlencoded; charset=UTF-8")
	req.Header.Set("Cookie2", "$Version=1")
	req.Header.Set("Accept-Language", "en-US")
	req.Header.Set("User-Agent", GOINSTA_USER_AGENT)

	client := &http.Client{
		Jar: insta.cookiejar,
	}
	if insta.proxy != "" {
		proxy, err := url.Parse(insta.proxy)
		if err != nil {
			return body, err
		}
		client.Transport = &http.Transport{Proxy: http.ProxyURL(proxy)}
	}

	resp, err := client.Do(req)
	if err != nil {
		return body, err
	}
	defer resp.Body.Close()

	u, _ = url.Parse(GOINSTA_API_URL)
	for _, value := range insta.cookiejar.Cookies(u) {
		if strings.Contains(value.Name, "csrftoken") {
			insta.Informations.Token = value.Value
		}
	}

	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	if resp.StatusCode != 200 && !o.IgnoreStatus {
		return nil, fmt.Errorf("Invalid status code %s", string(body))
	}

	return body, err
}
<commit_msg>add OptionalRequest<commit_after>package goinsta

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

type reqOptions struct {
	Endpoint     string
	PostData     string
	IsLoggedIn   bool
	IgnoreStatus bool
	Query        map[string]string
}

func (insta *Instagram) OptionalRequest(endpoint string, a ...interface{}) (body []byte, err error) {
	return insta.sendRequest(&reqOptions{
		Endpoint: fmt.Sprintf(endpoint, a...),
	})
}

func (insta *Instagram) sendSimpleRequest(endpoint string, a ...interface{}) (body []byte, err error) {
	return insta.sendRequest(&reqOptions{
		Endpoint: fmt.Sprintf(endpoint, a...),
	})
}

func (insta *Instagram) sendRequest(o *reqOptions) (body []byte, err error) {

	if !insta.IsLoggedIn && !o.IsLoggedIn {
		return nil, fmt.Errorf("not logged in")
	}

	method := "GET"
	if len(o.PostData) > 0 {
		method = "POST"
	}

	u, err := url.Parse(GOINSTA_API_URL + o.Endpoint)
	if err != nil {
		return nil, err
	}

	q := u.Query()
	for k, v := range o.Query {
		q.Add(k, v)
	}
	u.RawQuery = q.Encode()

	var req *http.Request
	req, err = http.NewRequest(method, u.String(), bytes.NewBuffer([]byte(o.PostData)))
	if err != nil {
		return
	}

	req.Header.Set("Connection", "close")
	req.Header.Set("Accept", "*/*")
	req.Header.Set("Content-type", "application/x-www-form-urlencoded; charset=UTF-8")
	req.Header.Set("Cookie2", "$Version=1")
	req.Header.Set("Accept-Language", "en-US")
	req.Header.Set("User-Agent", GOINSTA_USER_AGENT)

	client := &http.Client{
		Jar: insta.cookiejar,
	}
	if insta.proxy != "" {
		proxy, err := url.Parse(insta.proxy)
		if err != nil {
			return body, err
		}
		client.Transport = &http.Transport{Proxy: http.ProxyURL(proxy)}
	}

	resp, err := client.Do(req)
	if err != nil {
		return body, err
	}
	defer resp.Body.Close()

	u, _ = url.Parse(GOINSTA_API_URL)
	for _, value := range insta.cookiejar.Cookies(u) {
		if strings.Contains(value.Name, "csrftoken") {
			insta.Informations.Token = value.Value
		}
	}

	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	if resp.StatusCode != 200 && !o.IgnoreStatus {
		return nil, fmt.Errorf("Invalid status code %s", string(body))
	}

	return body, err
}
<|endoftext|>


<commit_before>package typhon

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/monzo/terrors"
	"google.golang.org/protobuf/encoding/protojson"
	"google.golang.org/protobuf/proto"
)

// A Request is Typhon's wrapper around http.Request, used by both clients and servers.
//
// Note that Typhon makes no guarantees that a Request is safe to access or mutate concurrently. If a single Request
// object is to be used by multiple goroutines concurrently, callers must make sure to properly synchronise accesses.
type Request struct {
	http.Request
	context.Context
	err      error // Any error from request construction; read by ErrorFilter
	hijacker http.Hijacker
	server   *Server
}

// unwrappedContext returns the most "unwrapped" Context possible for that in the request.
// This is useful as it's very often the case that Typhon users will use a parent request
// as a parent for a child request. The context library knows how to unwrap its own
// types to most efficiently perform certain operations (eg. cancellation chaining), but
// it can't do that with Typhon-wrapped contexts.
func (r *Request) unwrappedContext() context.Context {
	switch c := r.Context.(type) {
	case Request:
		return c.unwrappedContext()
	case *Request:
		return c.unwrappedContext()
	default:
		return c
	}
}

// Encode maps to to EncodeAsJSON
// TODO: Remove in the next major release and require encoding to explicitly go through either EncodeAsJSON, EncodeAsProtoJSON or EncodeAsProtobuf
func (r *Request) Encode(v interface{}) {
	r.EncodeAsJSON(v)
}

// EncodeAsJSON serialises the passed object as JSON into the body (and sets appropriate headers).
func (r *Request) EncodeAsJSON(v interface{}) {
	// If we were given an io.ReadCloser or an io.Reader (that is not also a json.Marshaler), use it directly
	switch v := v.(type) {
	case json.Marshaler:
	case io.ReadCloser:
		r.Body = v
		r.ContentLength = -1
		return
	case io.Reader:
		r.Body = ioutil.NopCloser(v)
		r.ContentLength = -1
		return
	}

	if err := json.NewEncoder(r).Encode(v); err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}
	r.Header.Set("Content-Type", "application/json")
}

// EncodeAsProtoJSON serialises the passed object as ProtoJSON into the body (and sets appropriate headers).
func (r *Request) EncodeAsProtoJSON(m proto.Message) {
	out, err := protojson.Marshal(m)
	if err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}

	n, err := r.Write(out)
	if err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}
	r.Header.Set("Content-Type", "application/jsonpb")
	r.ContentLength = int64(n)
}

// EncodeAsProtobuf serialises the passed object as protobuf into the body (and sets appropriate headers).
func (r *Request) EncodeAsProtobuf(m proto.Message) {
	out, err := proto.Marshal(m)
	if err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}

	n, err := r.Write(out)
	if err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}
	r.Header.Set("Content-Type", "application/protobuf")
	r.ContentLength = int64(n)
}

// Decode de-serialises the body into the passed object.
func (r Request) Decode(v interface{}) error {
	b, err := r.BodyBytes(true)
	if err != nil {
		return terrors.WrapWithCode(err, nil, terrors.ErrBadRequest)
	}

	switch r.Header.Get("Content-Type") {
	// application/x-protobuf is the "canonical" use, application/protobuf is defined in an expired IETF draft.
	// See: https://datatracker.ietf.org/doc/html/draft-rfernando-protocol-buffers-00#section-3.2
	// See: https://github.com/google/protorpc/blob/eb03145/python/protorpc/protobuf.py#L49-L51
	case "application/octet-stream", "application/x-google-protobuf", "application/protobuf", "application/x-protobuf":
		m, ok := v.(proto.Message)
		if !ok {
			return terrors.InternalService("invalid_type", "could not decode proto message", nil)
		}
		err = proto.Unmarshal(b, m)
	// Proper JSON handling requires the protojson package in Go. application/jsonpb is a suggestion by grpc-gateway:
	// https://github.com/grpc-ecosystem/grpc-gateway/blob/f4371f7/runtime/marshaler_registry.go#L89-L90
	// This is a backward compatibility break for those using google.golang.org/protobuf/proto.Message incorrectly.

	// Older versions of typhon marshal/unmarshal using json, to prevent a regression, we only use protojson if the
	// content-type explicitly declares that this message is protojson
	case "application/jsonpb", "application/protojson":
		m, ok := v.(proto.Message)
		if !ok {
			return terrors.InternalService("invalid_type", "could not decode proto message", nil)
		}
		err = protojson.Unmarshal(b, m)

	case "application/json", "text/json":
		m, ok := v.(proto.Message)
		if !ok {
			return terrors.InternalService("invalid_type", "could not decode proto message", nil)
		}
		err = json.Unmarshal(b, m)

	default:
		err = terrors.BadRequest(
				terrors.ErrBadRequest,
				"not a supported Content-Type",
				map[string]string{
					"content-type": r.Header.Get("Content-Type"),
				},
			)
	}

	return terrors.WrapWithCode(err, nil, terrors.ErrBadRequest)
}

// Write writes the passed bytes to the request's body.
func (r *Request) Write(b []byte) (n int, err error) {
	switch rc := r.Body.(type) {
	// In the "normal" case, the response body will be a buffer, to which we can write
	case io.Writer:
		n, err = rc.Write(b)
		if err != nil {
			return n, err
		}
	// If a caller manually sets Response.Body, then we may not be able to write to it. In that case, we need to be
	// cleverer.
	default:
		buf := &bufCloser{}
		if rc != nil {
			if _, err := io.Copy(buf, rc); err != nil {
				// This can be quite bad; we have consumed (and possibly lost) some of the original body
				return 0, err
			}
			// rc will never again be accessible: once it's copied it must be closed
			rc.Close()
		}
		r.Body = buf
		n, err = buf.Write(b)
		if err != nil {
			return n, err
		}
	}

	if r.ContentLength >= 0 {
		r.ContentLength += int64(n)
		// If this write pushed the content length above the chunking threshold,
		// set to -1 (unknown) to trigger chunked encoding
		if r.ContentLength >= chunkThreshold {
			r.ContentLength = -1
		}
	}
	return n, nil
}

// BodyBytes fully reads the request body and returns the bytes read.
//
// If consume is true, this is equivalent to ioutil.ReadAll; if false, the caller will observe the body to be in
// the same state that it was before (ie. any remaining unread body can be read again).
func (r *Request) BodyBytes(consume bool) ([]byte, error) {
	if consume {
		defer r.Body.Close()
		return ioutil.ReadAll(r.Body)
	}

	switch rc := r.Body.(type) {
	case *bufCloser:
		return rc.Bytes(), nil
	default:
		buf := &bufCloser{}
		r.Body = buf
		rdr := io.TeeReader(rc, buf)
		// rc will never again be accessible: once it's copied it must be closed
		defer rc.Close()
		return ioutil.ReadAll(rdr)
	}
}

// Send round-trips the request via the default Client. It does not block, instead returning a ResponseFuture
// representing the asynchronous operation to produce the response. It is equivalent to:
//
//  r.SendVia(Client)
func (r Request) Send() *ResponseFuture {
	return Send(r)
}

// SendVia round-trips the request via the passed Service. It does not block, instead returning a ResponseFuture
// representing the asynchronous operation to produce the response.
func (r Request) SendVia(svc Service) *ResponseFuture {
	return SendVia(r, svc)
}

// Response constructs a new Response to the request, and if non-nil, encodes the given body into it.
func (r Request) Response(body interface{}) Response {
	rsp := NewResponse(r)
	if body != nil {
		rsp.Encode(body)
	}
	return rsp
}

// ResponseWithCode constructs a new Response with the given status code to the request, and if non-nil, encodes the
// given body into it.
func (r Request) ResponseWithCode(body interface{}, statusCode int) Response {
	rsp := NewResponseWithCode(r, statusCode)
	if body != nil {
		rsp.Encode(body)
	}
	return rsp
}

func (r Request) String() string {
	if r.URL == nil {
		return "Request(Unknown)"
	}
	return fmt.Sprintf("Request(%s %s://%s%s)", r.Method, r.URL.Scheme, r.Host, r.URL.Path)
}

// NewRequest constructs a new Request with the given parameters, and if non-nil, encodes the given body into it.
func NewRequest(ctx context.Context, method, url string, body interface{}) Request {
	if ctx == nil {
		ctx = context.Background()
	}
	httpReq, err := http.NewRequest(method, url, nil)
	req := Request{
		Context: ctx,
		err:     err}
	if httpReq != nil {
		httpReq.ContentLength = 0
		httpReq.Body = &bufCloser{}
		req.Request = *httpReq

		// Attach any metadata in the context to the request as headers.
		meta := MetadataFromContext(ctx)
		for k, v := range meta {
			req.Header[strings.ToLower(k)] = v
		}
	}
	if body != nil && err == nil {
		req.EncodeAsJSON(body)
	}
	return req
}
<commit_msg>Capitalisation fix<commit_after>package typhon

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/monzo/terrors"
	"google.golang.org/protobuf/encoding/protojson"
	"google.golang.org/protobuf/proto"
)

// A Request is Typhon's wrapper around http.Request, used by both clients and servers.
//
// Note that Typhon makes no guarantees that a Request is safe to access or mutate concurrently. If a single Request
// object is to be used by multiple goroutines concurrently, callers must make sure to properly synchronise accesses.
type Request struct {
	http.Request
	context.Context
	err      error // Any error from request construction; read by ErrorFilter
	hijacker http.Hijacker
	server   *Server
}

// unwrappedContext returns the most "unwrapped" Context possible for that in the request.
// This is useful as it's very often the case that Typhon users will use a parent request
// as a parent for a child request. The context library knows how to unwrap its own
// types to most efficiently perform certain operations (eg. cancellation chaining), but
// it can't do that with Typhon-wrapped contexts.
func (r *Request) unwrappedContext() context.Context {
	switch c := r.Context.(type) {
	case Request:
		return c.unwrappedContext()
	case *Request:
		return c.unwrappedContext()
	default:
		return c
	}
}

// Encode maps to to EncodeAsJSON
// TODO: Remove in the next major release and require encoding to explicitly go through either EncodeAsJSON, EncodeAsProtoJSON or EncodeAsProtobuf
func (r *Request) Encode(v interface{}) {
	r.EncodeAsJSON(v)
}

// EncodeAsJSON serialises the passed object as JSON into the body (and sets appropriate headers).
func (r *Request) EncodeAsJSON(v interface{}) {
	// If we were given an io.ReadCloser or an io.Reader (that is not also a json.Marshaler), use it directly
	switch v := v.(type) {
	case json.Marshaler:
	case io.ReadCloser:
		r.Body = v
		r.ContentLength = -1
		return
	case io.Reader:
		r.Body = ioutil.NopCloser(v)
		r.ContentLength = -1
		return
	}

	if err := json.NewEncoder(r).Encode(v); err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}
	r.Header.Set("Content-Type", "application/json")
}

// EncodeAsProtoJSON serialises the passed object as ProtoJSON into the body (and sets appropriate headers).
func (r *Request) EncodeAsProtoJSON(m proto.Message) {
	out, err := protojson.Marshal(m)
	if err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}

	n, err := r.Write(out)
	if err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}
	r.Header.Set("Content-Type", "application/jsonpb")
	r.ContentLength = int64(n)
}

// EncodeAsProtobuf serialises the passed object as protobuf into the body (and sets appropriate headers).
func (r *Request) EncodeAsProtobuf(m proto.Message) {
	out, err := proto.Marshal(m)
	if err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}

	n, err := r.Write(out)
	if err != nil {
		r.err = terrors.Wrap(err, nil)
		return
	}
	r.Header.Set("Content-Type", "application/protobuf")
	r.ContentLength = int64(n)
}

// Decode de-serialises the body into the passed object.
func (r Request) Decode(v interface{}) error {
	b, err := r.BodyBytes(true)
	if err != nil {
		return terrors.WrapWithCode(err, nil, terrors.ErrBadRequest)
	}

	switch r.Header.Get("Content-Type") {
	// application/x-protobuf is the "canonical" use, application/protobuf is defined in an expired IETF draft.
	// See: https://datatracker.ietf.org/doc/html/draft-rfernando-protocol-buffers-00#section-3.2
	// See: https://github.com/google/protorpc/blob/eb03145/python/protorpc/protobuf.py#L49-L51
	case "application/octet-stream", "application/x-google-protobuf", "application/protobuf", "application/x-protobuf":
		m, ok := v.(proto.Message)
		if !ok {
			return terrors.InternalService("invalid_type", "could not decode proto message", nil)
		}
		err = proto.Unmarshal(b, m)
	// Proper JSON handling requires the protojson package in Go. application/jsonpb is a suggestion by grpc-gateway:
	// https://github.com/grpc-ecosystem/grpc-gateway/blob/f4371f7/runtime/marshaler_registry.go#L89-L90
	// This is a backward compatibility break for those using google.golang.org/protobuf/proto.Message incorrectly.

	// Older versions of typhon marshal/unmarshal using json, to prevent a regression, we only use protojson if the
	// content-type explicitly declares that this message is protojson
	case "application/jsonpb", "application/protojson":
		m, ok := v.(proto.Message)
		if !ok {
			return terrors.InternalService("invalid_type", "could not decode proto message", nil)
		}
		err = protojson.Unmarshal(b, m)

	case "application/json", "text/json":
		m, ok := v.(proto.Message)
		if !ok {
			return terrors.InternalService("invalid_type", "could not decode proto message", nil)
		}
		err = json.Unmarshal(b, m)

	default:
		err = terrors.BadRequest(
				terrors.ErrBadRequest,
				"not a supported Content-Type",
				map[string]string{
					"Content-Type": r.Header.Get("Content-Type"),
				},
			)
	}

	return terrors.WrapWithCode(err, nil, terrors.ErrBadRequest)
}

// Write writes the passed bytes to the request's body.
func (r *Request) Write(b []byte) (n int, err error) {
	switch rc := r.Body.(type) {
	// In the "normal" case, the response body will be a buffer, to which we can write
	case io.Writer:
		n, err = rc.Write(b)
		if err != nil {
			return n, err
		}
	// If a caller manually sets Response.Body, then we may not be able to write to it. In that case, we need to be
	// cleverer.
	default:
		buf := &bufCloser{}
		if rc != nil {
			if _, err := io.Copy(buf, rc); err != nil {
				// This can be quite bad; we have consumed (and possibly lost) some of the original body
				return 0, err
			}
			// rc will never again be accessible: once it's copied it must be closed
			rc.Close()
		}
		r.Body = buf
		n, err = buf.Write(b)
		if err != nil {
			return n, err
		}
	}

	if r.ContentLength >= 0 {
		r.ContentLength += int64(n)
		// If this write pushed the content length above the chunking threshold,
		// set to -1 (unknown) to trigger chunked encoding
		if r.ContentLength >= chunkThreshold {
			r.ContentLength = -1
		}
	}
	return n, nil
}

// BodyBytes fully reads the request body and returns the bytes read.
//
// If consume is true, this is equivalent to ioutil.ReadAll; if false, the caller will observe the body to be in
// the same state that it was before (ie. any remaining unread body can be read again).
func (r *Request) BodyBytes(consume bool) ([]byte, error) {
	if consume {
		defer r.Body.Close()
		return ioutil.ReadAll(r.Body)
	}

	switch rc := r.Body.(type) {
	case *bufCloser:
		return rc.Bytes(), nil
	default:
		buf := &bufCloser{}
		r.Body = buf
		rdr := io.TeeReader(rc, buf)
		// rc will never again be accessible: once it's copied it must be closed
		defer rc.Close()
		return ioutil.ReadAll(rdr)
	}
}

// Send round-trips the request via the default Client. It does not block, instead returning a ResponseFuture
// representing the asynchronous operation to produce the response. It is equivalent to:
//
//  r.SendVia(Client)
func (r Request) Send() *ResponseFuture {
	return Send(r)
}

// SendVia round-trips the request via the passed Service. It does not block, instead returning a ResponseFuture
// representing the asynchronous operation to produce the response.
func (r Request) SendVia(svc Service) *ResponseFuture {
	return SendVia(r, svc)
}

// Response constructs a new Response to the request, and if non-nil, encodes the given body into it.
func (r Request) Response(body interface{}) Response {
	rsp := NewResponse(r)
	if body != nil {
		rsp.Encode(body)
	}
	return rsp
}

// ResponseWithCode constructs a new Response with the given status code to the request, and if non-nil, encodes the
// given body into it.
func (r Request) ResponseWithCode(body interface{}, statusCode int) Response {
	rsp := NewResponseWithCode(r, statusCode)
	if body != nil {
		rsp.Encode(body)
	}
	return rsp
}

func (r Request) String() string {
	if r.URL == nil {
		return "Request(Unknown)"
	}
	return fmt.Sprintf("Request(%s %s://%s%s)", r.Method, r.URL.Scheme, r.Host, r.URL.Path)
}

// NewRequest constructs a new Request with the given parameters, and if non-nil, encodes the given body into it.
func NewRequest(ctx context.Context, method, url string, body interface{}) Request {
	if ctx == nil {
		ctx = context.Background()
	}
	httpReq, err := http.NewRequest(method, url, nil)
	req := Request{
		Context: ctx,
		err:     err}
	if httpReq != nil {
		httpReq.ContentLength = 0
		httpReq.Body = &bufCloser{}
		req.Request = *httpReq

		// Attach any metadata in the context to the request as headers.
		meta := MetadataFromContext(ctx)
		for k, v := range meta {
			req.Header[strings.ToLower(k)] = v
		}
	}
	if body != nil && err == nil {
		req.EncodeAsJSON(body)
	}
	return req
}
<|endoftext|>