<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package registry // import "helm.sh/helm/v3/internal/experimental/registry"

import (
	"encoding/json"
	"fmt"

	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"oras.land/oras-go/pkg/content"
	"oras.land/oras-go/pkg/oras"

	"helm.sh/helm/v3/pkg/chart"
)

// Pull downloads a chart from a registry
func (c *Client) Pull(ref string, options ...PullOption) (*PullResult, error) {
	operation := &pullOperation{
		withChart: true, // By default, always download the chart layer
	}
	for _, option := range options {
		option(operation)
	}
	if !operation.withChart && !operation.withProv {
		return nil, errors.New(
			"must specify at least one layer to pull (chart/prov)")
	}
	store := content.NewMemoryStore()
	allowedMediaTypes := []string{
		ConfigMediaType,
	}
	minNumDescriptors := 1 // 1 for the config
	if operation.withChart {
		minNumDescriptors++
		allowedMediaTypes = append(allowedMediaTypes, ChartLayerMediaType)
	}
	if operation.withProv {
		if !operation.ignoreMissingProv {
			minNumDescriptors++
		}
		allowedMediaTypes = append(allowedMediaTypes, ProvLayerMediaType)
	}
	manifest, descriptors, err := oras.Pull(ctx(c.out, c.debug), c.resolver, ref, store,
		oras.WithPullEmptyNameAllowed(),
		oras.WithAllowedMediaTypes(allowedMediaTypes))
	if err != nil {
		return nil, err
	}
	numDescriptors := len(descriptors)
	if numDescriptors < minNumDescriptors {
		return nil, errors.New(
			fmt.Sprintf("manifest does not contain minimum number of descriptors (%d), descriptors found: %d",
				minNumDescriptors, numDescriptors))
	}
	var configDescriptor *ocispec.Descriptor
	var chartDescriptor *ocispec.Descriptor
	var provDescriptor *ocispec.Descriptor
	for _, descriptor := range descriptors {
		d := descriptor
		switch d.MediaType {
		case ConfigMediaType:
			configDescriptor = &d
		case ChartLayerMediaType:
			chartDescriptor = &d
		case ProvLayerMediaType:
			provDescriptor = &d
		}
	}
	if configDescriptor == nil {
		return nil, errors.New(
			fmt.Sprintf("could not load config with mediatype %s", ConfigMediaType))
	}
	if operation.withChart && chartDescriptor == nil {
		return nil, errors.New(
			fmt.Sprintf("manifest does not contain a layer with mediatype %s",
				ChartLayerMediaType))
	}
	var provMissing bool
	if operation.withProv && provDescriptor == nil {
		if operation.ignoreMissingProv {
			provMissing = true
		} else {
			return nil, errors.New(
				fmt.Sprintf("manifest does not contain a layer with mediatype %s",
					ProvLayerMediaType))
		}
	}
	_, manifestData, ok := store.Get(manifest)
	if !ok {
		return nil, errors.Errorf("Unable to retrieve blob with digest %s", manifest.Digest)
	}
	_, configData, ok := store.Get(*configDescriptor)
	if !ok {
		return nil, errors.Errorf("Unable to retrieve blob with digest %s", configDescriptor.Digest)
	}
	var meta *chart.Metadata
	err = json.Unmarshal(configData, &meta)
	if err != nil {
		return nil, err
	}
	var chartData []byte
	if operation.withChart {
		var ok bool
		_, chartData, ok = store.Get(*chartDescriptor)
		if !ok {
			return nil, errors.Errorf("Unable to retrieve blob with digest %s", chartDescriptor.Digest)
		}
	}
	var provData []byte
	if operation.withProv && !provMissing {
		var ok bool
		_, provData, ok = store.Get(*provDescriptor)
		if !ok {
			return nil, errors.Errorf("Unable to retrieve blob with digest %s", provDescriptor.Digest)
		}
	}
	result := &PullResult{
		Manifest: &descriptorPullSummary{
			Data:   manifestData,
			Digest: manifest.Digest.String(),
			Size:   manifest.Size,
		},
		Config: &descriptorPullSummary{
			Data:   configData,
			Digest: configDescriptor.Digest.String(),
			Size:   configDescriptor.Size,
		},
		Chart: &descriptorPullSummaryWithMeta{
			Meta: meta,
		},
		Prov: &descriptorPullSummary{}, // prevent nil references
		Ref:  ref,
	}
	if chartData != nil {
		result.Chart.Data = chartData
		result.Chart.Digest = chartDescriptor.Digest.String()
		result.Chart.Size = chartDescriptor.Size
	}
	if provData != nil {
		result.Prov = &descriptorPullSummary{
			Data:   provData,
			Digest: provDescriptor.Digest.String(),
			Size:   provDescriptor.Size,
		}
	}
	fmt.Fprintf(c.out, "Pulled: %s\n", result.Ref)
	fmt.Fprintf(c.out, "Digest: %s\n", result.Manifest.Digest)
	return result, nil
}
<commit_msg>one-line JSON unmarshall<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package registry // import "helm.sh/helm/v3/internal/experimental/registry"

import (
	"encoding/json"
	"fmt"

	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"oras.land/oras-go/pkg/content"
	"oras.land/oras-go/pkg/oras"

	"helm.sh/helm/v3/pkg/chart"
)

// Pull downloads a chart from a registry
func (c *Client) Pull(ref string, options ...PullOption) (*PullResult, error) {
	operation := &pullOperation{
		withChart: true, // By default, always download the chart layer
	}
	for _, option := range options {
		option(operation)
	}
	if !operation.withChart && !operation.withProv {
		return nil, errors.New(
			"must specify at least one layer to pull (chart/prov)")
	}
	store := content.NewMemoryStore()
	allowedMediaTypes := []string{
		ConfigMediaType,
	}
	minNumDescriptors := 1 // 1 for the config
	if operation.withChart {
		minNumDescriptors++
		allowedMediaTypes = append(allowedMediaTypes, ChartLayerMediaType)
	}
	if operation.withProv {
		if !operation.ignoreMissingProv {
			minNumDescriptors++
		}
		allowedMediaTypes = append(allowedMediaTypes, ProvLayerMediaType)
	}
	manifest, descriptors, err := oras.Pull(ctx(c.out, c.debug), c.resolver, ref, store,
		oras.WithPullEmptyNameAllowed(),
		oras.WithAllowedMediaTypes(allowedMediaTypes))
	if err != nil {
		return nil, err
	}
	numDescriptors := len(descriptors)
	if numDescriptors < minNumDescriptors {
		return nil, errors.New(
			fmt.Sprintf("manifest does not contain minimum number of descriptors (%d), descriptors found: %d",
				minNumDescriptors, numDescriptors))
	}
	var configDescriptor *ocispec.Descriptor
	var chartDescriptor *ocispec.Descriptor
	var provDescriptor *ocispec.Descriptor
	for _, descriptor := range descriptors {
		d := descriptor
		switch d.MediaType {
		case ConfigMediaType:
			configDescriptor = &d
		case ChartLayerMediaType:
			chartDescriptor = &d
		case ProvLayerMediaType:
			provDescriptor = &d
		}
	}
	if configDescriptor == nil {
		return nil, errors.New(
			fmt.Sprintf("could not load config with mediatype %s", ConfigMediaType))
	}
	if operation.withChart && chartDescriptor == nil {
		return nil, errors.New(
			fmt.Sprintf("manifest does not contain a layer with mediatype %s",
				ChartLayerMediaType))
	}
	var provMissing bool
	if operation.withProv && provDescriptor == nil {
		if operation.ignoreMissingProv {
			provMissing = true
		} else {
			return nil, errors.New(
				fmt.Sprintf("manifest does not contain a layer with mediatype %s",
					ProvLayerMediaType))
		}
	}
	_, manifestData, ok := store.Get(manifest)
	if !ok {
		return nil, errors.Errorf("Unable to retrieve blob with digest %s", manifest.Digest)
	}
	_, configData, ok := store.Get(*configDescriptor)
	if !ok {
		return nil, errors.Errorf("Unable to retrieve blob with digest %s", configDescriptor.Digest)
	}
	var meta *chart.Metadata
	if err := json.Unmarshal(configData, &meta); err != nil {
		return nil, err
	}
	var chartData []byte
	if operation.withChart {
		var ok bool
		_, chartData, ok = store.Get(*chartDescriptor)
		if !ok {
			return nil, errors.Errorf("Unable to retrieve blob with digest %s", chartDescriptor.Digest)
		}
	}
	var provData []byte
	if operation.withProv && !provMissing {
		var ok bool
		_, provData, ok = store.Get(*provDescriptor)
		if !ok {
			return nil, errors.Errorf("Unable to retrieve blob with digest %s", provDescriptor.Digest)
		}
	}
	result := &PullResult{
		Manifest: &descriptorPullSummary{
			Data:   manifestData,
			Digest: manifest.Digest.String(),
			Size:   manifest.Size,
		},
		Config: &descriptorPullSummary{
			Data:   configData,
			Digest: configDescriptor.Digest.String(),
			Size:   configDescriptor.Size,
		},
		Chart: &descriptorPullSummaryWithMeta{
			Meta: meta,
		},
		Prov: &descriptorPullSummary{}, // prevent nil references
		Ref:  ref,
	}
	if chartData != nil {
		result.Chart.Data = chartData
		result.Chart.Digest = chartDescriptor.Digest.String()
		result.Chart.Size = chartDescriptor.Size
	}
	if provData != nil {
		result.Prov = &descriptorPullSummary{
			Data:   provData,
			Digest: provDescriptor.Digest.String(),
			Size:   provDescriptor.Size,
		}
	}
	fmt.Fprintf(c.out, "Pulled: %s\n", result.Ref)
	fmt.Fprintf(c.out, "Digest: %s\n", result.Manifest.Digest)
	return result, nil
}
<|endoftext|>


<commit_before>// Copyright 2013-2014 %name% authors.  All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"database/sql"
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"

	html "html/template"
	text "text/template"

	"github.com/fiorix/go-redis/redis"
	_ "github.com/go-sql-driver/mysql"
)

const (
	VERSION = "1.0"
	APPNAME = "%name%"
)

var (
	Config *ConfigData

	// Templates
	HTML *html.Template
	TEXT *text.Template

	// DBs
	MySQL *sql.DB
	Redis *redis.Client
)

func main() {
	Configfile := flag.String("c", "%name%.conf", "set config file")
	flag.Usage = func() {
		fmt.Println("Usage: %name% [-c %name%.conf]")
		os.Exit(1)
	}
	flag.Parse()

	var err error
	Config, err = LoadConfig(*Configfile)
	if err != nil {
		log.Fatal(err)
	}

	// Parse templates.
	HTML = html.Must(html.ParseGlob(Config.TemplatesDir + "/*.html"))
	TEXT = text.Must(text.ParseGlob(Config.TemplatesDir + "/*.txt"))

	// Set up databases.
	Redis = redis.New(Config.DB.Redis)
	MySQL, err = sql.Open("mysql", Config.DB.MySQL)
	if err != nil {
		log.Fatal(err)
	}

	// Set GOMAXPROCS and show server info.
	var cpuinfo string
	if n := runtime.NumCPU(); n > 1 {
		runtime.GOMAXPROCS(n)
		cpuinfo = fmt.Sprintf("%d CPUs", n)
	} else {
		cpuinfo = "1 CPU"
	}
	log.Printf("%s v%s (%s)", APPNAME, VERSION, cpuinfo)

	// Add routes, and run HTTP and HTTPS servers.
	routeHTTP()
	if Config.HTTP.Addr != "" {
		go listenHTTP()
	}
	if Config.HTTPS.Addr != "" {
		go listenHTTPS()
	}
	select {}
}
<commit_msg>Added support for log file<commit_after>// Copyright 2013-2014 %name% authors.  All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"database/sql"
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	html "html/template"
	text "text/template"

	"github.com/fiorix/go-redis/redis"
	_ "github.com/go-sql-driver/mysql"
)

const (
	VERSION = "1.0"
	APPNAME = "%name%"
)

var (
	Config *ConfigData

	// Templates
	HTML *html.Template
	TEXT *text.Template

	// DBs
	MySQL *sql.DB
	Redis *redis.Client
)

func main() {
	configFile := flag.String("c", "%name%.conf", "")
	logFile := flag.String("logfile", "", "")
	flag.Usage = func() {
		fmt.Println("Usage: %name% [-c %name%.conf] [-logfile FILE]")
		os.Exit(1)
	}
	flag.Parse()

	var err error
	Config, err = LoadConfig(*configFile)
	if err != nil {
		log.Fatal(err)
	}

	// Initialize log
	if *logFile != "" {
		setLog(*logFile)
	}

	// Parse templates.
	HTML = html.Must(html.ParseGlob(Config.TemplatesDir + "/*.html"))
	TEXT = text.Must(text.ParseGlob(Config.TemplatesDir + "/*.txt"))

	// Set up databases.
	Redis = redis.New(Config.DB.Redis)
	MySQL, err = sql.Open("mysql", Config.DB.MySQL)
	if err != nil {
		log.Fatal(err)
	}

	// Set GOMAXPROCS and show server info.
	var cpuinfo string
	if n := runtime.NumCPU(); n > 1 {
		runtime.GOMAXPROCS(n)
		cpuinfo = fmt.Sprintf("%d CPUs", n)
	} else {
		cpuinfo = "1 CPU"
	}
	log.Printf("%s v%s (%s)", APPNAME, VERSION, cpuinfo)

	// Add routes, and run HTTP and HTTPS servers.
	routeHTTP()
	if Config.HTTP.Addr != "" {
		go listenHTTP()
	}
	if Config.HTTPS.Addr != "" {
		go listenHTTPS()
	}
	select {}
}

func setLog(filename string) {
	f := openLog(filename)
	log.SetOutput(f)
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, syscall.SIGHUP)
	go func() {
		// Recycle log file on SIGHUP.
		<-sigc
		f.Close()
		f = openLog(filename)
		log.SetOutput(f)
	}()
}

func openLog(filename string) *os.File {
	f, err := os.OpenFile(
		filename,
		os.O_WRONLY|os.O_CREATE|os.O_APPEND,
		0666,
	)
	if err != nil {
		log.SetOutput(os.Stderr)
		log.Fatal(err)
	}
	return f
}
<|endoftext|>


<commit_before>package server

import (
	"gopkg.in/gin-gonic/gin.v1"
	"log"
	"net/http"
	"path/filepath"
	"database/sql"
)

type Login struct {
	User     string `form:"username" json:"username" binding:"required"`
	Password string `form:"password" json:"password" binding:"required"`
}

type Signup struct {
	User     string `form:"username" json:"username" binding:"required"`
	Password string `form:"password" json:"password" binding:"required"`
	Email string `form:"email" json:"email" binding:"required"`
}

func routers(r *gin.Engine) {

	r.LoadHTMLGlob(filepath.Join(staticPrefix, "views/*"))

	db := getDB()

	// 主页
	r.GET("/", func(c *gin.Context) {
		c.HTML(http.StatusOK, "main.tmpl", gin.H{
			"title": "psfe",
		})
	})

	// 注册GET
	r.GET("/signup", func(c *gin.Context) {
		c.HTML(http.StatusOK, "signup.tmpl", gin.H{
			"title": "Sign up",
		})
	})

	// 注册POST
	r.POST("/signup", func(c *gin.Context) {
		var form Signup
		if c.Bind(&form) == nil {
			var id string
			rows := db.QueryRow("select id from users where username = ?", form.User)

			err := rows.Scan(&id)

			// 表中无记录
			if err == sql.ErrNoRows {
				stmt, err := db.Prepare("insert into users(username, password, email)values(?,?,?)")
				if err != nil {
					log.Println(err)
				}

				rs, err := stmt.Exec(form.User, form.Password, form.Email)
				checkErr(err)

				// 获得影响行数
				_, err = rs.RowsAffected()

				if err == nil {
					c.JSON(http.StatusOK, gin.H{
						"errorNo": 0,
						"has": 0,
					})
				} else {
					checkErr(err)
				}
				stmt.Close()
			} else {
				checkErr(err)
				c.JSON(http.StatusOK, gin.H{
					"errorNo": 0,
					"has": 1,
				})
			}

			checkErr(err)
		}
	})

	// 登录GET
	r.GET("/login", func(c *gin.Context) {
		c.HTML(http.StatusOK, "login.tmpl", gin.H{
			"title": "Sign in",
		})
	})

	// 登录POST
	r.POST("/login", func(c *gin.Context) {
		var form Login
		if c.Bind(&form) == nil {
			var psw string
			rows := db.QueryRow("select password from users where username = ?", form.User)

			err := rows.Scan(&psw)
			if err == sql.ErrNoRows {
				c.JSON(http.StatusOK, gin.H{
					"errorNo": 0,
					"issigup": 0,
				})
				return
			}

			checkErr(err)

			if form.Password == psw {
				c.Redirect(http.StatusFound, "/")
			} else {
				c.JSON(http.StatusOK, gin.H{
					"errorNo": 0,
					"issigup": 1,
					"msg": "wrong password.",
				})
			}
		}
	})
}

func checkErr(err error) {
	if err != nil {
		log.Fatal(err)
	}

}
<commit_msg>update logup action<commit_after>package server

import (
	"gopkg.in/gin-gonic/gin.v1"
	"log"
	"net/http"
	"path/filepath"
	"database/sql"
)

type Login struct {
	User     string `form:"username" json:"username" binding:"required"`
	Password string `form:"password" json:"password" binding:"required"`
}

type Signup struct {
	User     string `form:"username" json:"username" binding:"required"`
	Password string `form:"password" json:"password" binding:"required"`
	Email string `form:"email" json:"email" binding:"required"`
}

func routers(r *gin.Engine) {

	r.LoadHTMLGlob(filepath.Join(staticPrefix, "views/*"))

	db := getDB()

	// 主页
	r.GET("/", func(c *gin.Context) {
		c.HTML(http.StatusOK, "main.tmpl", gin.H{
			"title": "psfe",
		})
	})

	// 注册GET
	r.GET("/signup", func(c *gin.Context) {
		c.HTML(http.StatusOK, "signup.tmpl", gin.H{
			"title": "Sign up",
		})
	})

	// 注册POST
	r.POST("/signup", func(c *gin.Context) {
		var form Signup
		if c.Bind(&form) == nil {
			var id string
			rows, err := db.Query("select id from users where username = ?", form.User)
			defer rows.Close()

			for rows.Next() {
				err := rows.Scan(&id)
				checkErr(err)
			}

			err = rows.Err()
			checkErr(err)

			// 表中无记录
			if id == "" {
				stmt, err := db.Prepare("insert into users(username, password, email)values(?,?,?)")
				checkErr(err)

				defer stmt.Close()

				_, err = stmt.Exec(form.User, form.Password, form.Email)
				if err != sql.ErrNoRows {
					c.JSON(http.StatusOK, gin.H{
						"errorNo": 0,
						"has": 0,
					})
				} else {
					checkErr(err)
				}
			} else {
				checkErr(err)
				c.JSON(http.StatusOK, gin.H{
					"errorNo": 0,
					"has": 1,
				})
			}

			checkErr(err)
		}
	})

	// 登录GET
	r.GET("/login", func(c *gin.Context) {
		c.HTML(http.StatusOK, "login.tmpl", gin.H{
			"title": "Sign in",
		})
	})

	// 登录POST
	r.POST("/login", func(c *gin.Context) {
		var form Login
		if c.Bind(&form) == nil {
			var psw string
			rows := db.QueryRow("select password from users where username = ?", form.User)

			err := rows.Scan(&psw)
			if err == sql.ErrNoRows {
				c.JSON(http.StatusOK, gin.H{
					"errorNo": 0,
					"issigup": 0,
				})
				return
			}

			checkErr(err)

			if form.Password == psw {
				c.Redirect(http.StatusFound, "/")
			} else {
				c.JSON(http.StatusOK, gin.H{
					"errorNo": 0,
					"issigup": 1,
					"msg": "wrong password.",
				})
			}
		}
	})
}

func checkErr(err error) {
	if err != nil {
		log.Fatal(err)
	}

}
<|endoftext|>


<commit_before>// pounce.go
package main

import (
	"bufio"
	"errors"
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/guelfey/go.dbus"
	"io"
	"net/http"
	"os"
	"time"
)

var (
	transactions map[string]Transaction = make(map[string]Transaction)
)

type Transaction struct {
	Url         string
	Destination string
}

func notify(msg string) {
	conn, err := dbus.SessionBus()
	if err != nil {
		fmt.Printf("Unable to connect to dbus session bus: %v\n", err)
	}

	obj := conn.Object("org.freedesktop.Notifications", "/org/freedesktop/Notifications")
	call := obj.Call("org.freedesktop.Notifications.Notify", 0, "", uint32(0),
		"", "pounce", msg, []string{}, map[string]dbus.Variant{}, int32(5000))
	if call.Err != nil {
		fmt.Printf("Error while trying to send notification: %v\n", call.Err)
	}
}

func download(url, output string, response chan<- *http.Response) {
	fmt.Printf("Retreiving from %v\n", url)
	resp, err := http.Get(url)

	if err != nil {
		fmt.Printf("Unable to get resource from '%v'. Error: %v\n", url, err)
	}
	transaction := Transaction{Url: resp.Request.URL.String(), Destination: output}
	transactions[resp.Request.URL.String()] = transaction
	response <- resp
}

func readFile(filename string) []string {
	var urls []string

	file, err := os.Open(filename)
	if err != nil {
		fmt.Printf("Problem opening input file!")
		panic(err)
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		urls = append(urls, scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		fmt.Printf("Problem reading input file!")
		panic(err)
	}

	return urls
}

func create(filename string) *os.File {
	file, err := os.Create(filename)
	if err != nil {
		fmt.Printf("Unable to create file!")
		panic(err)
	}

	return file
}

func save(file io.Writer, resp *http.Response) int64 {
	complete, err := io.Copy(file, resp.Body)
	if err != nil {
		fmt.Printf("Unable to write file, error: %v\n", err)
	}
	return complete
}

func main() {
	app := cli.NewApp()
	app.Name = "pounce"
	app.Version = "0.0.2"
	app.Usage = "A very simple file downloader in the vein of wget."
	app.Authors = []cli.Author{cli.Author{
		Name:  "Brian Tomlinson",
		Email: "darthlukan@gmail.com",
	}}
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "url, u",
			Value: "",
			Usage: "The input URL",
		},
		cli.StringFlag{
			Name:  "file, f",
			Value: "",
			Usage: "The input file",
		},
		cli.StringFlag{
			Name:  "outfile, o",
			Value: "",
			Usage: "The output file",
		},
		cli.StringFlag{
			Name:  "dir, d",
			Value: "",
			Usage: "The output directory",
		},
	}
	app.Action = func(c *cli.Context) {
		counter := 0
		url := c.String("url")
		inFile := c.String("file")
		outFile := c.String("outfile")
		outDir := c.String("dir")

		if url == "" && inFile == "" {
			fmt.Println(errors.New("Missing input arguments, please see 'pounce --help'\n"))
			return
		} else if outFile == "" && outDir == "" {
			fmt.Println(errors.New("Missing output arguments, please see 'pounce --help'\n"))
			return
		}

		startTime := time.Now()

		respChan := make(chan *http.Response)

		if inFile != "" && outDir != "" {
			urls := readFile(inFile)
			for _, url := range urls {
				go download(url, outDir, respChan)
				counter += 1
			}
		}

		if url != "" && outFile != "" {
			go download(url, outFile, respChan)
			counter += 1
		}

		for counter > 0 {
			select {
			case r := <-respChan:
				u := fmt.Sprintf("%v", r.Request.URL.String())
				if transaction, ok := transactions[u]; ok == true {
					defer r.Body.Close()
					f := create(transaction.Destination)
					defer f.Close()
					bytesWritten := save(f, r)
					endTime := time.Now()
					msg := fmt.Sprintf("Downloaded %vkb file '%v' in %v\n",
						bytesWritten/1024, f.Name(), endTime.Sub(startTime))
					go notify(msg)
					fmt.Printf(msg)
				} else {
					fmt.Printf("%v: %v.\n", "Problem processing transaction", r.Body)
				}
				counter--
			}
		}

	}
	app.Run(os.Args)
}
<commit_msg>this works for multi-download and single download<commit_after>// pounce.go
package main

import (
	"bufio"
	"errors"
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/guelfey/go.dbus"
	"io"
	"net/http"
	"os"
	"strings"
	"time"
)

var (
	transactions map[string]Transaction = make(map[string]Transaction)
)

type Transaction struct {
	Url         string
	Destination string
}

func notify(msg string) {
	conn, err := dbus.SessionBus()
	if err != nil {
		fmt.Printf("Unable to connect to dbus session bus: %v\n", err)
	}

	obj := conn.Object("org.freedesktop.Notifications", "/org/freedesktop/Notifications")
	call := obj.Call("org.freedesktop.Notifications.Notify", 0, "", uint32(0),
		"", "pounce", msg, []string{}, map[string]dbus.Variant{}, int32(5000))
	if call.Err != nil {
		fmt.Printf("Error while trying to send notification: %v\n", call.Err)
	}
}

func nameGenerator(url, destination string) string {
	var name string

	urlSplit := strings.Split(url, "/")
	if urlSplit[len(urlSplit)-1] != "" {
		name = urlSplit[len(urlSplit)-1]
	} else {
		name = urlSplit[len(urlSplit)-2]
	}
	return fmt.Sprintf("%v/%v", destination, name)
}

func download(url, output string, multi bool, response chan<- *http.Response) {
	fmt.Printf("Retreiving from %v\n", url)
	if multi {
		output = nameGenerator(url, output)
	}
	resp, err := http.Get(url)

	if err != nil {
		fmt.Printf("Unable to get resource from '%v'. Error: %v\n", url, err)
	}
	transaction := Transaction{Url: resp.Request.URL.String(), Destination: output}
	transactions[resp.Request.URL.String()] = transaction
	response <- resp
}

func readFile(filename string) []string {
	var urls []string

	file, err := os.Open(filename)
	if err != nil {
		fmt.Printf("Problem opening input file!")
		panic(err)
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		urls = append(urls, scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		fmt.Printf("Problem reading input file!")
		panic(err)
	}

	return urls
}

func create(filename string) *os.File {
	file, err := os.Create(filename)
	if err != nil {
		fmt.Printf("Unable to create file!")
		panic(err)
	}

	return file
}

func save(file io.Writer, resp *http.Response) int64 {
	complete, err := io.Copy(file, resp.Body)
	if err != nil {
		fmt.Printf("Unable to write file, error: %v\n", err)
	}
	return complete
}

func main() {
	app := cli.NewApp()
	app.Name = "pounce"
	app.Version = "0.0.2"
	app.Usage = "A very simple file downloader in the vein of wget."
	app.Authors = []cli.Author{cli.Author{
		Name:  "Brian Tomlinson",
		Email: "darthlukan@gmail.com",
	}}
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "url, u",
			Value: "",
			Usage: "The input URL",
		},
		cli.StringFlag{
			Name:  "file, f",
			Value: "",
			Usage: "The input file",
		},
		cli.StringFlag{
			Name:  "outfile, o",
			Value: "",
			Usage: "The output file",
		},
		cli.StringFlag{
			Name:  "dir, d",
			Value: "",
			Usage: "The output directory",
		},
	}
	app.Action = func(c *cli.Context) {
		counter := 0
		url := c.String("url")
		inFile := c.String("file")
		outFile := c.String("outfile")
		outDir := c.String("dir")

		if url == "" && inFile == "" {
			fmt.Println(errors.New("Missing input arguments, please see 'pounce --help'\n"))
			return
		} else if outFile == "" && outDir == "" {
			fmt.Println(errors.New("Missing output arguments, please see 'pounce --help'\n"))
			return
		}

		startTime := time.Now()

		respChan := make(chan *http.Response)

		if inFile != "" && outDir != "" {
			urls := readFile(inFile)
			for _, url := range urls {
				go download(url, outDir, true, respChan)
				counter += 1
			}
		}

		if url != "" && outFile != "" {
			go download(url, outFile, false, respChan)
			counter += 1
		}

		for counter > 0 {
			select {
			case r := <-respChan:
				u := fmt.Sprintf("%v", r.Request.URL.String())
				if transaction, ok := transactions[u]; ok == true {
					defer r.Body.Close()
					f := create(transaction.Destination)
					defer f.Close()
					bytesWritten := save(f, r)
					endTime := time.Now()
					msg := fmt.Sprintf("Downloaded %vkb file '%v' in %v\n",
						bytesWritten/1024, f.Name(), endTime.Sub(startTime))
					go notify(msg)
					fmt.Printf(msg)
				} else {
					fmt.Printf("%v: %v.\n", "Problem processing transaction", r.Body)
				}
				counter--
			}
		}

	}
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>// Copyright 2015,2016,2017,2018,2019 SeukWon Kang (kasworld@gmail.com)
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package primenum

import (
	"math"
	"runtime"
	"sort"
	"sync"
)

type PrimeIntList []int

func (pn PrimeIntList) FindPos(n int) (int, bool) {
	i := sort.SearchInts(pn, n)
	if i < len(pn) && pn[i] == n {
		// x is present at pn[i]
		return i, true
	} else {
		// x is not present in pn,
		// but i is the index where it would be inserted.
		return i, false
	}
}

// func (pn PrimeIntList) Sort() {
// 	if !sort.IntsAreSorted(pn) {
// 		sort.Ints(pn)
// 	}
// }

func (pn PrimeIntList) MaxCanCheck() int {
	last := pn[len(pn)-1]
	return last * last
}

func (pn PrimeIntList) CanFindIn(n int) bool {
	return pn.MaxCanCheck() > n
}

func (pn PrimeIntList) CalcPrime(n int) bool {
	to := int(math.Sqrt(float64(n)))
	for _, v := range pn {
		if n%v == 0 {
			return false
		}
		if v > to {
			break
		}
	}
	return true
}

func (pn *PrimeIntList) AppendFindTo(n int) {
	last := (*pn)[len(*pn)-1]
	if last >= n {
		return
	}
	if pn.MaxCanCheck() < n {
		pn.AppendFindTo(n / 2)
	}
	for i := last + 2; i < n; i += 2 {
		if pn.CalcPrime(i) {
			*pn = append(*pn, i)
		}
	}
}

func NewPrimeIntList(n int) PrimeIntList {
	pn := make(PrimeIntList, 0, n/10)
	pn = append(pn, []int{2, 3}...)
	pn.AppendFindTo(n)
	return pn
}

func (pn *PrimeIntList) MultiAppendFindTo(n int) {
	lastIndex := len(*pn) - 1
	last := (*pn)[lastIndex]
	if last >= n {
		return
	}
	if pn.MaxCanCheck() < n {
		pn.AppendFindTo(n / 2)
	}

	bufl := runtime.NumCPU() * 2

	var wgarg sync.WaitGroup
	var wgresult sync.WaitGroup

	// recv result
	appendCh := make(chan int, bufl)
	go func() {
		for n := range appendCh {
			*pn = append(*pn, n)
			wgresult.Done()
		}
	}()

	// prepare need check data
	argCh := make(chan int, (n-last-2)/2+1)
	for i := last + 2; i < n; i += 2 {
		argCh <- i
	}
	wgarg.Add(len(argCh))

	// run worker
	for i := 0; i < bufl; i++ {
		go func() {
			for n := range argCh {
				if pn.CalcPrime(n) {
					wgresult.Add(1)
					appendCh <- n
				}
				wgarg.Done()
			}
		}()
	}
	wgarg.Wait()
	wgresult.Wait()

	close(appendCh)
	close(argCh)
	sort.Ints((*pn)[lastIndex+1:])
}
<commit_msg>update<commit_after>// Copyright 2015,2016,2017,2018,2019 SeukWon Kang (kasworld@gmail.com)
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package primenum

import (
	"math"
	"runtime"
	"sort"
	"sync"
)

type PrimeIntList []int

func (pn PrimeIntList) FindPos(n int) (int, bool) {
	i := sort.SearchInts(pn, n)
	if i < len(pn) && pn[i] == n {
		// x is present at pn[i]
		return i, true
	} else {
		// x is not present in pn,
		// but i is the index where it would be inserted.
		return i, false
	}
}

// func (pn PrimeIntList) Sort() {
// 	if !sort.IntsAreSorted(pn) {
// 		sort.Ints(pn)
// 	}
// }

func (pn PrimeIntList) MaxCanCheck() int {
	last := pn[len(pn)-1]
	return last * last
}

func (pn PrimeIntList) CanFindIn(n int) bool {
	return pn.MaxCanCheck() > n
}

func (pn PrimeIntList) CalcPrime(n int) bool {
	to := int(math.Sqrt(float64(n)))
	for _, v := range pn {
		if n%v == 0 {
			return false
		}
		if v > to {
			break
		}
	}
	return true
}

func (pn *PrimeIntList) AppendFindTo(n int) {
	last := (*pn)[len(*pn)-1]
	if last >= n {
		return
	}
	if pn.MaxCanCheck() < n {
		pn.AppendFindTo(n / 2)
	}
	for i := last + 2; i < n; i += 2 {
		if pn.CalcPrime(i) {
			*pn = append(*pn, i)
		}
	}
}

func NewPrimeIntList(n int) PrimeIntList {
	pn := make(PrimeIntList, 0, n/10)
	pn = append(pn, []int{2, 3}...)
	pn.AppendFindTo(n)
	return pn
}

func (pn *PrimeIntList) MultiAppendFindTo(n int) {
	lastIndex := len(*pn) - 1
	last := (*pn)[lastIndex]
	if last >= n {
		return
	}
	if pn.MaxCanCheck() < n {
		pn.AppendFindTo(n / 2)
	}

	bufl := runtime.NumCPU() * 2

	var wgWorker sync.WaitGroup

	// recv result
	appendCh := make(chan int, bufl)
	go func() {
		for n := range appendCh {
			*pn = append(*pn, n)
		}
	}()

	// prepare need check data
	argCh := make(chan int, bufl)
	go func() {
		for i := last + 2; i < n; i += 2 {
			argCh <- i
		}
		close(argCh)
	}()

	// run worker
	for i := 0; i < bufl; i++ {
		wgWorker.Add(1)
		go func() {
			for n := range argCh {
				if pn.CalcPrime(n) {
					appendCh <- n
				}
			}
			wgWorker.Done()
		}()
	}
	wgWorker.Wait()
	close(appendCh)

	sort.Ints((*pn)[lastIndex+1:])
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 VMware, Inc. All Rights Reserved.
//
// This product is licensed to you under the Apache License, Version 2.0 (the "License").
// You may not use this product except in compliance with the License.
//
// This product may include a number of subcomponents with separate copyright notices and
// license terms. Your use of these subcomponents is subject to the terms and conditions
// of the subcomponent's license, as noted in the LICENSE file.

package rest

import (
	"crypto/rand"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

type Request struct {
	Method      string
	URL         string
	ContentType string
	Body        io.Reader
	Token       string
}

type Page struct {
	Items            []interface{} `json:"items"`
	NextPageLink     string        `json:"nextPageLink"`
	PreviousPageLink string        `json:"previousPageLink"`
}

type DocumentList struct {
	Items []interface{}
}

const appJson string = "application/json"

func AppendSlice(origSlice []interface{}, dataToAppend []interface{}) []interface{} {
	origLen := len(origSlice)
	newLen := origLen + len(dataToAppend)

	if newLen > cap(origSlice) {
		newSlice := make([]interface{}, (newLen+1)*2)
		copy(newSlice, origSlice)
		origSlice = newSlice
	}

	origSlice = origSlice[0:newLen]
	copy(origSlice[origLen:newLen], dataToAppend)

	return origSlice
}

func Get(client *http.Client, url string, token string) (res *http.Response, err error) {
	req := Request{"GET", url, "", nil, token}
	res, err = Do(client, &req)
	return
}

func GetList(client *http.Client, endpoint string, url string, token string) (result []byte, err error) {
	req := Request{"GET", url, "", nil, token}
	res, err := Do(client, &req)
	if err != nil {
		return
	}

	decoder := json.NewDecoder(res.Body)
	decoder.UseNumber()

	page := &Page{}
	err = decoder.Decode(page)
	if err != nil {
		return
	}

	documentList := &DocumentList{}
	documentList.Items = AppendSlice(documentList.Items, page.Items)

	for page.NextPageLink != "" {
		req = Request{"GET", endpoint + page.NextPageLink, "", nil, token}
		res, err = Do(client, &req)
		if err != nil {
			return
		}

		decoder = json.NewDecoder(res.Body)
		decoder.UseNumber()

		page.NextPageLink = ""
		page.PreviousPageLink = ""

		err = decoder.Decode(page)
		if err != nil {
			return
		}

		documentList.Items = AppendSlice(documentList.Items, page.Items)
	}

	result, err = json.Marshal(documentList)

	return
}

func Post(client *http.Client, url string, contentType string, body io.Reader, token string) (res *http.Response, err error) {
	if contentType == "" {
		contentType = appJson
	}

	req := Request{"POST", url, contentType, body, token}
	res, err = Do(client, &req)
	return
}

func Delete(client *http.Client, url string, token string) (res *http.Response, err error) {
	req := Request{"DELETE", url, "", nil, token}
	res, err = Do(client, &req)
	return
}

func Do(client *http.Client, req *Request) (res *http.Response, err error) {
	r, err := http.NewRequest(req.Method, req.URL, req.Body)
	if err != nil {
		return
	}
	if req.ContentType != "" {
		r.Header.Add("Content-Type", req.ContentType)
	}
	if req.Token != "" {
		r.Header.Add("Authorization", "Bearer "+req.Token)
	}
	res, err = client.Do(r)
	return
}

func MultipartUploadFile(client *http.Client, url, filePath string, params map[string]string, token string) (res *http.Response, err error) {
	file, err := os.Open(filePath)
	if err != nil {
		return
	}
	defer file.Close()
	return MultipartUpload(client, url, file, filepath.Base(filePath), params, token)
}

func MultipartUpload(client *http.Client, url string, reader io.Reader, filename string, params map[string]string, token string) (res *http.Response, err error) {
	// The mime/multipart package does not support streaming multipart data from disk,
	// at least not without complicated, problematic goroutines that simultaneously read/write into a buffer.
	// A much easier approach is to just construct the multipart request by hand, using io.MultiPart to
	// concatenate the parts of the request together into a single io.Reader.
	boundary := randomBoundary()
	parts := []io.Reader{}

	// Create a part for each key, val pair in params
	for k, v := range params {
		parts = append(parts, createFieldPart(k, v, boundary))
	}

	start := fmt.Sprintf("\r\n--%s\r\n", boundary)
	start += fmt.Sprintf("Content-Disposition: form-data; name=\"file\"; filename=\"%s\"\r\n", quoteEscaper.Replace(filename))
	start += fmt.Sprintf("Content-Type: application/octet-stream\r\n\r\n")
	end := fmt.Sprintf("\r\n--%s--", boundary)

	// The request will consist of a reader to begin the request, a reader which points
	// to the file data on disk, and a reader containing the closing boundary of the request.
	parts = append(parts, strings.NewReader(start), reader, strings.NewReader(end))

	contentType := fmt.Sprintf("multipart/form-data; boundary=%s", boundary)

	res, err = Do(client, &Request{"POST", url, contentType, io.MultiReader(parts...), token})

	return
}

// From https://golang.org/src/mime/multipart/writer.go
func randomBoundary() string {
	var buf [30]byte
	_, err := io.ReadFull(rand.Reader, buf[:])
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", buf[:])
}

// From https://golang.org/src/mime/multipart/writer.go
var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")

// Creates a reader that encapsulates a single multipart form part
func createFieldPart(fieldname, value, boundary string) io.Reader {
	str := fmt.Sprintf("\r\n--%s\r\n", boundary)
	str += fmt.Sprintf("Content-Disposition: form-data; name=\"%s\"\r\n\r\n", quoteEscaper.Replace(fieldname))
	str += value
	return strings.NewReader(str)
}
<commit_msg>Reture APIError in GetList response<commit_after>// Copyright (c) 2016 VMware, Inc. All Rights Reserved.
//
// This product is licensed to you under the Apache License, Version 2.0 (the "License").
// You may not use this product except in compliance with the License.
//
// This product may include a number of subcomponents with separate copyright notices and
// license terms. Your use of these subcomponents is subject to the terms and conditions
// of the subcomponent's license, as noted in the LICENSE file.

package rest

import (
	"crypto/rand"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

type Request struct {
	Method      string
	URL         string
	ContentType string
	Body        io.Reader
	Token       string
}

type Page struct {
	Items            []interface{} `json:"items"`
	NextPageLink     string        `json:"nextPageLink"`
	PreviousPageLink string        `json:"previousPageLink"`
}

type DocumentList struct {
	Items []interface{}
}

const appJson string = "application/json"

func AppendSlice(origSlice []interface{}, dataToAppend []interface{}) []interface{} {
	origLen := len(origSlice)
	newLen := origLen + len(dataToAppend)

	if newLen > cap(origSlice) {
		newSlice := make([]interface{}, (newLen+1)*2)
		copy(newSlice, origSlice)
		origSlice = newSlice
	}

	origSlice = origSlice[0:newLen]
	copy(origSlice[origLen:newLen], dataToAppend)

	return origSlice
}

func Get(client *http.Client, url string, token string) (res *http.Response, err error) {
	req := Request{"GET", url, "", nil, token}
	res, err = Do(client, &req)
	return
}

func GetList(client *http.Client, endpoint string, url string, token string) (result []byte, err error) {
	req := Request{"GET", url, "", nil, token}
	res, err := Do(client, &req)
	if err != nil {
		return
	}
	if res.StatusCode != 200 {
		errMsg := fmt.Sprintf("photon: HTTP %d: %v", res.StatusCode, res.Body)
		return nil, errors.New(errMsg)
	}

	decoder := json.NewDecoder(res.Body)
	decoder.UseNumber()

	page := &Page{}
	err = decoder.Decode(page)
	if err != nil {
		return
	}

	documentList := &DocumentList{}
	documentList.Items = AppendSlice(documentList.Items, page.Items)

	for page.NextPageLink != "" {
		req = Request{"GET", endpoint + page.NextPageLink, "", nil, token}
		res, err = Do(client, &req)
		if err != nil {
			return
		}

		decoder = json.NewDecoder(res.Body)
		decoder.UseNumber()

		page.NextPageLink = ""
		page.PreviousPageLink = ""

		err = decoder.Decode(page)
		if err != nil {
			return
		}

		documentList.Items = AppendSlice(documentList.Items, page.Items)
	}

	result, err = json.Marshal(documentList)

	return
}

func Post(client *http.Client, url string, contentType string, body io.Reader, token string) (res *http.Response, err error) {
	if contentType == "" {
		contentType = appJson
	}

	req := Request{"POST", url, contentType, body, token}
	res, err = Do(client, &req)
	return
}

func Delete(client *http.Client, url string, token string) (res *http.Response, err error) {
	req := Request{"DELETE", url, "", nil, token}
	res, err = Do(client, &req)
	return
}

func Do(client *http.Client, req *Request) (res *http.Response, err error) {
	r, err := http.NewRequest(req.Method, req.URL, req.Body)
	if err != nil {
		return
	}
	if req.ContentType != "" {
		r.Header.Add("Content-Type", req.ContentType)
	}
	if req.Token != "" {
		r.Header.Add("Authorization", "Bearer "+req.Token)
	}
	res, err = client.Do(r)
	return
}

func MultipartUploadFile(client *http.Client, url, filePath string, params map[string]string, token string) (res *http.Response, err error) {
	file, err := os.Open(filePath)
	if err != nil {
		return
	}
	defer file.Close()
	return MultipartUpload(client, url, file, filepath.Base(filePath), params, token)
}

func MultipartUpload(client *http.Client, url string, reader io.Reader, filename string, params map[string]string, token string) (res *http.Response, err error) {
	// The mime/multipart package does not support streaming multipart data from disk,
	// at least not without complicated, problematic goroutines that simultaneously read/write into a buffer.
	// A much easier approach is to just construct the multipart request by hand, using io.MultiPart to
	// concatenate the parts of the request together into a single io.Reader.
	boundary := randomBoundary()
	parts := []io.Reader{}

	// Create a part for each key, val pair in params
	for k, v := range params {
		parts = append(parts, createFieldPart(k, v, boundary))
	}

	start := fmt.Sprintf("\r\n--%s\r\n", boundary)
	start += fmt.Sprintf("Content-Disposition: form-data; name=\"file\"; filename=\"%s\"\r\n", quoteEscaper.Replace(filename))
	start += fmt.Sprintf("Content-Type: application/octet-stream\r\n\r\n")
	end := fmt.Sprintf("\r\n--%s--", boundary)

	// The request will consist of a reader to begin the request, a reader which points
	// to the file data on disk, and a reader containing the closing boundary of the request.
	parts = append(parts, strings.NewReader(start), reader, strings.NewReader(end))

	contentType := fmt.Sprintf("multipart/form-data; boundary=%s", boundary)

	res, err = Do(client, &Request{"POST", url, contentType, io.MultiReader(parts...), token})

	return
}

// From https://golang.org/src/mime/multipart/writer.go
func randomBoundary() string {
	var buf [30]byte
	_, err := io.ReadFull(rand.Reader, buf[:])
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", buf[:])
}

// From https://golang.org/src/mime/multipart/writer.go
var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")

// Creates a reader that encapsulates a single multipart form part
func createFieldPart(fieldname, value, boundary string) io.Reader {
	str := fmt.Sprintf("\r\n--%s\r\n", boundary)
	str += fmt.Sprintf("Content-Disposition: form-data; name=\"%s\"\r\n\r\n", quoteEscaper.Replace(fieldname))
	str += value
	return strings.NewReader(str)
}
<|endoftext|>


<commit_before>package discovery

import (
	"net"
	"strconv"
	"strings"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
)

var grpcDialContext = grpc.DialContext

/*
NewGrpcClient attempts to establish grpc client connections with the specified
options to the endpoints connected with the etcd prefix. If multiple
destinations are linked to the destination (e.g. through a service
subdirectory), a connection is attempted to each backend in random order.
In case of connection errors the target is skipped and the next one is
attempted.

Once established, aborted connections will not be reestablished to a different
endpoint.
*/
func NewGrpcClient(ctx context.Context, client etcd.KV, path string,
	opts ...grpc.DialOption) (*grpc.ClientConn, error) {
	var configs []*ExportedServiceRecord
	var configPerms []int
	var getResponse *etcd.GetResponse
	var realPath, slashPath string
	var conn *grpc.ClientConn
	var err error

	if path[0] == '/' {
		realPath = path
	} else {
		realPath = "/ns/service/" + realPath
	}

	getResponse, err = client.Get(ctx, realPath, etcd.WithPrefix())
	if err != nil {
		return nil, err
	}

	if realPath[len(realPath)-1] == '/' {
		slashPath = realPath
	} else {
		slashPath = realPath + "/"
	}

	for _, kv := range getResponse.Kvs {
		if string(kv.Key) == realPath || strings.HasPrefix(string(kv.Key), slashPath) {
			var config = new(ExportedServiceRecord)
			err = proto.Unmarshal(kv.Value, config)
			if err == nil {
				configs = append(configs, config)
			}
		}
	}

	if len(configs) == 0 {
		if err == nil {
			err = grpc.Errorf(codes.Unavailable, "No services were found at %s",
				realPath)
		}

		return nil, err
	}

	// Try to connect to different ports at random.
	configPerms = make([]int, len(configs))
	for _, i := range configPerms {
		var config = configs[i]
		var dest = net.JoinHostPort(
			config.Address, strconv.Itoa(int(config.Port)))

		conn, err = grpcDialContext(ctx, dest, opts...)
		if err == nil {
			return conn, err
		}
	}

	return nil, err
}
<commit_msg>Fix the same bugs found by unit tests as we did for simple_client.go.<commit_after>package discovery

import (
	"math/rand"
	"net"
	"strconv"
	"strings"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
)

var grpcDialContext = grpc.DialContext

/*
NewGrpcClient attempts to establish grpc client connections with the specified
options to the endpoints connected with the etcd prefix. If multiple
destinations are linked to the destination (e.g. through a service
subdirectory), a connection is attempted to each backend in random order.
In case of connection errors the target is skipped and the next one is
attempted.

Once established, aborted connections will not be reestablished to a different
endpoint.
*/
func NewGrpcClient(ctx context.Context, client etcd.KV, path string,
	opts ...grpc.DialOption) (*grpc.ClientConn, error) {
	var configs []*ExportedServiceRecord
	var configPerms []int
	var getResponse *etcd.GetResponse
	var realPath, slashPath string
	var conn *grpc.ClientConn
	var err error

	if path[0] == '/' {
		realPath = path
	} else {
		realPath = "/ns/service/" + path
	}

	getResponse, err = client.Get(ctx, realPath, etcd.WithPrefix())
	if err != nil {
		return nil, err
	}

	if realPath[len(realPath)-1] == '/' {
		slashPath = realPath
	} else {
		slashPath = realPath + "/"
	}

	for _, kv := range getResponse.Kvs {
		if string(kv.Key) == realPath || strings.HasPrefix(string(kv.Key), slashPath) {
			var config = new(ExportedServiceRecord)
			err = proto.Unmarshal(kv.Value, config)
			if err == nil {
				configs = append(configs, config)
			}
		}
	}

	if len(configs) == 0 {
		if err == nil {
			err = grpc.Errorf(codes.Unavailable, "No services were found at %s",
				realPath)
		}

		return nil, err
	}

	// Try to connect to different ports at random.
	configPerms = rand.Perm(len(configs))
	for _, i := range configPerms {
		var config = configs[i]
		var dest = net.JoinHostPort(
			config.Address, strconv.Itoa(int(config.Port)))

		conn, err = grpcDialContext(ctx, dest, opts...)
		if err == nil {
			return conn, err
		}
	}

	return nil, err
}
<|endoftext|>


<commit_before>package main

import (
	// import plugins to ensure they're bound into the executable
	_ "github.com/30x/apidApigeeSync"
	_ "github.com/30x/apidVerifyAPIKey"
	_ "github.com/30x/apidGatewayDeploy"

	// other imports
	"github.com/30x/apid"
	"github.com/30x/apid/factory"
	"flag"
	"os"
)

func main() {
	configFlag := flag.String("config", "", "path to the yaml config file [./apid_config.yaml]")

	flag.Parse()

	configFile := *configFlag
	if configFile != "" {
		os.Setenv("APID_CONFIG_FILE", configFile)
	}

	apid.Initialize(factory.DefaultServicesFactory())

	log := apid.Log()
	log.Debug("initializing...")

	apid.InitializePlugins()

	// start client API listener
	log.Debug("listening...")

	api := apid.API()
	err := api.Listen()
	if err != nil {
		log.Print(err)
	}
}
<commit_msg>Add a -clean flag to the cmd<commit_after>package main

import (
	// import plugins to ensure they're bound into the executable
	_ "github.com/30x/apidApigeeSync"
	//_ "github.com/30x/apidVerifyAPIKey"
	_ "github.com/30x/apidGatewayDeploy"

	// other imports
	"github.com/30x/apid"
	"github.com/30x/apid/factory"
	"flag"
	"os"
)

func main() {
	configFlag := flag.String("config", "", "path to the yaml config file [./apid_config.yaml]")
	cleanFlag := flag.Bool("clean", false, "start clean, deletes all existing data from local_storage_path")

	configFile := *configFlag
	if configFile != "" {
		os.Setenv("APID_CONFIG_FILE", configFile)
	}

	flag.Parse()

	apid.Initialize(factory.DefaultServicesFactory())

	log := apid.Log()
	config := apid.Config()

	if *cleanFlag {
		localStorage := config.GetString("local_storage_path")
		log.Infof("removing existing data from: %s", localStorage)
		err := os.RemoveAll(localStorage)
		if err != nil {
			log.Panic("Failed to clean data directory: %v", err)
		}
	}

	log.Debug("initializing...")

	apid.InitializePlugins()

	// start client API listener
	log.Debug("listening...")

	api := apid.API()
	err := api.Listen()
	if err != nil {
		log.Print(err)
	}
}
<|endoftext|>


<commit_before>package netudp

import (
	"V-switch/conf"
	"V-switch/plane"
	"V-switch/tools"
	"log"
	"net"
	"os"
	"strconv"
)

func init() {

	myport := conf.GetConfigItem("PORT")

	if len(myport) < 2 {
		conf.SetConfigItem("PORT", "22000")
	}

	plane.VSwitch.Server = UdpCreateServer(conf.GetConfigItem("PORT"))
	go UDPReadMessage(plane.VSwitch.Server)

}

/* A Simple function to verify error */
func CheckError(err error) {
	if err != nil {
		log.Println("[UDP] problem: ", err.Error())

	}
}

func UdpCreateServer(port string) *net.UDPConn {

	log.Println("[UDP][SERVER] Starting UDP listener")

	LocalAddr := tools.GetLocalIp() + ":" + port

	ServerAddr, err := net.ResolveUDPAddr("udp", LocalAddr)
	CheckError(err)

	/* Now listen at selected port */
	ServerConn, err := net.ListenUDP("udp", ServerAddr)

	// Setting the read buffer to avoid congestion

	ServerConn.SetReadBuffer(4194304)

	if err != nil {
		log.Println("[UDP][SERVER] Error listening at port ", port, ":", err.Error())
		os.Exit(1)
	}

	log.Println("[UDP][SERVER] Now listening at port ", port)
	return ServerConn

}

func UDPReadMessage(ServerConn *net.UDPConn) {

	log.Println("[UDP][SERVER] Reading thread started")

	tmp_MTU, _ := strconv.Atoi(conf.GetConfigItem("MTU")) // encryption of MTU max size

	defer ServerConn.Close()

	buf := make([]byte, 2*tmp_MTU) // enough for the payload , even if encrypted ang gob encoded
	log.Println("[UDP][SERVER] Read MTU set to ", tmp_MTU)

	for {

		n, addr, err := ServerConn.ReadFromUDP(buf)
		log.Println("[UDP][SERVER] Received ", n, "bytes from ", addr.String()) // just for debug

		if err != nil {
			log.Println("[UDP][SERVER] Error while reading: ", err.Error())
		} else {

			plane.UdpToPlane <- buf[:n]

		}

	}

}

func UDPEngineStart() {

	log.Println("[UDP] Engine init ")

}
<commit_msg>UDP MTU buffer set<commit_after>package netudp

import (
	"V-switch/conf"
	"V-switch/plane"
	"V-switch/tools"
	"log"
	"net"
	"os"
	"strconv"
)

func init() {

	myport := conf.GetConfigItem("PORT")

	if len(myport) < 2 {
		conf.SetConfigItem("PORT", "22000")
	}

	plane.VSwitch.Server = UdpCreateServer(conf.GetConfigItem("PORT"))
	go UDPReadMessage(plane.VSwitch.Server)

}

/* A Simple function to verify error */
func CheckError(err error) {
	if err != nil {
		log.Println("[UDP] problem: ", err.Error())

	}
}

func UdpCreateServer(port string) *net.UDPConn {

	log.Println("[UDP][SERVER] Starting UDP listener")

	LocalAddr := tools.GetLocalIp() + ":" + port

	ServerAddr, err := net.ResolveUDPAddr("udp", LocalAddr)
	CheckError(err)

	/* Now listen at selected port */
	ServerConn, err := net.ListenUDP("udp", ServerAddr)

	// Setting the read buffer to avoid congestion

	ServerConn.SetReadBuffer(4194304)

	if err != nil {
		log.Println("[UDP][SERVER] Error listening at port ", port, ":", err.Error())
		os.Exit(1)
	}

	log.Println("[UDP][SERVER] Now listening at port ", port)
	return ServerConn

}

func UDPReadMessage(ServerConn *net.UDPConn) {

	log.Println("[UDP][SERVER] Reading thread started")

	readbuffer, _ := strconv.Atoi(conf.GetConfigItem("MTU")) // at least the  MTU max size

	defer ServerConn.Close()

	buf := make([]byte, 3*readbuffer) // enough for the payload , even if encrypted ang gob encoded
	log.Println("[UDP][SERVER] Read MTU set to ", tmp_MTU)

	for {

		n, addr, err := ServerConn.ReadFromUDP(buf)
		log.Println("[UDP][SERVER] Received ", n, "bytes from ", addr.String()) // just for debug

		if err != nil {
			log.Println("[UDP][SERVER] Error while reading: ", err.Error())
		} else {

			plane.UdpToPlane <- buf[:n]

		}

	}

}

func UDPEngineStart() {

	log.Println("[UDP] Engine init ")

}
<|endoftext|>


<commit_before>package vm

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"math/big"
	"os"
	"testing"

	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/state"
	"github.com/ethereum/go-ethereum/trie"
	"github.com/obscuren/mutan"
)

type TestEnv struct{}

func (TestEnv) Origin() []byte        { return nil }
func (TestEnv) BlockNumber() *big.Int { return nil }
func (TestEnv) BlockHash() []byte     { return nil }
func (TestEnv) PrevHash() []byte      { return nil }
func (TestEnv) Coinbase() []byte      { return nil }
func (TestEnv) Time() int64           { return 0 }
func (TestEnv) GasLimit() *big.Int    { return nil }
func (TestEnv) Difficulty() *big.Int  { return nil }
func (TestEnv) Value() *big.Int       { return nil }
func (TestEnv) AddLog(state.Log)      {}

func (TestEnv) Transfer(from, to Account, amount *big.Int) error {
	return nil
}

// This is likely to fail if anything ever gets looked up in the state trie :-)
func (TestEnv) State() *state.State {
	return state.New(trie.New(nil, ""))
}

const mutcode = `
var x = 0;
for i := 0; i < 10; i++ {
	x = i
}

return x`

func setup(level logger.LogLevel, typ Type) (*Closure, VirtualMachine) {
	code, err := ethutil.Compile(mutcode, true)
	if err != nil {
		log.Fatal(err)
	}

	// Pipe output to /dev/null
	logger.AddLogSystem(logger.NewStdLogSystem(ioutil.Discard, log.LstdFlags, level))

	ethutil.ReadConfig(".ethtest", "/tmp/ethtest", "")

	stateObject := state.NewStateObject([]byte{'j', 'e', 'f', 'f'})
	callerClosure := NewClosure(nil, stateObject, stateObject, code, big.NewInt(1000000), big.NewInt(0))

	return callerClosure, New(TestEnv{}, typ)
}

var big9 = ethutil.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000009")

func TestDebugVm(t *testing.T) {
	if mutan.Version < "0.6" {
		t.Skip("skipping for mutan version", mutan.Version, " < 0.6")
	}

	closure, vm := setup(logger.DebugLevel, DebugVmTy)
	ret, _, e := closure.Call(vm, nil)
	if e != nil {
		t.Fatalf("Call returned error: %v", e)
	}
	if !bytes.Equal(ret, big9) {
		t.Errorf("Wrong return value '%x', want '%x'", ret, big9)
	}
}

func TestVm(t *testing.T) {
	if mutan.Version < "0.6" {
		t.Skip("skipping for mutan version", mutan.Version, " < 0.6")
	}

	closure, vm := setup(logger.DebugLevel, StandardVmTy)
	ret, _, e := closure.Call(vm, nil)
	if e != nil {
		t.Fatalf("Call returned error: %v", e)
	}
	if !bytes.Equal(ret, big9) {
		t.Errorf("Wrong return value '%x', want '%x'", ret, big9)
	}
}

func BenchmarkDebugVm(b *testing.B) {
	closure, vm := setup(logger.InfoLevel, DebugVmTy)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		closure.Call(vm, nil)
	}
}

func BenchmarkVm(b *testing.B) {
	closure, vm := setup(logger.InfoLevel, StandardVmTy)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		closure.Call(vm, nil)
	}
}

func RunCode(mutCode string, typ Type) []byte {
	code, err := ethutil.Compile(mutCode, true)
	if err != nil {
		log.Fatal(err)
	}

	logger.AddLogSystem(logger.NewStdLogSystem(os.Stdout, log.LstdFlags, logger.InfoLevel))

	ethutil.ReadConfig(".ethtest", "/tmp/ethtest", "")

	stateObject := state.NewStateObject([]byte{'j', 'e', 'f', 'f'})
	closure := NewClosure(nil, stateObject, stateObject, code, big.NewInt(1000000), big.NewInt(0))

	vm := New(TestEnv{}, typ)
	ret, _, e := closure.Call(vm, nil)
	if e != nil {
		fmt.Println(e)
	}

	return ret
}

func TestBuildInSha256(t *testing.T) {
	ret := RunCode(`
	var in = 42
	var out = 0

	call(0x2, 0, 10000, in, out)

	return out
	`, DebugVmTy)

	exp := crypto.Sha256(ethutil.LeftPadBytes([]byte{42}, 32))
	if bytes.Compare(ret, exp) != 0 {
		t.Errorf("Expected %x, got %x", exp, ret)
	}
}

func TestBuildInRipemd(t *testing.T) {
	ret := RunCode(`
	var in = 42
	var out = 0

	call(0x3, 0, 10000, in, out)

	return out
	`, DebugVmTy)

	exp := ethutil.RightPadBytes(crypto.Ripemd160(ethutil.LeftPadBytes([]byte{42}, 32)), 32)
	if bytes.Compare(ret, exp) != 0 {
		t.Errorf("Expected %x, got %x", exp, ret)
	}
}
<commit_msg>Remove references to mutan<commit_after>package vm

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"math/big"
	"os"
	"testing"

	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/state"
	"github.com/ethereum/go-ethereum/trie"
	// "github.com/obscuren/mutan"
)

type TestEnv struct{}

func (TestEnv) Origin() []byte        { return nil }
func (TestEnv) BlockNumber() *big.Int { return nil }
func (TestEnv) BlockHash() []byte     { return nil }
func (TestEnv) PrevHash() []byte      { return nil }
func (TestEnv) Coinbase() []byte      { return nil }
func (TestEnv) Time() int64           { return 0 }
func (TestEnv) GasLimit() *big.Int    { return nil }
func (TestEnv) Difficulty() *big.Int  { return nil }
func (TestEnv) Value() *big.Int       { return nil }
func (TestEnv) AddLog(state.Log)      {}

func (TestEnv) Transfer(from, to Account, amount *big.Int) error {
	return nil
}

// This is likely to fail if anything ever gets looked up in the state trie :-)
func (TestEnv) State() *state.State {
	return state.New(trie.New(nil, ""))
}

const mutcode = `
var x = 0;
for i := 0; i < 10; i++ {
	x = i
}

return x`

func setup(level logger.LogLevel, typ Type) (*Closure, VirtualMachine) {
	code, err := ethutil.Compile(mutcode, true)
	if err != nil {
		log.Fatal(err)
	}

	// Pipe output to /dev/null
	logger.AddLogSystem(logger.NewStdLogSystem(ioutil.Discard, log.LstdFlags, level))

	ethutil.ReadConfig(".ethtest", "/tmp/ethtest", "")

	stateObject := state.NewStateObject([]byte{'j', 'e', 'f', 'f'})
	callerClosure := NewClosure(nil, stateObject, stateObject, code, big.NewInt(1000000), big.NewInt(0))

	return callerClosure, New(TestEnv{}, typ)
}

var big9 = ethutil.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000009")

func TestDebugVm(t *testing.T) {
	// if mutan.Version < "0.6" {
	// 	t.Skip("skipping for mutan version", mutan.Version, " < 0.6")
	// }

	closure, vm := setup(logger.DebugLevel, DebugVmTy)
	ret, _, e := closure.Call(vm, nil)
	if e != nil {
		t.Fatalf("Call returned error: %v", e)
	}
	if !bytes.Equal(ret, big9) {
		t.Errorf("Wrong return value '%x', want '%x'", ret, big9)
	}
}

func TestVm(t *testing.T) {
	// if mutan.Version < "0.6" {
	// 	t.Skip("skipping for mutan version", mutan.Version, " < 0.6")
	// }

	closure, vm := setup(logger.DebugLevel, StandardVmTy)
	ret, _, e := closure.Call(vm, nil)
	if e != nil {
		t.Fatalf("Call returned error: %v", e)
	}
	if !bytes.Equal(ret, big9) {
		t.Errorf("Wrong return value '%x', want '%x'", ret, big9)
	}
}

func BenchmarkDebugVm(b *testing.B) {
	closure, vm := setup(logger.InfoLevel, DebugVmTy)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		closure.Call(vm, nil)
	}
}

func BenchmarkVm(b *testing.B) {
	closure, vm := setup(logger.InfoLevel, StandardVmTy)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		closure.Call(vm, nil)
	}
}

func RunCode(mutCode string, typ Type) []byte {
	code, err := ethutil.Compile(mutCode, true)
	if err != nil {
		log.Fatal(err)
	}

	logger.AddLogSystem(logger.NewStdLogSystem(os.Stdout, log.LstdFlags, logger.InfoLevel))

	ethutil.ReadConfig(".ethtest", "/tmp/ethtest", "")

	stateObject := state.NewStateObject([]byte{'j', 'e', 'f', 'f'})
	closure := NewClosure(nil, stateObject, stateObject, code, big.NewInt(1000000), big.NewInt(0))

	vm := New(TestEnv{}, typ)
	ret, _, e := closure.Call(vm, nil)
	if e != nil {
		fmt.Println(e)
	}

	return ret
}

func TestBuildInSha256(t *testing.T) {
	ret := RunCode(`
	var in = 42
	var out = 0

	call(0x2, 0, 10000, in, out)

	return out
	`, DebugVmTy)

	exp := crypto.Sha256(ethutil.LeftPadBytes([]byte{42}, 32))
	if bytes.Compare(ret, exp) != 0 {
		t.Errorf("Expected %x, got %x", exp, ret)
	}
}

func TestBuildInRipemd(t *testing.T) {
	ret := RunCode(`
	var in = 42
	var out = 0

	call(0x3, 0, 10000, in, out)

	return out
	`, DebugVmTy)

	exp := ethutil.RightPadBytes(crypto.Ripemd160(ethutil.LeftPadBytes([]byte{42}, 32)), 32)
	if bytes.Compare(ret, exp) != 0 {
		t.Errorf("Expected %x, got %x", exp, ret)
	}
}
<|endoftext|>


<commit_before>package s3api

import (
	"fmt"
	"net/http"
	"os"
	"strings"
	"sync"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
)

type Action string

type Iam interface {
	Check(f http.HandlerFunc, actions ...Action) http.HandlerFunc
}

type IdentityAccessManagement struct {
	m sync.RWMutex

	identities []*Identity
	domain     string
}

type Identity struct {
	Name        string
	Credentials []*Credential
	Actions     []Action
}

type Credential struct {
	AccessKey string
	SecretKey string
}

func (action Action) isAdmin() bool {
	return strings.HasPrefix(string(action), s3_constants.ACTION_ADMIN)
}

func (action Action) isOwner(bucket string) bool {
	return string(action) == s3_constants.ACTION_ADMIN+":"+bucket
}

func (action Action) overBucket(bucket string) bool {
	return strings.HasSuffix(string(action), ":"+bucket) || strings.HasSuffix(string(action), ":*")
}

func (action Action) getPermission() Permission {
	switch act := strings.Split(string(action), ":")[0]; act {
	case s3_constants.ACTION_ADMIN:
		return Permission("FULL_CONTROL")
	case s3_constants.ACTION_WRITE:
		return Permission("WRITE")
	case s3_constants.ACTION_READ:
		return Permission("READ")
	default:
		return Permission("")
	}
}

func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManagement {
	iam := &IdentityAccessManagement{
		domain: option.DomainName,
	}
	if option.Config != "" {
		if err := iam.loadS3ApiConfigurationFromFile(option.Config); err != nil {
			glog.Fatalf("fail to load config file %s: %v", option.Config, err)
		}
	} else {
		if err := iam.loadS3ApiConfigurationFromFiler(option); err != nil {
			glog.Warningf("fail to load config: %v", err)
		}
	}
	return iam
}

func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFiler(option *S3ApiServerOption) (err error) {
	var content []byte
	err = pb.WithFilerClient(option.Filer, option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		content, err = filer.ReadInsideFiler(client, filer.IamConfigDirecotry, filer.IamIdentityFile)
		return err
	})
	if err != nil {
		return fmt.Errorf("read S3 config: %v", err)
	}
	return iam.loadS3ApiConfigurationFromBytes(content)
}

func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFile(fileName string) error {
	content, readErr := os.ReadFile(fileName)
	if readErr != nil {
		glog.Warningf("fail to read %s : %v", fileName, readErr)
		return fmt.Errorf("fail to read %s : %v", fileName, readErr)
	}
	return iam.loadS3ApiConfigurationFromBytes(content)
}

func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromBytes(content []byte) error {
	s3ApiConfiguration := &iam_pb.S3ApiConfiguration{}
	if err := filer.ParseS3ConfigurationFromBytes(content, s3ApiConfiguration); err != nil {
		glog.Warningf("unmarshal error: %v", err)
		return fmt.Errorf("unmarshal error: %v", err)
	}
	if err := iam.loadS3ApiConfiguration(s3ApiConfiguration); err != nil {
		return err
	}
	return nil
}

func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3ApiConfiguration) error {
	var identities []*Identity
	for _, ident := range config.Identities {
		t := &Identity{
			Name:        ident.Name,
			Credentials: nil,
			Actions:     nil,
		}
		for _, action := range ident.Actions {
			t.Actions = append(t.Actions, Action(action))
		}
		for _, cred := range ident.Credentials {
			t.Credentials = append(t.Credentials, &Credential{
				AccessKey: cred.AccessKey,
				SecretKey: cred.SecretKey,
			})
		}
		identities = append(identities, t)
	}
	iam.m.Lock()
	// atomically switch
	iam.identities = identities
	iam.m.Unlock()
	return nil
}

func (iam *IdentityAccessManagement) isEnabled() bool {
	iam.m.RLock()
	defer iam.m.RUnlock()
	return len(iam.identities) > 0
}

func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identity *Identity, cred *Credential, found bool) {

	iam.m.RLock()
	defer iam.m.RUnlock()
	for _, ident := range iam.identities {
		for _, cred := range ident.Credentials {
			// println("checking", ident.Name, cred.AccessKey)
			if cred.AccessKey == accessKey {
				return ident, cred, true
			}
		}
	}
	glog.V(1).Infof("could not find accessKey %s", accessKey)
	return nil, nil, false
}

func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, found bool) {
	iam.m.RLock()
	defer iam.m.RUnlock()
	for _, ident := range iam.identities {
		if ident.Name == "anonymous" {
			return ident, true
		}
	}
	return nil, false
}

func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) http.HandlerFunc {

	if !iam.isEnabled() {
		return f
	}

	return func(w http.ResponseWriter, r *http.Request) {
		identity, errCode := iam.authRequest(r, action)
		if errCode == s3err.ErrNone {
			if identity != nil && identity.Name != "" {
				r.Header.Set(xhttp.AmzIdentityId, identity.Name)
				if identity.isAdmin() {
					r.Header.Set(xhttp.AmzIsAdmin, "true")
				}
			}
			f(w, r)
			return
		}
		s3err.WriteErrorResponse(w, r, errCode)
	}
}

// check whether the request has valid access keys
func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) (*Identity, s3err.ErrorCode) {
	var identity *Identity
	var s3Err s3err.ErrorCode
	var found bool
	switch getRequestAuthType(r) {
	case authTypeStreamingSigned:
		return identity, s3err.ErrNone
	case authTypeUnknown:
		glog.V(3).Infof("unknown auth type")
		return identity, s3err.ErrAccessDenied
	case authTypePresignedV2, authTypeSignedV2:
		glog.V(3).Infof("v2 auth type")
		identity, s3Err = iam.isReqAuthenticatedV2(r)
	case authTypeSigned, authTypePresigned:
		glog.V(3).Infof("v4 auth type")
		identity, s3Err = iam.reqSignatureV4Verify(r)
	case authTypePostPolicy:
		glog.V(3).Infof("post policy auth type")
		return identity, s3err.ErrNone
	case authTypeJWT:
		glog.V(3).Infof("jwt auth type")
		return identity, s3err.ErrNotImplemented
	case authTypeAnonymous:
		identity, found = iam.lookupAnonymous()
		if !found {
			return identity, s3err.ErrAccessDenied
		}
	default:
		return identity, s3err.ErrNotImplemented
	}

	if s3Err != s3err.ErrNone {
		return identity, s3Err
	}

	glog.V(3).Infof("user name: %v actions: %v, action: %v", identity.Name, identity.Actions, action)

	bucket, _ := getBucketAndObject(r)

	if !identity.canDo(action, bucket) {
		return identity, s3err.ErrAccessDenied
	}

	return identity, s3err.ErrNone

}

func (iam *IdentityAccessManagement) authUser(r *http.Request) (*Identity, s3err.ErrorCode) {
	var identity *Identity
	var s3Err s3err.ErrorCode
	var found bool
	switch getRequestAuthType(r) {
	case authTypeStreamingSigned:
		return identity, s3err.ErrNone
	case authTypeUnknown:
		glog.V(3).Infof("unknown auth type")
		return identity, s3err.ErrAccessDenied
	case authTypePresignedV2, authTypeSignedV2:
		glog.V(3).Infof("v2 auth type")
		identity, s3Err = iam.isReqAuthenticatedV2(r)
	case authTypeSigned, authTypePresigned:
		glog.V(3).Infof("v4 auth type")
		identity, s3Err = iam.reqSignatureV4Verify(r)
	case authTypePostPolicy:
		glog.V(3).Infof("post policy auth type")
		return identity, s3err.ErrNone
	case authTypeJWT:
		glog.V(3).Infof("jwt auth type")
		return identity, s3err.ErrNotImplemented
	case authTypeAnonymous:
		identity, found = iam.lookupAnonymous()
		if !found {
			return identity, s3err.ErrAccessDenied
		}
	default:
		return identity, s3err.ErrNotImplemented
	}

	glog.V(3).Infof("auth error: %v", s3Err)
	if s3Err != s3err.ErrNone {
		return identity, s3Err
	}
	return identity, s3err.ErrNone
}

func (identity *Identity) canDo(action Action, bucket string) bool {
	if identity.isAdmin() {
		return true
	}
	for _, a := range identity.Actions {
		if a == action {
			return true
		}
	}
	if bucket == "" {
		return false
	}
	limitedByBucket := string(action) + ":" + bucket
	adminLimitedByBucket := s3_constants.ACTION_ADMIN + ":" + bucket
	for _, a := range identity.Actions {
		act := string(a)
		if strings.HasSuffix(act, "*") {
			if strings.HasPrefix(limitedByBucket, act[:len(act)-1]) {
				return true
			}
			if strings.HasPrefix(adminLimitedByBucket, act[:len(act)-1]) {
				return true
			}
		} else {
			if act == limitedByBucket {
				return true
			}
			if act == adminLimitedByBucket {
				return true
			}
		}
	}
	return false
}

func (identity *Identity) isAdmin() bool {
	for _, a := range identity.Actions {
		if a == "Admin" {
			return true
		}
	}
	return false
}
<commit_msg>force overwrite s3-is-admin https://github.com/chrislusf/seaweedfs/issues/2433<commit_after>package s3api

import (
	"fmt"
	"net/http"
	"os"
	"strings"
	"sync"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
)

type Action string

type Iam interface {
	Check(f http.HandlerFunc, actions ...Action) http.HandlerFunc
}

type IdentityAccessManagement struct {
	m sync.RWMutex

	identities []*Identity
	domain     string
}

type Identity struct {
	Name        string
	Credentials []*Credential
	Actions     []Action
}

type Credential struct {
	AccessKey string
	SecretKey string
}

func (action Action) isAdmin() bool {
	return strings.HasPrefix(string(action), s3_constants.ACTION_ADMIN)
}

func (action Action) isOwner(bucket string) bool {
	return string(action) == s3_constants.ACTION_ADMIN+":"+bucket
}

func (action Action) overBucket(bucket string) bool {
	return strings.HasSuffix(string(action), ":"+bucket) || strings.HasSuffix(string(action), ":*")
}

func (action Action) getPermission() Permission {
	switch act := strings.Split(string(action), ":")[0]; act {
	case s3_constants.ACTION_ADMIN:
		return Permission("FULL_CONTROL")
	case s3_constants.ACTION_WRITE:
		return Permission("WRITE")
	case s3_constants.ACTION_READ:
		return Permission("READ")
	default:
		return Permission("")
	}
}

func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManagement {
	iam := &IdentityAccessManagement{
		domain: option.DomainName,
	}
	if option.Config != "" {
		if err := iam.loadS3ApiConfigurationFromFile(option.Config); err != nil {
			glog.Fatalf("fail to load config file %s: %v", option.Config, err)
		}
	} else {
		if err := iam.loadS3ApiConfigurationFromFiler(option); err != nil {
			glog.Warningf("fail to load config: %v", err)
		}
	}
	return iam
}

func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFiler(option *S3ApiServerOption) (err error) {
	var content []byte
	err = pb.WithFilerClient(option.Filer, option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		content, err = filer.ReadInsideFiler(client, filer.IamConfigDirecotry, filer.IamIdentityFile)
		return err
	})
	if err != nil {
		return fmt.Errorf("read S3 config: %v", err)
	}
	return iam.loadS3ApiConfigurationFromBytes(content)
}

func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFile(fileName string) error {
	content, readErr := os.ReadFile(fileName)
	if readErr != nil {
		glog.Warningf("fail to read %s : %v", fileName, readErr)
		return fmt.Errorf("fail to read %s : %v", fileName, readErr)
	}
	return iam.loadS3ApiConfigurationFromBytes(content)
}

func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromBytes(content []byte) error {
	s3ApiConfiguration := &iam_pb.S3ApiConfiguration{}
	if err := filer.ParseS3ConfigurationFromBytes(content, s3ApiConfiguration); err != nil {
		glog.Warningf("unmarshal error: %v", err)
		return fmt.Errorf("unmarshal error: %v", err)
	}
	if err := iam.loadS3ApiConfiguration(s3ApiConfiguration); err != nil {
		return err
	}
	return nil
}

func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3ApiConfiguration) error {
	var identities []*Identity
	for _, ident := range config.Identities {
		t := &Identity{
			Name:        ident.Name,
			Credentials: nil,
			Actions:     nil,
		}
		for _, action := range ident.Actions {
			t.Actions = append(t.Actions, Action(action))
		}
		for _, cred := range ident.Credentials {
			t.Credentials = append(t.Credentials, &Credential{
				AccessKey: cred.AccessKey,
				SecretKey: cred.SecretKey,
			})
		}
		identities = append(identities, t)
	}
	iam.m.Lock()
	// atomically switch
	iam.identities = identities
	iam.m.Unlock()
	return nil
}

func (iam *IdentityAccessManagement) isEnabled() bool {
	iam.m.RLock()
	defer iam.m.RUnlock()
	return len(iam.identities) > 0
}

func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identity *Identity, cred *Credential, found bool) {

	iam.m.RLock()
	defer iam.m.RUnlock()
	for _, ident := range iam.identities {
		for _, cred := range ident.Credentials {
			// println("checking", ident.Name, cred.AccessKey)
			if cred.AccessKey == accessKey {
				return ident, cred, true
			}
		}
	}
	glog.V(1).Infof("could not find accessKey %s", accessKey)
	return nil, nil, false
}

func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, found bool) {
	iam.m.RLock()
	defer iam.m.RUnlock()
	for _, ident := range iam.identities {
		if ident.Name == "anonymous" {
			return ident, true
		}
	}
	return nil, false
}

func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) http.HandlerFunc {

	if !iam.isEnabled() {
		return f
	}

	return func(w http.ResponseWriter, r *http.Request) {
		identity, errCode := iam.authRequest(r, action)
		if errCode == s3err.ErrNone {
			if identity != nil && identity.Name != "" {
				r.Header.Set(xhttp.AmzIdentityId, identity.Name)
				if identity.isAdmin() {
					r.Header.Set(xhttp.AmzIsAdmin, "true")
				} else if _, ok := r.Header[xhttp.AmzIsAdmin]; ok {
					r.Header.Del(xhttp.AmzIsAdmin)
				}
			}
			f(w, r)
			return
		}
		s3err.WriteErrorResponse(w, r, errCode)
	}
}

// check whether the request has valid access keys
func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) (*Identity, s3err.ErrorCode) {
	var identity *Identity
	var s3Err s3err.ErrorCode
	var found bool
	switch getRequestAuthType(r) {
	case authTypeStreamingSigned:
		return identity, s3err.ErrNone
	case authTypeUnknown:
		glog.V(3).Infof("unknown auth type")
		return identity, s3err.ErrAccessDenied
	case authTypePresignedV2, authTypeSignedV2:
		glog.V(3).Infof("v2 auth type")
		identity, s3Err = iam.isReqAuthenticatedV2(r)
	case authTypeSigned, authTypePresigned:
		glog.V(3).Infof("v4 auth type")
		identity, s3Err = iam.reqSignatureV4Verify(r)
	case authTypePostPolicy:
		glog.V(3).Infof("post policy auth type")
		return identity, s3err.ErrNone
	case authTypeJWT:
		glog.V(3).Infof("jwt auth type")
		return identity, s3err.ErrNotImplemented
	case authTypeAnonymous:
		identity, found = iam.lookupAnonymous()
		if !found {
			return identity, s3err.ErrAccessDenied
		}
	default:
		return identity, s3err.ErrNotImplemented
	}

	if s3Err != s3err.ErrNone {
		return identity, s3Err
	}

	glog.V(3).Infof("user name: %v actions: %v, action: %v", identity.Name, identity.Actions, action)

	bucket, _ := getBucketAndObject(r)

	if !identity.canDo(action, bucket) {
		return identity, s3err.ErrAccessDenied
	}

	return identity, s3err.ErrNone

}

func (iam *IdentityAccessManagement) authUser(r *http.Request) (*Identity, s3err.ErrorCode) {
	var identity *Identity
	var s3Err s3err.ErrorCode
	var found bool
	switch getRequestAuthType(r) {
	case authTypeStreamingSigned:
		return identity, s3err.ErrNone
	case authTypeUnknown:
		glog.V(3).Infof("unknown auth type")
		return identity, s3err.ErrAccessDenied
	case authTypePresignedV2, authTypeSignedV2:
		glog.V(3).Infof("v2 auth type")
		identity, s3Err = iam.isReqAuthenticatedV2(r)
	case authTypeSigned, authTypePresigned:
		glog.V(3).Infof("v4 auth type")
		identity, s3Err = iam.reqSignatureV4Verify(r)
	case authTypePostPolicy:
		glog.V(3).Infof("post policy auth type")
		return identity, s3err.ErrNone
	case authTypeJWT:
		glog.V(3).Infof("jwt auth type")
		return identity, s3err.ErrNotImplemented
	case authTypeAnonymous:
		identity, found = iam.lookupAnonymous()
		if !found {
			return identity, s3err.ErrAccessDenied
		}
	default:
		return identity, s3err.ErrNotImplemented
	}

	glog.V(3).Infof("auth error: %v", s3Err)
	if s3Err != s3err.ErrNone {
		return identity, s3Err
	}
	return identity, s3err.ErrNone
}

func (identity *Identity) canDo(action Action, bucket string) bool {
	if identity.isAdmin() {
		return true
	}
	for _, a := range identity.Actions {
		if a == action {
			return true
		}
	}
	if bucket == "" {
		return false
	}
	limitedByBucket := string(action) + ":" + bucket
	adminLimitedByBucket := s3_constants.ACTION_ADMIN + ":" + bucket
	for _, a := range identity.Actions {
		act := string(a)
		if strings.HasSuffix(act, "*") {
			if strings.HasPrefix(limitedByBucket, act[:len(act)-1]) {
				return true
			}
			if strings.HasPrefix(adminLimitedByBucket, act[:len(act)-1]) {
				return true
			}
		} else {
			if act == limitedByBucket {
				return true
			}
			if act == adminLimitedByBucket {
				return true
			}
		}
	}
	return false
}

func (identity *Identity) isAdmin() bool {
	for _, a := range identity.Actions {
		if a == "Admin" {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package libgodelbrot

import (
    "image"
    "math"
    "functorama.com/demo/base"
)

func Recolor(desc *Info, gray image.Image) *image.NRGBA {
    // CAUTION lossy conversion
    iterlim := desc.UserRequest.IterateLimit

    dfac := makeDrawFacade(desc)
    palette := dfac.Colors()

    bnd := gray.Bounds()
    bright := image.NewNRGBA(bnd)
    scale := float64(0xff) / float64(0xffff)

    for x := bnd.Min.X; x < bnd.Max.X; x++ {
        for y := bnd.Min.Y; y < bnd.Max.Y; y++ {
            bigdiv, _, _, _ := gray.At(x, y).RGBA()
            invdiv := uint8(math.Floor(scale * float64(bigdiv)))
            member := base.BaseMandelbrot{
                InvDivergence: invdiv,
                InSet: invdiv == iterlim,
            }
            col := palette.Color(member)
            bright.Set(x, y, col)
        }
    }

    return bright
}<commit_msg>colorbrot without FPU<commit_after>package libgodelbrot

import (
    "image"
    "functorama.com/demo/base"
)

func Recolor(desc *Info, gray image.Image) *image.NRGBA {
    // CAUTION lossy conversion
    iterlim := desc.UserRequest.IterateLimit

    dfac := makeDrawFacade(desc)
    palette := dfac.Colors()

    bnd := gray.Bounds()
    bright := image.NewNRGBA(bnd)

    for x := bnd.Min.X; x < bnd.Max.X; x++ {
        for y := bnd.Min.Y; y < bnd.Max.Y; y++ {
            bigdiv, _, _, _ := gray.At(x, y).RGBA()
            invdiv := uint8(bigdiv >> 8)
            member := base.BaseMandelbrot{
                InvDivergence: invdiv,
                InSet: invdiv == iterlim,
            }
            col := palette.Color(member)
            bright.Set(x, y, col)
        }
    }

    return bright
}<|endoftext|>


<commit_before>package emulator

import (
	"testing"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	apitesting "k8s.io/kubernetes/pkg/api/testing"
	"k8s.io/kubernetes/pkg/api/testapi"
	"net/http"
	"k8s.io/kubernetes/pkg/runtime"
	"bytes"
	"io/ioutil"
	"reflect"
	"k8s.io/kubernetes/pkg/client/restclient"
	"fmt"
	"net/url"
	"strings"
)

func testPodsData() *api.PodList {
	pods := &api.PodList{
		ListMeta: unversioned.ListMeta{
			ResourceVersion: "15",
		},
		Items: []api.Pod{
			{
				ObjectMeta: api.ObjectMeta{Name: "foo", Namespace: "test", ResourceVersion: "10"},
				Spec:       apitesting.DeepEqualSafePodSpec(),
			},
			{
				ObjectMeta: api.ObjectMeta{Name: "bar", Namespace: "test", ResourceVersion: "11"},
				Spec:       apitesting.DeepEqualSafePodSpec(),
			},
		},
	}

	return pods
}

func testServicesData() *api.ServiceList {
	svc := &api.ServiceList{
		ListMeta: unversioned.ListMeta{
			ResourceVersion: "16",
		},
		Items: []api.Service{
			{
				ObjectMeta: api.ObjectMeta{Name: "baz", Namespace: "test", ResourceVersion: "12"},
				Spec: api.ServiceSpec{
					SessionAffinity: "None",
					Type:            api.ServiceTypeClusterIP,
				},
			},
		},
	}
	return svc
}

func testReplicationControllersData() *api.ReplicationControllerList {
	rc := &api.ReplicationControllerList{
		ListMeta: unversioned.ListMeta{
			ResourceVersion: "17",
		},
		Items: []api.ReplicationController{
			{
				ObjectMeta: api.ObjectMeta{Name: "rc1", Namespace: "test", ResourceVersion: "18"},
				Spec: api.ReplicationControllerSpec{
					Replicas: 1,
				},
			},
		},
	}
	return rc
}

// RESTClient provides a fake RESTClient interface.
type RESTClient struct {
	Client               *http.Client
	NegotiatedSerializer runtime.NegotiatedSerializer

	Req  *http.Request
	Resp *http.Response
	Err  error
}

func (c *RESTClient) Get() *restclient.Request {
	return c.request("GET")
}

func (c *RESTClient) Put() *restclient.Request {
	return c.request("PUT")
}

func (c *RESTClient) Patch(_ api.PatchType) *restclient.Request {
	return c.request("PATCH")
}

func (c *RESTClient) Post() *restclient.Request {
	return c.request("POST")
}

func (c *RESTClient) Delete() *restclient.Request {
	return c.request("DELETE")
}

func (c *RESTClient) request(verb string) *restclient.Request {
	config := restclient.ContentConfig{
		ContentType:          runtime.ContentTypeJSON,
		GroupVersion:         testapi.Default.GroupVersion(),
		NegotiatedSerializer: c.NegotiatedSerializer,
	}
	ns := c.NegotiatedSerializer
	serializer, _ := ns.SerializerForMediaType(runtime.ContentTypeJSON, nil)
	streamingSerializer, _ := ns.StreamingSerializerForMediaType(runtime.ContentTypeJSON, nil)
	internalVersion := unversioned.GroupVersion{
		Group:   testapi.Default.GroupVersion().Group,
		Version: runtime.APIVersionInternal,
	}
	internalVersion.Version = runtime.APIVersionInternal
	serializers := restclient.Serializers{
		Encoder:             ns.EncoderForVersion(serializer, *testapi.Default.GroupVersion()),
		Decoder:             ns.DecoderToVersion(serializer, internalVersion),
		StreamingSerializer: streamingSerializer,
		Framer:              streamingSerializer.Framer,
	}
	return restclient.NewRequest(c, verb, &url.URL{Host: "localhost"}, "", config, serializers, nil, nil)
}

func (c *RESTClient) Do(req *http.Request) (*http.Response, error) {
	if c.Err != nil {
		return nil, c.Err
	}
	c.Req = req

	// //localhost/pods?resourceVersion=0
	parts := splitPath(req.URL.Path)
	if len(parts) < 1 {
		return nil, fmt.Errorf("Missing resource in REST client request url")
	}

	var obj runtime.Object

	switch parts[0] {
		case "pods":
			obj = testPodsData()
		case "services":
			obj = testServicesData()
		case "replicationcontrollers":
			obj = testReplicationControllersData()
		default:
			return nil, fmt.Errorf("Resource %s not recognized", parts[0])
	}

	header := http.Header{}
	header.Set("Content-Type", runtime.ContentTypeJSON)
	body := ioutil.NopCloser(bytes.NewReader([]byte(runtime.EncodeOrDie(testapi.Default.Codec(), obj))))
	c.Resp = &http.Response{StatusCode: 200, Header: header, Body: body}

	return c.Resp, nil
}

// splitPath returns the segments for a URL path.
func splitPath(path string) []string {
        path = strings.Trim(path, "/")
        if path == "" {
                return []string{}
        }
        return strings.Split(path, "/")
}

func compareItems(expected, actual interface{}) bool {
	if reflect.TypeOf(expected).Kind() != reflect.Slice {
		return false
	}

	if reflect.TypeOf(actual).Kind() != reflect.Slice {
		return false
	}

	expectedSlice := reflect.ValueOf(expected)
	expectedMap := make(map[string]interface{})
	for i := 0; i < expectedSlice.Len(); i++ {
		meta := expectedSlice.Index(i).FieldByName("ObjectMeta").Interface().(api.ObjectMeta)
		key := strings.Join([]string{meta.Namespace, meta.Name, meta.ResourceVersion}, "/")
		expectedMap[key] = expectedSlice.Index(i).Interface()
	}

	actualMap := make(map[string]interface{})
	actualSlice := reflect.ValueOf(actual)
	for i := 0; i < actualSlice.Len(); i++ {
		meta := actualSlice.Index(i).FieldByName("ObjectMeta").Interface().(api.ObjectMeta)
		key := strings.Join([]string{meta.Namespace, meta.Name, meta.ResourceVersion}, "/")
		actualMap[key] = actualSlice.Index(i).Interface()
	}

	return reflect.DeepEqual(expectedMap, actualMap)
}

func TestSyncPods(t *testing.T) {
	fakeClient := &RESTClient{
		NegotiatedSerializer: testapi.Default.NegotiatedSerializer(),
	}

	data := testPodsData()
	expected := data.Items

	emulator := NewClientEmulator()

	err := emulator.sync(fakeClient)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}

	storedItems := emulator.PodCache.List()
	actual := make([]api.Pod, 0, len(storedItems))
	for _, value := range storedItems {
		item, ok := value.(*api.Pod)
		if !ok {
			t.Errorf("Expected api.Pod type, found different")
		}
		actual = append(actual, *item)
	}
	if !compareItems(expected, actual) {
		t.Errorf("unexpected object: expected: %#v\n actual: %#v", expected, actual)
	}

}

func TestSyncServices(t *testing.T) {
	fakeClient := &RESTClient{
		NegotiatedSerializer: testapi.Default.NegotiatedSerializer(),
	}

	data := testServicesData()
	expected := data.Items

	emulator := NewClientEmulator()

	err := emulator.sync(fakeClient)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}

	storedItems := emulator.ServiceCache.List()
	actual := make([]api.Service, 0, len(storedItems))
	for _, value := range storedItems {
		item, ok := value.(*api.Service)
		if !ok {
			t.Errorf("Expected api.Service type, found different")
		}
		actual = append(actual, *item)
	}

	if !compareItems(expected, actual) {
		t.Errorf("unexpected object: expected: %#v\n actual: %#v", expected, actual)
	}
}

func TestSyncReplicationControllers(t *testing.T) {
	fakeClient := &RESTClient{
		NegotiatedSerializer: testapi.Default.NegotiatedSerializer(),
	}

	data := testReplicationControllersData()
	expected := data.Items

	emulator := NewClientEmulator()

	err := emulator.sync(fakeClient)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}

	storedItems := emulator.ReplicationControllerCache.List()
	actual := make([]api.ReplicationController, 0, len(storedItems))
	for _, value := range storedItems {
		item, ok := value.(*api.ReplicationController)
		if !ok {
			t.Errorf("Expected api.Service type, found different")
		}
		actual = append(actual, *item)
	}

	if !compareItems(expected, actual) {
		t.Errorf("unexpected object: expected: %#v\n actual: %#v", expected, actual)
	}
}
<commit_msg>make the resource source data functions configurable<commit_after>package emulator

import (
	"testing"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	apitesting "k8s.io/kubernetes/pkg/api/testing"
	"k8s.io/kubernetes/pkg/api/testapi"
	"net/http"
	"k8s.io/kubernetes/pkg/runtime"
	"bytes"
	"io/ioutil"
	"reflect"
	"k8s.io/kubernetes/pkg/client/restclient"
	"fmt"
	"net/url"
	"strings"
)

func testPodsData() *api.PodList {
	pods := &api.PodList{
		ListMeta: unversioned.ListMeta{
			ResourceVersion: "15",
		},
		Items: []api.Pod{
			{
				ObjectMeta: api.ObjectMeta{Name: "foo", Namespace: "test", ResourceVersion: "10"},
				Spec:       apitesting.DeepEqualSafePodSpec(),
			},
			{
				ObjectMeta: api.ObjectMeta{Name: "bar", Namespace: "test", ResourceVersion: "11"},
				Spec:       apitesting.DeepEqualSafePodSpec(),
			},
		},
	}

	return pods
}

func testServicesData() *api.ServiceList {
	svc := &api.ServiceList{
		ListMeta: unversioned.ListMeta{
			ResourceVersion: "16",
		},
		Items: []api.Service{
			{
				ObjectMeta: api.ObjectMeta{Name: "baz", Namespace: "test", ResourceVersion: "12"},
				Spec: api.ServiceSpec{
					SessionAffinity: "None",
					Type:            api.ServiceTypeClusterIP,
				},
			},
		},
	}
	return svc
}

func testReplicationControllersData() *api.ReplicationControllerList {
	rc := &api.ReplicationControllerList{
		ListMeta: unversioned.ListMeta{
			ResourceVersion: "17",
		},
		Items: []api.ReplicationController{
			{
				ObjectMeta: api.ObjectMeta{Name: "rc1", Namespace: "test", ResourceVersion: "18"},
				Spec: api.ReplicationControllerSpec{
					Replicas: 1,
				},
			},
		},
	}
	return rc
}

// RESTClient provides a fake RESTClient interface.
type RESTClient struct {
	NegotiatedSerializer runtime.NegotiatedSerializer

	Req  *http.Request
	Resp *http.Response
	Err  error

	podsDataSource func() *api.PodList
	servicesDataSource func() *api.ServiceList
	replicationControllersDataSource func() *api.ReplicationControllerList
}

func (c *RESTClient) Pods() *api.PodList {
	return c.podsDataSource()
}

func (c *RESTClient) Services() *api.ServiceList {
	return c.servicesDataSource()
}

func (c *RESTClient) ReplicationControllers() *api.ReplicationControllerList {
	return c.replicationControllersDataSource()
}

func (c *RESTClient) Get() *restclient.Request {
	return c.request("GET")
}

func (c *RESTClient) Put() *restclient.Request {
	return c.request("PUT")
}

func (c *RESTClient) Patch(_ api.PatchType) *restclient.Request {
	return c.request("PATCH")
}

func (c *RESTClient) Post() *restclient.Request {
	return c.request("POST")
}

func (c *RESTClient) Delete() *restclient.Request {
	return c.request("DELETE")
}

func (c *RESTClient) request(verb string) *restclient.Request {
	config := restclient.ContentConfig{
		ContentType:          runtime.ContentTypeJSON,
		GroupVersion:         testapi.Default.GroupVersion(),
		NegotiatedSerializer: c.NegotiatedSerializer,
	}
	ns := c.NegotiatedSerializer
	serializer, _ := ns.SerializerForMediaType(runtime.ContentTypeJSON, nil)
	streamingSerializer, _ := ns.StreamingSerializerForMediaType(runtime.ContentTypeJSON, nil)
	internalVersion := unversioned.GroupVersion{
		Group:   testapi.Default.GroupVersion().Group,
		Version: runtime.APIVersionInternal,
	}
	internalVersion.Version = runtime.APIVersionInternal
	serializers := restclient.Serializers{
		Encoder:             ns.EncoderForVersion(serializer, *testapi.Default.GroupVersion()),
		Decoder:             ns.DecoderToVersion(serializer, internalVersion),
		StreamingSerializer: streamingSerializer,
		Framer:              streamingSerializer.Framer,
	}
	return restclient.NewRequest(c, verb, &url.URL{Host: "localhost"}, "", config, serializers, nil, nil)
}

func (c *RESTClient) Do(req *http.Request) (*http.Response, error) {
	if c.Err != nil {
		return nil, c.Err
	}
	c.Req = req

	// //localhost/pods?resourceVersion=0
	parts := splitPath(req.URL.Path)
	if len(parts) < 1 {
		return nil, fmt.Errorf("Missing resource in REST client request url")
	}

	var obj runtime.Object

	switch parts[0] {
		case "pods":
			obj = c.Pods()
		case "services":
			obj = c.Services()
		case "replicationcontrollers":
			obj = c.ReplicationControllers()
		default:
			return nil, fmt.Errorf("Resource %s not recognized", parts[0])
	}

	header := http.Header{}
	header.Set("Content-Type", runtime.ContentTypeJSON)
	body := ioutil.NopCloser(bytes.NewReader([]byte(runtime.EncodeOrDie(testapi.Default.Codec(), obj))))
	c.Resp = &http.Response{StatusCode: 200, Header: header, Body: body}

	return c.Resp, nil
}

func NewRestClient() *RESTClient {

	client := &RESTClient{
		NegotiatedSerializer: testapi.Default.NegotiatedSerializer(),
		podsDataSource: testPodsData,
		servicesDataSource: testServicesData,
		replicationControllersDataSource: testReplicationControllersData,
	}

	return client
}

// splitPath returns the segments for a URL path.
func splitPath(path string) []string {
        path = strings.Trim(path, "/")
        if path == "" {
                return []string{}
        }
        return strings.Split(path, "/")
}

func compareItems(expected, actual interface{}) bool {
	if reflect.TypeOf(expected).Kind() != reflect.Slice {
		return false
	}

	if reflect.TypeOf(actual).Kind() != reflect.Slice {
		return false
	}

	expectedSlice := reflect.ValueOf(expected)
	expectedMap := make(map[string]interface{})
	for i := 0; i < expectedSlice.Len(); i++ {
		meta := expectedSlice.Index(i).FieldByName("ObjectMeta").Interface().(api.ObjectMeta)
		key := strings.Join([]string{meta.Namespace, meta.Name, meta.ResourceVersion}, "/")
		expectedMap[key] = expectedSlice.Index(i).Interface()
	}

	actualMap := make(map[string]interface{})
	actualSlice := reflect.ValueOf(actual)
	for i := 0; i < actualSlice.Len(); i++ {
		meta := actualSlice.Index(i).FieldByName("ObjectMeta").Interface().(api.ObjectMeta)
		key := strings.Join([]string{meta.Namespace, meta.Name, meta.ResourceVersion}, "/")
		actualMap[key] = actualSlice.Index(i).Interface()
	}

	return reflect.DeepEqual(expectedMap, actualMap)
}

func TestSyncPods(t *testing.T) {

	fakeClient := NewRestClient()
	expected := fakeClient.Pods().Items
	emulator := NewClientEmulator()

	err := emulator.sync(fakeClient)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}

	storedItems := emulator.PodCache.List()
	actual := make([]api.Pod, 0, len(storedItems))
	for _, value := range storedItems {
		item, ok := value.(*api.Pod)
		if !ok {
			t.Errorf("Expected api.Pod type, found different")
		}
		actual = append(actual, *item)
	}
	if !compareItems(expected, actual) {
		t.Errorf("unexpected object: expected: %#v\n actual: %#v", expected, actual)
	}

}

func TestSyncServices(t *testing.T) {

	fakeClient := NewRestClient()
	expected := fakeClient.Services().Items
	emulator := NewClientEmulator()

	err := emulator.sync(fakeClient)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}

	storedItems := emulator.ServiceCache.List()
	actual := make([]api.Service, 0, len(storedItems))
	for _, value := range storedItems {
		item, ok := value.(*api.Service)
		if !ok {
			t.Errorf("Expected api.Service type, found different")
		}
		actual = append(actual, *item)
	}

	if !compareItems(expected, actual) {
		t.Errorf("unexpected object: expected: %#v\n actual: %#v", expected, actual)
	}
}

func TestSyncReplicationControllers(t *testing.T) {

	fakeClient := NewRestClient()
	expected := fakeClient.ReplicationControllers().Items
	emulator := NewClientEmulator()

	err := emulator.sync(fakeClient)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}

	storedItems := emulator.ReplicationControllerCache.List()
	actual := make([]api.ReplicationController, 0, len(storedItems))
	for _, value := range storedItems {
		item, ok := value.(*api.ReplicationController)
		if !ok {
			t.Errorf("Expected api.Service type, found different")
		}
		actual = append(actual, *item)
	}

	if !compareItems(expected, actual) {
		t.Errorf("unexpected object: expected: %#v\n actual: %#v", expected, actual)
	}
}
<|endoftext|>