<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/fs"
	"io/ioutil"
	"os"
	"path/filepath"

	"kmodules.xyz/resource-metadata/apis/meta/v1alpha1"

	diff "github.com/yudai/gojsondiff"
	"github.com/yudai/gojsondiff/formatter"
	"sigs.k8s.io/yaml"
)

func check(filename string) (string, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return "", err
	}

	var original map[string]interface{}
	err = yaml.Unmarshal(data, &original)
	if err != nil {
		return "", err
	}
	sorted, err := json.Marshal(&original)
	if err != nil {
		return "", err
	}

	var rd v1alpha1.ResourceDescriptor
	err = yaml.Unmarshal(data, &rd)
	if err != nil {
		return "", err
	}
	parsed, err := json.Marshal(rd)
	if err != nil {
		return "", err
	}

	// Then, Check them
	differ := diff.New()
	d, err := differ.Compare(sorted, parsed)
	if err != nil {
		fmt.Printf("Failed to unmarshal file: %s\n", err.Error())
		os.Exit(3)
	}

	if d.Modified() {
		config := formatter.AsciiFormatterConfig{
			ShowArrayIndex: true,
			Coloring:       true,
		}

		f := formatter.NewAsciiFormatter(original, config)
		result, err := f.Format(d)
		if err != nil {
			return "", err
		}
		return result, nil
	} else {
		data, err := yaml.Marshal(rd)
		if err != nil {
			return "", err
		}
		err = ioutil.WriteFile(filename, data, 0644)
		if err != nil {
			return "", err
		}
	}

	return "", nil
}

func main() {
	err := filepath.Walk("./hub/resourcedescriptors/", func(path string, info fs.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		ext := filepath.Ext(info.Name())
		if ext != ".yml" && ext != ".yaml" && ext != ".json" {
			return nil
		}

		d, err := check(path)
		if err != nil {
			return err
		}
		if d != "" {
			return fmt.Errorf("parsing error in file %s: %s", path, d)
		}
		return nil
	})
	if err != nil {
		panic(err)
	}
}
<commit_msg>Fix build<commit_after>/*
Copyright AppsCode Inc. and Contributors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"fmt"
	"io/fs"
	"io/ioutil"
	"os"
	"path/filepath"

	"kmodules.xyz/resource-metadata/apis/meta/v1alpha1"

	diff "github.com/yudai/gojsondiff"
	"github.com/yudai/gojsondiff/formatter"
	"sigs.k8s.io/yaml"
)

func check(filename string) (string, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return "", err
	}

	var original map[string]interface{}
	err = yaml.Unmarshal(data, &original)
	if err != nil {
		return "", err
	}
	sorted, err := json.Marshal(&original)
	if err != nil {
		return "", err
	}

	var rd v1alpha1.ResourceDescriptor
	err = yaml.Unmarshal(data, &rd)
	if err != nil {
		return "", err
	}
	parsed, err := json.Marshal(rd)
	if err != nil {
		return "", err
	}

	// Then, Check them
	differ := diff.New()
	d, err := differ.Compare(sorted, parsed)
	if err != nil {
		fmt.Printf("Failed to unmarshal file: %s\n", err.Error())
		os.Exit(3)
	}

	if d.Modified() {
		config := formatter.AsciiFormatterConfig{
			ShowArrayIndex: true,
			Coloring:       true,
		}

		f := formatter.NewAsciiFormatter(original, config)
		result, err := f.Format(d)
		if err != nil {
			return "", err
		}
		return result, nil
	} else {
		data, err := yaml.Marshal(rd)
		if err != nil {
			return "", err
		}
		err = ioutil.WriteFile(filename, data, 0644)
		if err != nil {
			return "", err
		}
	}

	return "", nil
}

func main() {
	err := filepath.Walk("./hub/resourcedescriptors/", func(path string, info fs.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		ext := filepath.Ext(info.Name())
		if ext != ".yml" && ext != ".yaml" && ext != ".json" {
			return nil
		}

		d, err := check(path)
		if err != nil {
			return err
		}
		if d != "" {
			return fmt.Errorf("parsing error in file %s: %s", path, d)
		}
		return nil
	})
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The race detector does not understand ParFor synchronization.
// +build !race

package runtime_test

import (
	. "runtime"
	"testing"
	"unsafe"
)

var gdata []uint64

// Simple serial sanity test for parallelfor.
func TestParFor(t *testing.T) {
	const P = 1
	const N = 20
	data := make([]uint64, N)
	for i := uint64(0); i < N; i++ {
		data[i] = i
	}
	desc := NewParFor(P)
	// Avoid making func a closure: parfor cannot invoke them.
	// Since it doesn't happen in the C code, it's not worth doing
	// just for the test.
	gdata = data
	ParForSetup(desc, P, N, nil, true, func(desc *ParFor, i uint32) {
		data := gdata
		data[i] = data[i]*data[i] + 1
	})
	ParForDo(desc)
	for i := uint64(0); i < N; i++ {
		if data[i] != i*i+1 {
			t.Fatalf("Wrong element %d: %d", i, data[i])
		}
	}
}

// Test that nonblocking parallelfor does not block.
func TestParFor2(t *testing.T) {
	const P = 7
	const N = 1003
	data := make([]uint64, N)
	for i := uint64(0); i < N; i++ {
		data[i] = i
	}
	desc := NewParFor(P)
	ParForSetup(desc, P, N, (*byte)(unsafe.Pointer(&data)), false, func(desc *ParFor, i uint32) {
		d := *(*[]uint64)(unsafe.Pointer(desc.Ctx))
		d[i] = d[i]*d[i] + 1
	})
	for p := 0; p < P; p++ {
		ParForDo(desc)
	}
	for i := uint64(0); i < N; i++ {
		if data[i] != i*i+1 {
			t.Fatalf("Wrong element %d: %d", i, data[i])
		}
	}
}

// Test that iterations are properly distributed.
func TestParForSetup(t *testing.T) {
	const P = 11
	const N = 101
	desc := NewParFor(P)
	for n := uint32(0); n < N; n++ {
		for p := uint32(1); p <= P; p++ {
			ParForSetup(desc, p, n, nil, true, func(desc *ParFor, i uint32) {})
			sum := uint32(0)
			size0 := uint32(0)
			end0 := uint32(0)
			for i := uint32(0); i < p; i++ {
				begin, end := ParForIters(desc, i)
				size := end - begin
				sum += size
				if i == 0 {
					size0 = size
					if begin != 0 {
						t.Fatalf("incorrect begin: %d (n=%d, p=%d)", begin, n, p)
					}
				} else {
					if size != size0 && size != size0+1 {
						t.Fatalf("incorrect size: %d/%d (n=%d, p=%d)", size, size0, n, p)
					}
					if begin != end0 {
						t.Fatalf("incorrect begin/end: %d/%d (n=%d, p=%d)", begin, end0, n, p)
					}
				}
				end0 = end
			}
			if sum != n {
				t.Fatalf("incorrect sum: %d/%d (p=%d)", sum, n, p)
			}
		}
	}
}

// Test parallel parallelfor.
func TestParForParallel(t *testing.T) {
	if GOARCH != "amd64" {
		t.Log("temporarily disabled, see http://golang.org/issue/4155")
		return
	}

	N := uint64(1e7)
	if testing.Short() {
		N /= 10
	}
	data := make([]uint64, N)
	for i := uint64(0); i < N; i++ {
		data[i] = i
	}
	P := GOMAXPROCS(-1)
	c := make(chan bool, P)
	desc := NewParFor(uint32(P))
	gdata = data
	ParForSetup(desc, uint32(P), uint32(N), nil, false, func(desc *ParFor, i uint32) {
		data := gdata
		data[i] = data[i]*data[i] + 1
	})
	for p := 1; p < P; p++ {
		go func() {
			ParForDo(desc)
			c <- true
		}()
	}
	ParForDo(desc)
	for p := 1; p < P; p++ {
		<-c
	}
	for i := uint64(0); i < N; i++ {
		if data[i] != i*i+1 {
			t.Fatalf("Wrong element %d: %d", i, data[i])
		}
	}

	data, desc = nil, nil
	GC()
}
<commit_msg>runtime: re-enable TestParForParallel<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The race detector does not understand ParFor synchronization.
// +build !race

package runtime_test

import (
	. "runtime"
	"testing"
	"unsafe"
)

var gdata []uint64

// Simple serial sanity test for parallelfor.
func TestParFor(t *testing.T) {
	const P = 1
	const N = 20
	data := make([]uint64, N)
	for i := uint64(0); i < N; i++ {
		data[i] = i
	}
	desc := NewParFor(P)
	// Avoid making func a closure: parfor cannot invoke them.
	// Since it doesn't happen in the C code, it's not worth doing
	// just for the test.
	gdata = data
	ParForSetup(desc, P, N, nil, true, func(desc *ParFor, i uint32) {
		data := gdata
		data[i] = data[i]*data[i] + 1
	})
	ParForDo(desc)
	for i := uint64(0); i < N; i++ {
		if data[i] != i*i+1 {
			t.Fatalf("Wrong element %d: %d", i, data[i])
		}
	}
}

// Test that nonblocking parallelfor does not block.
func TestParFor2(t *testing.T) {
	const P = 7
	const N = 1003
	data := make([]uint64, N)
	for i := uint64(0); i < N; i++ {
		data[i] = i
	}
	desc := NewParFor(P)
	ParForSetup(desc, P, N, (*byte)(unsafe.Pointer(&data)), false, func(desc *ParFor, i uint32) {
		d := *(*[]uint64)(unsafe.Pointer(desc.Ctx))
		d[i] = d[i]*d[i] + 1
	})
	for p := 0; p < P; p++ {
		ParForDo(desc)
	}
	for i := uint64(0); i < N; i++ {
		if data[i] != i*i+1 {
			t.Fatalf("Wrong element %d: %d", i, data[i])
		}
	}
}

// Test that iterations are properly distributed.
func TestParForSetup(t *testing.T) {
	const P = 11
	const N = 101
	desc := NewParFor(P)
	for n := uint32(0); n < N; n++ {
		for p := uint32(1); p <= P; p++ {
			ParForSetup(desc, p, n, nil, true, func(desc *ParFor, i uint32) {})
			sum := uint32(0)
			size0 := uint32(0)
			end0 := uint32(0)
			for i := uint32(0); i < p; i++ {
				begin, end := ParForIters(desc, i)
				size := end - begin
				sum += size
				if i == 0 {
					size0 = size
					if begin != 0 {
						t.Fatalf("incorrect begin: %d (n=%d, p=%d)", begin, n, p)
					}
				} else {
					if size != size0 && size != size0+1 {
						t.Fatalf("incorrect size: %d/%d (n=%d, p=%d)", size, size0, n, p)
					}
					if begin != end0 {
						t.Fatalf("incorrect begin/end: %d/%d (n=%d, p=%d)", begin, end0, n, p)
					}
				}
				end0 = end
			}
			if sum != n {
				t.Fatalf("incorrect sum: %d/%d (p=%d)", sum, n, p)
			}
		}
	}
}

// Test parallel parallelfor.
func TestParForParallel(t *testing.T) {
	N := uint64(1e7)
	if testing.Short() {
		N /= 10
	}
	data := make([]uint64, N)
	for i := uint64(0); i < N; i++ {
		data[i] = i
	}
	P := GOMAXPROCS(-1)
	c := make(chan bool, P)
	desc := NewParFor(uint32(P))
	gdata = data
	ParForSetup(desc, uint32(P), uint32(N), nil, false, func(desc *ParFor, i uint32) {
		data := gdata
		data[i] = data[i]*data[i] + 1
	})
	for p := 1; p < P; p++ {
		go func() {
			ParForDo(desc)
			c <- true
		}()
	}
	ParForDo(desc)
	for p := 1; p < P; p++ {
		<-c
	}
	for i := uint64(0); i < N; i++ {
		if data[i] != i*i+1 {
			t.Fatalf("Wrong element %d: %d", i, data[i])
		}
	}

	data, desc = nil, nil
	GC()
}
<|endoftext|>


<commit_before>/*
 * Script that runs srcgraph make -test on a set of test repositories
 */
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/aybabtme/color/brush"
)

type TestCase struct {
	Name     string
	CloneURL string
	CommitID string
}

var tests = []TestCase{
	{"go-sample-0", "https://github.com/sgtest/go-sample-0", "1dd4664fec342c0727850380931429a5850a4402"},
	{"python-sample-0", "https://github.com/sgtest/python-sample-0", "7f4959756bdfc406f318fedfe9f8e8ba98dfe48b"},
	{"python-sample-1", "https://github.com/sgtest/python-sample-1", "0b7a781f8bebf79d36fa8aacb400ca6b892cba95"},
	{"javascript-nodejs-sample-0", "https://github.com/sgtest/javascript-nodejs-sample-0", "e10faf45fd536676a48bbbdb6ab650e7721782bb"},
	{"javascript-nodejs-xrefs-0", "https://github.com/sgtest/javascript-nodejs-xrefs-0", "a82948d15bfcbac86530caf0e9c0929e6c41c353"},
}

var verbose = flag.Bool("verbose", false, "if verbose is true, print individual test output")
var list = flag.Bool("list", false, "list all test cases")
var match = flag.String("match", "", "match cases (by name substring)")

func main() {
	flag.Parse()
	if *list {
		for _, test := range tests {
			fmt.Printf("%s (%s:%s)\n", test.Name, test.CloneURL, test.CommitID)
		}
		os.Exit(0)
	}

	var exitCode = 0
	testDir, err := ioutil.TempDir("", "sg-test")
	if err != nil {
		log.Print(err)
		os.Exit(1)
	}
	defer os.RemoveAll(testDir)

	for _, test := range tests {
		if *match != "" && !strings.Contains(test.Name, *match) {
			continue
		}

		fmt.Printf("Testing %s...\n", test.Name)
		err := testRepo(test, testDir)
		if err != nil {
			exitCode = 1
			fmt.Printf(brush.Red("Test %s failed\n").String(), test.Name)
		} else {
			fmt.Printf(brush.Green("Test %s succeeded\n").String(), test.Name)
		}
	}

	os.Exit(exitCode)
}

func testRepo(t TestCase, testDir string) error {
	{
		cloneCmd := exec.Command("git", "clone", t.CloneURL, t.Name)
		cloneCmd.Dir = testDir
		if *verbose {
			cloneCmd.Stdout, cloneCmd.Stderr = os.Stdout, os.Stderr
		}
		err := cloneCmd.Run()
		if err != nil {
			return err
		}
	}
	{
		checkoutCmd := exec.Command("git", "checkout", t.CommitID)
		checkoutCmd.Dir = filepath.Join(testDir, t.Name)
		if *verbose {
			checkoutCmd.Stdout, checkoutCmd.Stderr = os.Stdout, os.Stderr
		}
		err := checkoutCmd.Run()
		if err != nil {
			return fmt.Errorf("Error (%s)", err)
		}
	}
	{
		srcgraphCmd := exec.Command("srcgraph", "make", "-test")
		srcgraphCmd.Dir = filepath.Join(testDir, t.Name)
		if *verbose {
			srcgraphCmd.Stdout, srcgraphCmd.Stderr = os.Stdout, os.Stderr
		}
		err := srcgraphCmd.Run()
		if err != nil {
			return fmt.Errorf("Error (%s)", err)
		}
	}
	return nil
}
<commit_msg>remove old test_srcgraph<commit_after><|endoftext|>


<commit_before>/*
 * Mini Copy, (C) 2014,2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"time"

	"github.com/cheggaaa/pb"
	"github.com/minio-io/cli"
	"github.com/minio-io/mc/pkg/client"
	"github.com/minio-io/mc/pkg/console"
	"github.com/minio-io/mc/pkg/qdb"
	"github.com/minio-io/minio/pkg/iodine"
	"github.com/minio-io/minio/pkg/utils/log"
)

const (
	printDate = "2006-01-02 15:04:05 MST"
)

// printBuckets lists buckets and its metadata
func printBuckets(v []*client.Bucket) {
	for _, b := range v {
		console.Infof("%23s %13s %s\n", b.CreationDate.Local().Format(printDate), "", b.Name)
	}
}

// printObjects prints a metadata of a list of objects
func printObjects(v []*client.Item) {
	if len(v) > 0 {
		// Items are already sorted
		for _, b := range v {
			printObject(b.LastModified, b.Size, b.Key)
		}
	}
}

// printObject prints object meta-data
func printObject(date time.Time, v int64, key string) {
	console.Infof("%23s %13s %s\n", date.Local().Format(printDate), pb.FormatBytes(v), key)
}

func doListBuckets(clnt client.Client, urlStr string) {
	var err error
	var buckets []*client.Bucket

	buckets, err = clnt.ListBuckets()
	for i := 0; i < globalMaxRetryFlag && err != nil; i++ {
		buckets, err = clnt.ListBuckets()
		time.Sleep(time.Duration(i*i) * time.Second)
	}
	if err != nil {
		log.Debug.Println(iodine.New(err, nil))
		console.Fatalln("mc: Unable to list buckets for URL [%s]\n", urlStr)
	}
	printBuckets(buckets)
}

func doListObjects(clnt client.Client, bucket, object, urlStr string) {
	var err error
	var items []*client.Item

	items, err = clnt.ListObjects(bucket, object)
	for i := 0; i < globalMaxRetryFlag && err != nil; i++ {
		items, err = clnt.ListObjects(bucket, object)
		// Progressively longer delays
		time.Sleep(time.Duration(i*i) * time.Second)
	}
	if err != nil {
		log.Debug.Println(iodine.New(err, nil))
		console.Fatalf("mc: Unable to list objects for URL [%s]\n", urlStr)
	}
	printObjects(items)
}

// runListCmd lists objects inside a bucket
func runListCmd(ctx *cli.Context) {
	if len(ctx.Args()) < 1 {
		cli.ShowCommandHelpAndExit(ctx, "ls", 1) // last argument is exit code
	}
	config, err := getMcConfig()
	if err != nil {
		log.Debug.Println(iodine.New(err, nil))
		console.Fatalln("mc: Unable to get config")
	}
	for _, arg := range ctx.Args() {
		u, err := parseURL(arg, config.GetMapString("Aliases"))
		if err != nil {
			switch iodine.ToError(err).(type) {
			case errUnsupportedScheme:
				log.Debug.Println(iodine.New(err, nil))
				console.Fatalf("mc: Unable to parse URL [%s], %s\n", arg, guessPossibleURL(arg))
			default:
				log.Debug.Println(iodine.New(err, nil))
				console.Fatalf("mc: Unable to parse URL [%s]\n", arg)
			}
		}
		doListCmd(mcClientManager{}, u, globalDebugFlag)
	}
}
func doListCmd(manager clientManager, u string, debug bool) {
	clnt, err := manager.getNewClient(u, debug)
	if err != nil {
		log.Debug.Println(iodine.New(err, nil))
		console.Fatalf("mc: Unable to instantiate a new client for URL [%s]\n", u)
	}

	bucket, object, err := url2Object(u)
	if err != nil {
		log.Debug.Println(iodine.New(err, nil))
		console.Fatalf("mc: Unable to decode bucket and object name from the URL [%s]\n", u)
	}

	// ListBuckets() will not be called for fsClient() as its not needed.
	if bucket == "" && getURLType(u) != urlFS {
		doListBuckets(clnt, u)
	} else {
		doListObjects(clnt, bucket, object, u)
	}
}
<commit_msg>fixing build<commit_after>/*
 * Mini Copy, (C) 2014,2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"time"

	"github.com/cheggaaa/pb"
	"github.com/minio-io/cli"
	"github.com/minio-io/mc/pkg/client"
	"github.com/minio-io/mc/pkg/console"
	"github.com/minio-io/minio/pkg/iodine"
	"github.com/minio-io/minio/pkg/utils/log"
)

const (
	printDate = "2006-01-02 15:04:05 MST"
)

// printBuckets lists buckets and its metadata
func printBuckets(v []*client.Bucket) {
	for _, b := range v {
		console.Infof("%23s %13s %s\n", b.CreationDate.Local().Format(printDate), "", b.Name)
	}
}

// printObjects prints a metadata of a list of objects
func printObjects(v []*client.Item) {
	if len(v) > 0 {
		// Items are already sorted
		for _, b := range v {
			printObject(b.LastModified, b.Size, b.Key)
		}
	}
}

// printObject prints object meta-data
func printObject(date time.Time, v int64, key string) {
	console.Infof("%23s %13s %s\n", date.Local().Format(printDate), pb.FormatBytes(v), key)
}

func doListBuckets(clnt client.Client, urlStr string) {
	var err error
	var buckets []*client.Bucket

	buckets, err = clnt.ListBuckets()
	for i := 0; i < globalMaxRetryFlag && err != nil; i++ {
		buckets, err = clnt.ListBuckets()
		time.Sleep(time.Duration(i*i) * time.Second)
	}
	if err != nil {
		log.Debug.Println(iodine.New(err, nil))
		console.Fatalln("mc: Unable to list buckets for URL [%s]\n", urlStr)
	}
	printBuckets(buckets)
}

func doListObjects(clnt client.Client, bucket, object, urlStr string) {
	var err error
	var items []*client.Item

	items, err = clnt.ListObjects(bucket, object)
	for i := 0; i < globalMaxRetryFlag && err != nil; i++ {
		items, err = clnt.ListObjects(bucket, object)
		// Progressively longer delays
		time.Sleep(time.Duration(i*i) * time.Second)
	}
	if err != nil {
		log.Debug.Println(iodine.New(err, nil))
		console.Fatalf("mc: Unable to list objects for URL [%s]\n", urlStr)
	}
	printObjects(items)
}

// runListCmd lists objects inside a bucket
func runListCmd(ctx *cli.Context) {
	if len(ctx.Args()) < 1 {
		cli.ShowCommandHelpAndExit(ctx, "ls", 1) // last argument is exit code
	}
	config, err := getMcConfig()
	if err != nil {
		log.Debug.Println(iodine.New(err, nil))
		console.Fatalln("mc: Unable to get config")
	}
	for _, arg := range ctx.Args() {
		u, err := parseURL(arg, config.GetMapString("Aliases"))
		if err != nil {
			switch iodine.ToError(err).(type) {
			case errUnsupportedScheme:
				log.Debug.Println(iodine.New(err, nil))
				console.Fatalf("mc: Unable to parse URL [%s], %s\n", arg, guessPossibleURL(arg))
			default:
				log.Debug.Println(iodine.New(err, nil))
				console.Fatalf("mc: Unable to parse URL [%s]\n", arg)
			}
		}
		doListCmd(mcClientManager{}, u, globalDebugFlag)
	}
}
func doListCmd(manager clientManager, u string, debug bool) {
	clnt, err := manager.getNewClient(u, debug)
	if err != nil {
		log.Debug.Println(iodine.New(err, nil))
		console.Fatalf("mc: Unable to instantiate a new client for URL [%s]\n", u)
	}

	bucket, object, err := url2Object(u)
	if err != nil {
		log.Debug.Println(iodine.New(err, nil))
		console.Fatalf("mc: Unable to decode bucket and object name from the URL [%s]\n", u)
	}

	// ListBuckets() will not be called for fsClient() as its not needed.
	if bucket == "" && getURLType(u) != urlFS {
		doListBuckets(clnt, u)
	} else {
		doListObjects(clnt, bucket, object, u)
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"github.com/Songmu/retry"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/spf13/cobra"
	"os"
	"time"
)

// cpCmd represents the cp command
var cpCmd = &cobra.Command{
	Use:   "cp",
	Short: "put 3 bject with N retry commmand",
	Run: func(cmd *cobra.Command, args []string) {

		err := execute()
		if err != nil {
			fmt.Println(err)
		}
	},
}

func execute() error {

	file, err := os.Open(in)
	if err != nil {
		return fmt.Errorf("err opening file: %s", err)
	}
	defer file.Close()

	cre := credentials.NewStaticCredentials(
		access,
		secret,
		"")

	svc := s3.New(session.New(), &aws.Config{
		Credentials: cre,
		Region:      aws.String(region),
	})

	err = retry.Retry(uint(count), time.Duration(interval)*time.Second, func() error {
		// return error once in a while
		_, err := svc.PutObject(&s3.PutObjectInput{
			Bucket: aws.String(bucket),
			Key:    aws.String(out),
			Body:   file,
		})
		return err
	})
	if err != nil {
		return fmt.Errorf("faild to retire the host: %s", err)
	}
	return nil
}

var count, interval int
var region, access, secret, bucket, in, out string

func init() {

	s3Cmd.AddCommand(cpCmd)

	cpCmd.Flags().StringVar(&region, "region", "ap-northeast-1", "s3 region")
	cpCmd.Flags().StringVar(&access, "access", "", "aws access key id")
	cpCmd.Flags().StringVar(&secret, "secret", "", "aws secret key id")

	cpCmd.Flags().IntVar(&count, "count", 3, "retry max count")
	cpCmd.Flags().IntVar(&interval, "interval", 1, "retry interval second")

	cpCmd.Flags().StringVar(&bucket, "bucket", "", "s3 bucket")
	cpCmd.Flags().StringVar(&in, "in", "", "input file name")
	cpCmd.Flags().StringVar(&out, "out", "", "output file")
}
<commit_msg>fix typo<commit_after>package cmd

import (
	"fmt"
	"github.com/Songmu/retry"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/spf13/cobra"
	"os"
	"time"
)

// cpCmd represents the cp command
var cpCmd = &cobra.Command{
	Use:   "cp",
	Short: "put s3 object with N retry commmand",
	Run: func(cmd *cobra.Command, args []string) {

		err := execute()
		if err != nil {
			fmt.Println(err)
		}
	},
}

func execute() error {

	file, err := os.Open(in)
	if err != nil {
		return fmt.Errorf("err opening file: %s", err)
	}
	defer file.Close()

	cre := credentials.NewStaticCredentials(
		access,
		secret,
		"")

	svc := s3.New(session.New(), &aws.Config{
		Credentials: cre,
		Region:      aws.String(region),
	})

	err = retry.Retry(uint(count), time.Duration(interval)*time.Second, func() error {
		// return error once in a while
		_, err := svc.PutObject(&s3.PutObjectInput{
			Bucket: aws.String(bucket),
			Key:    aws.String(out),
			Body:   file,
		})
		return err
	})
	if err != nil {
		return fmt.Errorf("faild to retire the host: %s", err)
	}
	return nil
}

var count, interval int
var region, access, secret, bucket, in, out string

func init() {

	s3Cmd.AddCommand(cpCmd)

	cpCmd.Flags().StringVar(&region, "region", "ap-northeast-1", "s3 region")
	cpCmd.Flags().StringVar(&access, "access", "", "aws access key id")
	cpCmd.Flags().StringVar(&secret, "secret", "", "aws secret key id")

	cpCmd.Flags().IntVar(&count, "count", 3, "retry max count")
	cpCmd.Flags().IntVar(&interval, "interval", 1, "retry interval second")

	cpCmd.Flags().StringVar(&bucket, "bucket", "", "s3 bucket")
	cpCmd.Flags().StringVar(&in, "in", "", "input file name")
	cpCmd.Flags().StringVar(&out, "out", "", "output file")
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package fs

import (
	"container/list"
	"errors"
	"fmt"
	"log"
	"os"
	"path"
	"reflect"
	"sort"
	"time"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
)

const dirSeparator = '/'

// Implementation detail, do not touch.
//
// How long we cache the most recent listing for a particular directory from
// GCS before regarding it as stale.
//
// Intended to paper over performance issues caused by quick follow-up calls;
// for example when the fuse VFS performs a readdir followed quickly by a
// lookup for each child. The drawback is that this increases the time before a
// write by a foreign machine within a recently-listed directory will be seen
// locally.
//
// TODO(jacobsa): Set this according to real-world performance issues when the
// kernel does e.g. ReadDir followed by Lookup. Can probably be set quite
// small.
//
// TODO(jacobsa): Can this be moved to a decorator implementation of gcs.Bucket
// instead of living here?
var ListingCacheTTL = 10 * time.Second

// Implementation detail, do not touch.
//
// How long we remember that we took some action on the contents of a directory
// (linking or unlinking), and pretend the action is reflected in the listing
// even if it is not.
//
// Intended to paper over the fact that GCS doesn't offer list-your-own-writes
// consistency: it may be an arbitrarily long time before you see the creation
// or deletion of an object in a subsequent listing, and even if you see it in
// one listing you may not in the next. The drawback is that modifications to
// recently-modified directories by foreign machines will not be reflected
// locally for awhile.
//
// TODO(jacobsa): Set this according to information about listing staleness
// distributions from the GCS team.
//
// TODO(jacobsa): Can this be moved to a decorator implementation of gcs.Bucket
// instead of living here?
var ChildActionMemoryTTL = 5 * time.Minute

// See the childModifications field of dir.
type childModification struct {
	time time.Time
	name string
	node fusefs.Node
}

// A "directory" in GCS, defined by an object name prefix.
//
// For example, if the bucket contains objects "foo/bar" and "foo/baz", this
// implicitly defines the directory "foo/". No matter what the contents of the
// bucket, there is an implicit root directory "".
type dir struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger
	clock  timeutil.Clock
	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// INVARIANT: objectPrefix is "" (representing the root directory) or ends
	// with dirSeparator.
	objectPrefix string

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// Our current best understanding of the contents of the directory in GCS,
	// formed by listing the bucket and then patching according to child addition
	// and removal records at the time, and patched since then by subsequent
	// additions and removals.
	//
	// The time after which this should be generated anew from a new listing is
	// also stored. This is set to the time at which the listing completed plus
	// ListingCacheTTL.
	//
	// INVARIANT: contents != nil
	// INVARIANT: All nodes are of type *dir or *file.
	// INVARIANT: All nodes are indexed by names that agree with node contents.
	contents           map[string]fusefs.Node // GUARDED_BY(mu)
	contentsExpiration time.Time              // GUARDED_BY(mu)

	// A collection of children that have recently been added or removed locally
	// and the time at which it happened, ordered by the sequence in which it
	// happened. Elements M with M.node == nil are removals; all others are
	// additions.
	//
	// For a record M in this list with M's age less than ChildActionMemoryTTL,
	// any listing from the bucket should be augmented by pretending M just
	// happened.
	//
	// TODO(jacobsa): Make sure to test link followed by unlink, and unlink
	// followed by link.
	//
	// TODO(jacobsa): Make sure to test that these expire eventually, i.e. that
	// foreign overwrites, deletes, and recreates are reflected eventually.
	//
	// INVARIANT: All elements are of type childModification.
	// INVARIANT: Contains no duplicate names.
	// INVARIANT: For each M with M.node == nil, contents does not contain M.name.
	// INVARIANT: For each M with M.node != nil, contents[M.name] == M.node.
	childModifications list.List // GUARDED_BY(mu)

	// An index of childModifications by name.
	//
	// INVARIANT: childModificationsIndex != nil
	// INVARIANT: For all names N in the map, the indexed modification has name N.
	// INVARIANT: Contains exactly the set of names in childModifications.
	childModificationsIndex map[string]*list.Element // GUARDED_BY(mu)
}

// Make sure dir implements the interfaces we think it does.
var (
	// TODO(jacobsa): I think we want to embed fusefs.NdoeRef in all of our
	// fusefs.Node types, so that we better benefit from fusefs.Server node
	// caching.
	_ fusefs.Node               = &dir{}
	_ fusefs.NodeCreater        = &dir{}
	_ fusefs.NodeMknoder        = &dir{}
	_ fusefs.NodeStringLookuper = &dir{}

	_ fusefs.Handle             = &dir{}
	_ fusefs.HandleReadDirAller = &dir{}
)

func newDir(
	logger *log.Logger,
	clock timeutil.Clock,
	bucket gcs.Bucket,
	objectPrefix string) *dir {
	// Check the object prefix.
	if objectPrefix != "" && objectPrefix[len(objectPrefix)-1] != dirSeparator {
		panic("Unexpected object prefix: " + objectPrefix)
	}

	// Create the dir struct.
	d := &dir{
		logger:                  logger,
		clock:                   clock,
		bucket:                  bucket,
		objectPrefix:            objectPrefix,
		contents:                make(map[string]fusefs.Node),
		childModificationsIndex: make(map[string]*list.Element),
	}

	d.mu = syncutil.NewInvariantMutex(func() { d.checkInvariants() })

	return d
}

func (d *dir) checkInvariants() {
	// Check that maps are non-nil.
	if d.contents == nil || d.childModificationsIndex == nil {
		panic("Expected contents and childModificationsIndex to be non-nil.")
	}

	// Check each element of the contents map.
	for name, node := range d.contents {
		if node == nil {
			panic("nil node for name: " + name)
		}

		d, okDir := node.(*dir)
		f, okFile := node.(*file)
		if !okDir && !okFile {
			panic(fmt.Sprintf("Unexpected node type: %v", reflect.TypeOf(node)))
		}

		if okDir && name != path.Base(d.objectPrefix) {
			panic(fmt.Sprintf("Name mismatch: %s vs. %s", name, d.objectPrefix))
		}

		if okFile && name != path.Base(f.objectName) {
			panic(fmt.Sprintf("Name mismatch: %s vs. %s", name, f.objectName))
		}
	}

	// Check each child modification. Build a list of names we've seen while
	// doing so.
	var childModificationsNames sort.StringSlice
	for e := d.childModifications.Front(); e != nil; e = e.Next() {
		m := e.Value.(childModification)
		childModificationsNames = append(childModificationsNames, m.name)

		if m.node == nil {
			if n, ok := d.contents[m.name]; ok {
				panic(fmt.Sprintf("d.contents[%s] == %v for removal", m.name, n))
			}
		} else {
			if n := d.contents[m.name]; n != m.node {
				panic(fmt.Sprintf("d.contents[%s] == %v, not %v", m.name, n, m.node))
			}
		}
	}

	// Check that there were no duplicate names.
	sort.Sort(childModificationsNames)
	for i, name := range childModificationsNames {
		if i == 0 {
			continue
		}

		if name == childModificationsNames[i-1] {
			panic("Duplicated name in childModifications: " + name)
		}
	}

	panic("TODO")
}

// Ensure that d.contents is fresh and usable. Must be called before using
// d.contents.
//
// TODO(jacobsa): If contents hasn't expired, return immediately. Otherwise
// list parasitically while holding the lock (why not, we can make this more
// subtle later if we must) and modify the result appropriately.
//
// EXCLUSIVE_LOCKS_REQUIRED(d.mu)
func (d *dir) ensureContents(ctx context.Context) error

func (d *dir) Attr() fuse.Attr {
	return fuse.Attr{
		// TODO(jacobsa): Reflect that we allow writes now. Make sure to test.
		// TODO(jacobsa): Expose ACLs from GCS?
		Mode: os.ModeDir | 0500,
	}
}

// LOCKS_EXCLUDED(d.mu)
func (d *dir) ReadDirAll(ctx context.Context) (ents []fuse.Dirent, err error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.logger.Printf("ReadDirAll: [%s]/%s", d.bucket.Name(), d.objectPrefix)

	// Ensure that we can use d.contents.
	if err = d.ensureContents(ctx); err != nil {
		err = fmt.Errorf("d.ensureContents: %v", err)
		return
	}

	// Read out entries from the contents map.
	for name, node := range d.contents {
		ent := fuse.Dirent{
			Name: name,
		}

		if _, ok := node.(*dir); ok {
			ent.Type = fuse.DT_Dir
		} else {
			ent.Type = fuse.DT_File
		}

		ents = append(ents, ent)
	}

	return
}

// LOCKS_EXCLUDED(d.mu)
func (d *dir) Lookup(
	ctx context.Context,
	name string) (n fusefs.Node, err error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.logger.Printf("Lookup: ([%s]/%s) %s", d.bucket.Name(), d.objectPrefix, name)

	// Ensure that we can use d.contents.
	if err = d.ensureContents(ctx); err != nil {
		err = fmt.Errorf("d.ensureContents: %v", err)
		return
	}

	// Find the object within the map.
	var ok bool
	if n, ok = d.contents[name]; ok {
		return
	}

	err = fuse.ENOENT
	return
}

func (d *dir) Create(
	ctx context.Context,
	req *fuse.CreateRequest,
	resp *fuse.CreateResponse) (
	fusefs.Node,
	fusefs.Handle,
	error) {
	// Tell fuse to use Mknod followed by Open, rather than re-implementing much
	// of Open here.
	return nil, nil, fuse.ENOSYS
}

func (d *dir) Mknod(
	ctx context.Context,
	req *fuse.MknodRequest) (fusefs.Node, error) {
	return nil, errors.New("TODO(jacobsa): Support Mknod.")
}
<commit_msg>Finished dir.checkInvariants.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package fs

import (
	"container/list"
	"errors"
	"fmt"
	"log"
	"os"
	"path"
	"reflect"
	"sort"
	"time"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
)

const dirSeparator = '/'

// Implementation detail, do not touch.
//
// How long we cache the most recent listing for a particular directory from
// GCS before regarding it as stale.
//
// Intended to paper over performance issues caused by quick follow-up calls;
// for example when the fuse VFS performs a readdir followed quickly by a
// lookup for each child. The drawback is that this increases the time before a
// write by a foreign machine within a recently-listed directory will be seen
// locally.
//
// TODO(jacobsa): Set this according to real-world performance issues when the
// kernel does e.g. ReadDir followed by Lookup. Can probably be set quite
// small.
//
// TODO(jacobsa): Can this be moved to a decorator implementation of gcs.Bucket
// instead of living here?
var ListingCacheTTL = 10 * time.Second

// Implementation detail, do not touch.
//
// How long we remember that we took some action on the contents of a directory
// (linking or unlinking), and pretend the action is reflected in the listing
// even if it is not.
//
// Intended to paper over the fact that GCS doesn't offer list-your-own-writes
// consistency: it may be an arbitrarily long time before you see the creation
// or deletion of an object in a subsequent listing, and even if you see it in
// one listing you may not in the next. The drawback is that modifications to
// recently-modified directories by foreign machines will not be reflected
// locally for awhile.
//
// TODO(jacobsa): Set this according to information about listing staleness
// distributions from the GCS team.
//
// TODO(jacobsa): Can this be moved to a decorator implementation of gcs.Bucket
// instead of living here?
var ChildActionMemoryTTL = 5 * time.Minute

// See the childModifications field of dir.
type childModification struct {
	time time.Time
	name string
	node fusefs.Node
}

// A "directory" in GCS, defined by an object name prefix.
//
// For example, if the bucket contains objects "foo/bar" and "foo/baz", this
// implicitly defines the directory "foo/". No matter what the contents of the
// bucket, there is an implicit root directory "".
type dir struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger
	clock  timeutil.Clock
	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// INVARIANT: objectPrefix is "" (representing the root directory) or ends
	// with dirSeparator.
	objectPrefix string

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// Our current best understanding of the contents of the directory in GCS,
	// formed by listing the bucket and then patching according to child addition
	// and removal records at the time, and patched since then by subsequent
	// additions and removals.
	//
	// The time after which this should be generated anew from a new listing is
	// also stored. This is set to the time at which the listing completed plus
	// ListingCacheTTL.
	//
	// INVARIANT: contents != nil
	// INVARIANT: All nodes are of type *dir or *file.
	// INVARIANT: All nodes are indexed by names that agree with node contents.
	contents           map[string]fusefs.Node // GUARDED_BY(mu)
	contentsExpiration time.Time              // GUARDED_BY(mu)

	// A collection of children that have recently been added or removed locally
	// and the time at which it happened, ordered by the sequence in which it
	// happened. Elements M with M.node == nil are removals; all others are
	// additions.
	//
	// For a record M in this list with M's age less than ChildActionMemoryTTL,
	// any listing from the bucket should be augmented by pretending M just
	// happened.
	//
	// TODO(jacobsa): Make sure to test link followed by unlink, and unlink
	// followed by link.
	//
	// TODO(jacobsa): Make sure to test that these expire eventually, i.e. that
	// foreign overwrites, deletes, and recreates are reflected eventually.
	//
	// INVARIANT: All elements are of type childModification.
	// INVARIANT: Contains no duplicate names.
	// INVARIANT: For each M with M.node == nil, contents does not contain M.name.
	// INVARIANT: For each M with M.node != nil, contents[M.name] == M.node.
	childModifications list.List // GUARDED_BY(mu)

	// An index of childModifications by name.
	//
	// INVARIANT: childModificationsIndex != nil
	// INVARIANT: For all names N in the map, the indexed modification has name N.
	// INVARIANT: Contains exactly the set of names in childModifications.
	childModificationsIndex map[string]*list.Element // GUARDED_BY(mu)
}

// Make sure dir implements the interfaces we think it does.
var (
	// TODO(jacobsa): I think we want to embed fusefs.NdoeRef in all of our
	// fusefs.Node types, so that we better benefit from fusefs.Server node
	// caching.
	_ fusefs.Node               = &dir{}
	_ fusefs.NodeCreater        = &dir{}
	_ fusefs.NodeMknoder        = &dir{}
	_ fusefs.NodeStringLookuper = &dir{}

	_ fusefs.Handle             = &dir{}
	_ fusefs.HandleReadDirAller = &dir{}
)

func newDir(
	logger *log.Logger,
	clock timeutil.Clock,
	bucket gcs.Bucket,
	objectPrefix string) *dir {
	// Check the object prefix.
	if objectPrefix != "" && objectPrefix[len(objectPrefix)-1] != dirSeparator {
		panic("Unexpected object prefix: " + objectPrefix)
	}

	// Create the dir struct.
	d := &dir{
		logger:                  logger,
		clock:                   clock,
		bucket:                  bucket,
		objectPrefix:            objectPrefix,
		contents:                make(map[string]fusefs.Node),
		childModificationsIndex: make(map[string]*list.Element),
	}

	d.mu = syncutil.NewInvariantMutex(func() { d.checkInvariants() })

	return d
}

func (d *dir) checkInvariants() {
	// Check that maps are non-nil.
	if d.contents == nil || d.childModificationsIndex == nil {
		panic("Expected contents and childModificationsIndex to be non-nil.")
	}

	// Check each element of the contents map.
	for name, node := range d.contents {
		if node == nil {
			panic("nil node for name: " + name)
		}

		d, okDir := node.(*dir)
		f, okFile := node.(*file)
		if !okDir && !okFile {
			panic(fmt.Sprintf("Unexpected node type: %v", reflect.TypeOf(node)))
		}

		if okDir && name != path.Base(d.objectPrefix) {
			panic(fmt.Sprintf("Name mismatch: %s vs. %s", name, d.objectPrefix))
		}

		if okFile && name != path.Base(f.objectName) {
			panic(fmt.Sprintf("Name mismatch: %s vs. %s", name, f.objectName))
		}
	}

	// Check each child modification. Build a list of names we've seen while
	// doing so.
	var listNames sort.StringSlice
	for e := d.childModifications.Front(); e != nil; e = e.Next() {
		m := e.Value.(childModification)
		listNames = append(listNames, m.name)

		if m.node == nil {
			if n, ok := d.contents[m.name]; ok {
				panic(fmt.Sprintf("d.contents[%s] == %v for removal", m.name, n))
			}
		} else {
			if n := d.contents[m.name]; n != m.node {
				panic(fmt.Sprintf("d.contents[%s] == %v, not %v", m.name, n, m.node))
			}
		}
	}

	sort.Sort(listNames)

	// Check that there were no duplicate names.
	for i, name := range listNames {
		if i == 0 {
			continue
		}

		if name == listNames[i-1] {
			panic("Duplicated name in childModifications: " + name)
		}
	}

	// Check the index. Build a list of names it contains While doing so.
	var indexNames sort.StringSlice
	for name, e := range d.childModificationsIndex {
		indexNames = append(indexNames, name)

		m := e.Value.(childModification)
		if m.name != name {
			panic(fmt.Sprintf("Index name mismatch: %s vs. %s", m.name, name))
		}
	}

	sort.Sort(indexNames)

	// Check that the index contains the same set of names.
	if !reflect.DeepEqual(listNames, indexNames) {
		panic(fmt.Sprintf("Names mismatch:\n%v\n%v", listNames, indexNames))
	}
}

// Ensure that d.contents is fresh and usable. Must be called before using
// d.contents.
//
// TODO(jacobsa): If contents hasn't expired, return immediately. Otherwise
// list parasitically while holding the lock (why not, we can make this more
// subtle later if we must) and modify the result appropriately.
//
// EXCLUSIVE_LOCKS_REQUIRED(d.mu)
func (d *dir) ensureContents(ctx context.Context) error

func (d *dir) Attr() fuse.Attr {
	return fuse.Attr{
		// TODO(jacobsa): Reflect that we allow writes now. Make sure to test.
		// TODO(jacobsa): Expose ACLs from GCS?
		Mode: os.ModeDir | 0500,
	}
}

// LOCKS_EXCLUDED(d.mu)
func (d *dir) ReadDirAll(ctx context.Context) (ents []fuse.Dirent, err error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.logger.Printf("ReadDirAll: [%s]/%s", d.bucket.Name(), d.objectPrefix)

	// Ensure that we can use d.contents.
	if err = d.ensureContents(ctx); err != nil {
		err = fmt.Errorf("d.ensureContents: %v", err)
		return
	}

	// Read out entries from the contents map.
	for name, node := range d.contents {
		ent := fuse.Dirent{
			Name: name,
		}

		if _, ok := node.(*dir); ok {
			ent.Type = fuse.DT_Dir
		} else {
			ent.Type = fuse.DT_File
		}

		ents = append(ents, ent)
	}

	return
}

// LOCKS_EXCLUDED(d.mu)
func (d *dir) Lookup(
	ctx context.Context,
	name string) (n fusefs.Node, err error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.logger.Printf("Lookup: ([%s]/%s) %s", d.bucket.Name(), d.objectPrefix, name)

	// Ensure that we can use d.contents.
	if err = d.ensureContents(ctx); err != nil {
		err = fmt.Errorf("d.ensureContents: %v", err)
		return
	}

	// Find the object within the map.
	var ok bool
	if n, ok = d.contents[name]; ok {
		return
	}

	err = fuse.ENOENT
	return
}

func (d *dir) Create(
	ctx context.Context,
	req *fuse.CreateRequest,
	resp *fuse.CreateResponse) (
	fusefs.Node,
	fusefs.Handle,
	error) {
	// Tell fuse to use Mknod followed by Open, rather than re-implementing much
	// of Open here.
	return nil, nil, fuse.ENOSYS
}

func (d *dir) Mknod(
	ctx context.Context,
	req *fuse.MknodRequest) (fusefs.Node, error) {
	return nil, errors.New("TODO(jacobsa): Support Mknod.")
}
<|endoftext|>


<commit_before>package instances

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/cozy/cozy-stack/model/instance/lifecycle"
	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/labstack/echo/v4"
)

// contentMismatchFixer fixes the 64k bug
func contentMismatchFixer(c echo.Context) error {
	corruptedSuffix := "-corrupted"

	domain := c.Param("domain")
	inst, err := lifecycle.GetInstance(domain)
	if err != nil {
		return fmt.Errorf("Cannot find instance %s", domain)
	}

	body := struct {
		DryRun bool `json:"dry_run"`
	}{
		DryRun: true,
	}

	// Try to get the dru_run param from the body. If there is no body, ignore
	// it
	_ = json.NewDecoder(c.Request().Body).Decode(&body)

	buf := new(bytes.Buffer)
	encoder := json.NewEncoder(buf)

	logCh := make(chan *vfs.FsckLog)
	go func() {
		fs := inst.VFS()
		err = fs.Fsck(func(log *vfs.FsckLog) { logCh <- log })
		close(logCh)
	}()

	for log := range logCh {
		if !log.IsFile && !log.IsVersion && log.DirDoc != nil {
			log.DirDoc.DirsChildren = nil
			log.DirDoc.FilesChildren = nil
		}
		if errenc := encoder.Encode(log); errenc != nil {
			return errenc
		}
	}

	var content map[string]interface{}

	type resEntry struct {
		FilePath  string `json:"filepath"`
		ID        string `json:"id"`
		CreatedAt string `json:"created_at"`
		UpdatedAt string `json:"updated_at"`
	}
	resStruct := struct {
		DryRun  bool       `json:"dry_run"`
		Updated []resEntry `json:"updated"`
		Removed []resEntry `json:"removed"`
		Domain  string     `json:"domain"`
	}{
		Domain:  domain,
		DryRun:  body.DryRun,
		Removed: []resEntry{},
		Updated: []resEntry{},
	}

	scanner := bufio.NewScanner(buf)
	for scanner.Scan() {
		err = json.NewDecoder(bytes.NewReader(scanner.Bytes())).Decode(&content)
		if err != nil {
			return err
		}

		// Filtering the 64kb mismatch issue
		if content["type"] != "content_mismatch" {
			continue
		}

		contentMismatch := struct {
			SizeIndex int64 `json:"size_index"`
			SizeFile  int64 `json:"size_file"`
		}{}
		marshaled, _ := json.Marshal(content["content_mismatch"])
		err = json.Unmarshal(marshaled, &contentMismatch)
		if err != nil {
			return err
		}

		// SizeFile should be a multiple of 64k shorter than SizeIndex
		size := int64(64 * 1024)

		isSmallFile := contentMismatch.SizeIndex <= size && contentMismatch.SizeFile == 0
		isMultiple64 := (contentMismatch.SizeIndex-contentMismatch.SizeFile)%size == 0
		if !isMultiple64 && !isSmallFile {
			continue
		}

		// Removes/update
		fileDoc := content["file_doc"].(map[string]interface{})

		doc := &vfs.FileDoc{}
		err = couchdb.GetDoc(inst, consts.Files, fileDoc["_id"].(string), doc)
		if err != nil {
			return err
		}
		instanceVFS := inst.VFS()

		// Checks if the file is trashed
		if fileDoc["restore_path"] != nil {
			// This is a trashed file, just delete it
			resStruct.Removed = append(resStruct.Removed, resEntry{
				ID:        fileDoc["_id"].(string),
				FilePath:  fileDoc["path"].(string),
				CreatedAt: doc.CreatedAt.String(),
				UpdatedAt: doc.UpdatedAt.String(),
			})
			if !body.DryRun {
				err := instanceVFS.DestroyFile(doc)
				if err != nil {
					return err
				}
			}
			continue
		}

		// Fixing :
		// - Appending a corrupted suffix to the file
		// - Force the file index size to the real file size
		newFileDoc := doc.Clone().(*vfs.FileDoc)

		newFileDoc.DocName = doc.DocName + corruptedSuffix
		newFileDoc.ByteSize = contentMismatch.SizeFile

		resStruct.Updated = append(resStruct.Updated, resEntry{
			ID:        fileDoc["_id"].(string),
			FilePath:  fileDoc["path"].(string),
			CreatedAt: doc.CreatedAt.String(),
			UpdatedAt: doc.UpdatedAt.String(),
		})
		if !body.DryRun {
			// Let the UpdateFileDoc handles the file doc update. For swift
			// layout V1, the file should also be renamed
			err := instanceVFS.UpdateFileDoc(doc, newFileDoc)
			if err != nil {
				return err
			}
		}
	}

	return c.JSON(http.StatusOK, resStruct)
}
<commit_msg>Split the content mismatch function<commit_after>package instances

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"

	"github.com/cozy/cozy-stack/model/instance"
	"github.com/cozy/cozy-stack/model/instance/lifecycle"
	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/labstack/echo/v4"
)

type mismatchStruct struct {
	SizeIndex int64 `json:"size_index"`
	SizeFile  int64 `json:"size_file"`
}

// resEntry contains an out entry of a 64k content mismatch
type resEntry struct {
	FilePath  string `json:"filepath"`
	ID        string `json:"id"`
	CreatedAt string `json:"created_at"`
	UpdatedAt string `json:"updated_at"`
}

type resStruct struct {
	DryRun  bool       `json:"dry_run"`
	Updated []resEntry `json:"updated"`
	Removed []resEntry `json:"removed"`
	Domain  string     `json:"domain"`
}

// contentMismatchFixer fixes the 64k bug
func contentMismatchFixer(c echo.Context) error {
	domain := c.Param("domain")
	inst, err := lifecycle.GetInstance(domain)
	if err != nil {
		return fmt.Errorf("Cannot find instance %s", domain)
	}

	body := struct {
		DryRun bool `json:"dry_run"`
	}{
		DryRun: true,
	}

	// Try to get the dru_run param from the body. If there is no body, ignore
	// it
	_ = json.NewDecoder(c.Request().Body).Decode(&body)

	// Get the FSCK data from the instance
	buf, err := getFSCK(inst)
	if err != nil {
		return err
	}

	var content map[string]interface{}
	res := &resStruct{
		Domain:  domain,
		DryRun:  body.DryRun,
		Removed: []resEntry{},
		Updated: []resEntry{},
	}

	scanner := bufio.NewScanner(buf)
	for scanner.Scan() {
		err = json.NewDecoder(bytes.NewReader(scanner.Bytes())).Decode(&content)
		if err != nil {
			return err
		}

		// Filtering the 64kb mismatch issue
		if content["type"] != "content_mismatch" {
			continue
		}

		// Prepare the struct & ensure the data should be fixed
		contentMismatch, err := prepareMismatchStruct(content)
		if err != nil {
			return err
		}
		if !is64ContentMismatch(contentMismatch) {
			continue
		}

		// Finally, fixing the file
		err = fixFile(content, contentMismatch, inst, res, body.DryRun)
		if err != nil {
			return err
		}
	}

	return c.JSON(http.StatusOK, res)
}

func getFSCK(inst *instance.Instance) (io.Reader, error) {
	buf := new(bytes.Buffer)
	encoder := json.NewEncoder(buf)

	logCh := make(chan *vfs.FsckLog)
	go func() {
		fs := inst.VFS()
		_ = fs.Fsck(func(log *vfs.FsckLog) { logCh <- log })
		close(logCh)
	}()

	for log := range logCh {
		if !log.IsFile && !log.IsVersion && log.DirDoc != nil {
			log.DirDoc.DirsChildren = nil
			log.DirDoc.FilesChildren = nil
		}
		if errenc := encoder.Encode(log); errenc != nil {
			return nil, errenc
		}
	}

	return buf, nil
}

func prepareMismatchStruct(content map[string]interface{}) (*mismatchStruct, error) {
	contentMismatch := &mismatchStruct{}
	marshaled, _ := json.Marshal(content["content_mismatch"])
	if err := json.Unmarshal(marshaled, &contentMismatch); err != nil {
		return nil, err
	}

	return contentMismatch, nil
}

// is64ContentMismatch ensures we are treating a 64k content mismatch
func is64ContentMismatch(contentMismatch *mismatchStruct) bool {
	// SizeFile should be a multiple of 64k shorter than SizeIndex
	size := int64(64 * 1024)

	isSmallFile := contentMismatch.SizeIndex <= size && contentMismatch.SizeFile == 0
	isMultiple64 := (contentMismatch.SizeIndex-contentMismatch.SizeFile)%size == 0

	return isMultiple64 || isSmallFile
}

// fixFile fixes a content-mismatch file
// Trashed:
// - Removes it if the file
// Not Trashed:
// - Appending a corrupted suffix to the file
// - Force the file index size to the real file size
func fixFile(content map[string]interface{}, contentMismatch *mismatchStruct, inst *instance.Instance, res *resStruct, dryRun bool) error {
	corruptedSuffix := "-corrupted"

	// Removes/update
	fileDoc := content["file_doc"].(map[string]interface{})

	doc := &vfs.FileDoc{}
	err := couchdb.GetDoc(inst, consts.Files, fileDoc["_id"].(string), doc)
	if err != nil {
		return err
	}
	instanceVFS := inst.VFS()

	// File is trashed
	if fileDoc["restore_path"] != nil {
		// This is a trashed file, just delete it
		res.Removed = append(res.Removed, resEntry{
			ID:        fileDoc["_id"].(string),
			FilePath:  fileDoc["path"].(string),
			CreatedAt: doc.CreatedAt.String(),
			UpdatedAt: doc.UpdatedAt.String(),
		})

		if !dryRun {
			return instanceVFS.DestroyFile(doc)
		}
		return nil
	}

	// File is not trashed, updating it
	newFileDoc := doc.Clone().(*vfs.FileDoc)

	newFileDoc.DocName = doc.DocName + corruptedSuffix
	newFileDoc.ByteSize = contentMismatch.SizeFile

	res.Updated = append(res.Updated, resEntry{
		ID:        fileDoc["_id"].(string),
		FilePath:  fileDoc["path"].(string),
		CreatedAt: doc.CreatedAt.String(),
		UpdatedAt: doc.UpdatedAt.String(),
	})
	if !dryRun {
		// Let the UpdateFileDoc handles the file doc update. For swift
		// layout V1, the file should also be renamed
		return instanceVFS.UpdateFileDoc(doc, newFileDoc)
	}

	return nil
}
<|endoftext|>


<commit_before>package levigo

import (
	"bytes"
	"fmt"
	"math/rand"
	"os"
	"path/filepath"
	"testing"
	"time"
)

func init() {
	rand.Seed(int64(time.Now().Nanosecond()))
}

// This testcase is a port of leveldb's c_test.c.
func TestC(t *testing.T) {
	if GetLevelDBMajorVersion() <= 0 {
		t.Errorf("Major version cannot be less than zero")
	}

	dbname := tempDir(t)
	defer deleteDBDirectory(t, dbname)
	env := NewDefaultEnv()
	cache := NewLRUCache(1 << 20)

	options := NewOptions()
	// options.SetComparator(cmp)
	options.SetErrorIfExists(true)
	options.SetCache(cache)
	options.SetEnv(env)
	options.SetInfoLog(nil)
	options.SetWriteBufferSize(1 << 20)
	options.SetParanoidChecks(true)
	options.SetMaxOpenFiles(10)
	options.SetBlockSize(1024)
	options.SetBlockRestartInterval(8)
	options.SetCompression(NoCompression)

	roptions := NewReadOptions()
	roptions.SetVerifyChecksums(true)
	roptions.SetFillCache(false)

	woptions := NewWriteOptions()
	woptions.SetSync(true)

	_ = DestroyDatabase(dbname, options)

	db, err := Open(dbname, options)
	if err == nil {
		t.Errorf("Open on missing db should have failed")
	}

	options.SetCreateIfMissing(true)
	db, err = Open(dbname, options)
	if err != nil {
		t.Fatalf("Open failed: %v", err)
	}

	putKey := []byte("foo")
	putValue := []byte("hello")
	err = db.Put(woptions, putKey, putValue)
	if err != nil {
		t.Errorf("Put failed: %v", err)
	}

	CheckGet(t, "after Put", db, roptions, putKey, putValue)

	wb := NewWriteBatch()
	wb.Put([]byte("foo"), []byte("a"))
	wb.Clear()
	wb.Put([]byte("bar"), []byte("b"))
	wb.Put([]byte("box"), []byte("c"))
	wb.Delete([]byte("bar"))
	err = db.Write(woptions, wb)
	if err != nil {
		t.Errorf("Write batch failed: %v", err)
	}
	CheckGet(t, "after WriteBatch", db, roptions, []byte("foo"), []byte("hello"))
	CheckGet(t, "after WriteBatch", db, roptions, []byte("bar"), nil)
	CheckGet(t, "after WriteBatch", db, roptions, []byte("box"), []byte("c"))
	// TODO: WriteBatch iteration isn't easy. Suffers same problems as
	// Comparator.
	// wbiter := &TestWBIter{t: t}
	// wb.Iterate(wbiter)
	// if wbiter.pos != 3 {
	// 	t.Errorf("After Iterate, on the wrong pos: %d", wbiter.pos)
	// }
	wb.Close()

	iter := db.NewIterator(roptions)
	if iter.Valid() {
		t.Errorf("Read iterator should not be valid, yet")
	}
	iter.SeekToFirst()
	if !iter.Valid() {
		t.Errorf("Read iterator should be valid after seeking to first record")
	}
	CheckIter(t, iter, []byte("box"), []byte("c"))
	iter.Next()
	CheckIter(t, iter, []byte("foo"), []byte("hello"))
	iter.Prev()
	CheckIter(t, iter, []byte("box"), []byte("c"))
	iter.Prev()
	if iter.Valid() {
		t.Errorf("Read iterator should not be valid after go back past the first record")
	}
	iter.SeekToLast()
	CheckIter(t, iter, []byte("foo"), []byte("hello"))
	iter.Seek([]byte("b"))
	CheckIter(t, iter, []byte("box"), []byte("c"))
	if iter.GetError() != nil {
		t.Errorf("Read iterator has an error we didn't expect: %v", iter.GetError())
	}
	iter.Close()

	// approximate sizes
	n := 20000
	woptions.SetSync(false)
	for i := 0; i < n; i++ {
		keybuf := []byte(fmt.Sprintf("k%020d", i))
		valbuf := []byte(fmt.Sprintf("v%020d", i))
		err := db.Put(woptions, keybuf, valbuf)
		if err != nil {
			t.Errorf("Put error in approximate size test: %v", err)
		}
	}

	ranges := []Range{
		{[]byte("a"), []byte("k00000000000000010000")},
		{[]byte("k00000000000000010000"), []byte("z")},
	}
	sizes := db.GetApproximateSizes(ranges)
	if len(sizes) == 2 {
		if sizes[0] <= 0 {
			t.Errorf("First size range was %d", sizes[0])
		}
		if sizes[1] <= 0 {
			t.Errorf("Second size range was %d", sizes[1])
		}
	} else {
		t.Errorf("Expected 2 approx. sizes back, got %d", len(sizes))
	}

	// property
	prop := db.PropertyValue("nosuchprop")
	if prop != "" {
		t.Errorf("property nosuchprop should not have a value")
	}
	prop = db.PropertyValue("leveldb.stats")
	if prop == "" {
		t.Errorf("property leveldb.stats should have a value")
	}

	// snapshot
	snap := db.NewSnapshot()
	err = db.Delete(woptions, []byte("foo"))
	if err != nil {
		t.Errorf("Delete during snapshot test errored: %v", err)
	}
	roptions.SetSnapshot(snap)
	CheckGet(t, "from snapshot", db, roptions, []byte("foo"), []byte("hello"))
	roptions.SetSnapshot(nil)
	CheckGet(t, "from snapshot", db, roptions, []byte("foo"), nil)
	db.ReleaseSnapshot(snap)

	// repair
	db.Close()
	options.SetCreateIfMissing(false)
	options.SetErrorIfExists(false)
	err = RepairDatabase(dbname, options)
	if err != nil {
		t.Errorf("Repairing db failed: %v", err)
	}
	db, err = Open(dbname, options)
	if err != nil {
		t.Errorf("Unable to open repaired db: %v", err)
	}
	CheckGet(t, "repair", db, roptions, []byte("foo"), nil)
	CheckGet(t, "repair", db, roptions, []byte("bar"), nil)
	CheckGet(t, "repair", db, roptions, []byte("box"), []byte("c"))
	options.SetCreateIfMissing(true)
	options.SetErrorIfExists(true)

	// filter
	policy := NewBloomFilter(10)
	db.Close()
	DestroyDatabase(dbname, options)
	options.SetFilterPolicy(policy)
	db, err = Open(dbname, options)
	if err != nil {
		t.Fatalf("Unable to recreate db for filter tests: %v", err)
	}
	err = db.Put(woptions, []byte("foo"), []byte("foovalue"))
	if err != nil {
		t.Errorf("Unable to put 'foo' with filter: %v", err)
	}
	err = db.Put(woptions, []byte("bar"), []byte("barvalue"))
	if err != nil {
		t.Errorf("Unable to put 'bar' with filter: %v", err)
	}
	db.CompactRange(Range{nil, nil})
	CheckGet(t, "filter", db, roptions, []byte("foo"), []byte("foovalue"))
	CheckGet(t, "filter", db, roptions, []byte("bar"), []byte("barvalue"))
	options.SetFilterPolicy(nil)
	policy.Close()

	// cleanup
	db.Close()
	options.Close()
	roptions.Close()
	woptions.Close()
	cache.Close()
	// DestroyComparator(cmp)
	env.Close()
}

func TestNilSlicesInDb(t *testing.T) {
	dbname := tempDir(t)
	defer deleteDBDirectory(t, dbname)
	options := NewOptions()
	options.SetErrorIfExists(true)
	options.SetCreateIfMissing(true)
	ro := NewReadOptions()
	_ = DestroyDatabase(dbname, options)
	db, err := Open(dbname, options)
	if err != nil {
		t.Fatalf("Database could not be opened: %v", err)
	}
	defer db.Close()
	val, err := db.Get(ro, []byte("missing"))
	if err != nil {
		t.Errorf("Get failed: %v", err)
	}
	if val != nil {
		t.Errorf("A key not in the db should return nil, not %v", val)
	}
	wo := NewWriteOptions()
	db.Put(wo, nil, []byte("love"))
	val, err = db.Get(ro, nil)
	if !bytes.Equal([]byte("love"), val) {
		t.Errorf("Get should see the nil key: %v", val)
	}
	val, err = db.Get(ro, []byte{})
	if !bytes.Equal([]byte("love"), val) {
		t.Errorf("Get shouldn't distinguish between nil key and empty slice key: %v", val)
	}

	err = db.Put(wo, []byte("nilvalue"), nil)
	if err != nil {
		t.Errorf("nil value Put errored: %v", err)
	}
	// Compare with the []byte("missing") case. We expect Get to return a
	// []byte{} here, but expect a nil returned there.
	CheckGet(t, "nil value Put", db, ro, []byte("nilvalue"), []byte{})

	err = db.Put(wo, []byte("emptyvalue"), []byte{})
	if err != nil {
		t.Errorf("empty value Put errored: %v", err)
	}
	CheckGet(t, "empty value Put", db, ro, []byte("emptyvalue"), []byte{})

	err = db.Delete(wo, nil)
	if err != nil {
		t.Errorf("nil key Delete errored: %v", err)
	}
	err = db.Delete(wo, []byte{})
	if err != nil {
		t.Errorf("empty slice key Delete errored: %v", err)
	}

}

func TestIterationValidityLimits(t *testing.T) {
	dbname := tempDir(t)
	defer deleteDBDirectory(t, dbname)
	options := NewOptions()
	options.SetErrorIfExists(true)
	options.SetCreateIfMissing(true)
	ro := NewReadOptions()
	wo := NewWriteOptions()
	_ = DestroyDatabase(dbname, options)
	db, err := Open(dbname, options)
	if err != nil {
		t.Fatalf("Database could not be opened: %v", err)
	}
	defer db.Close()
	db.Put(wo, []byte("bat"), []byte("somedata"))
	db.Put(wo, []byte("done"), []byte("somedata"))
	it := db.NewIterator(ro)
	defer it.Close()
	if it.Valid() {
		t.Errorf("new Iterator was valid")
	}
	it.Seek([]byte("bat"))
	if !it.Valid() {
		t.Errorf("Seek to %#v failed.", []byte("bat"))
	}
	if !bytes.Equal([]byte("bat"), it.Key()) {
		t.Errorf("did not seek to []byte(\"bat\")")
	}
	key := it.Key()
	it.Next()
	if bytes.Equal(key, it.Key()) {
		t.Errorf("key should be a copy of last key")
	}
	it.Next()
	if it.Valid() {
		t.Errorf("iterating off the db should result in an invalid iterator")
	}
	err = it.GetError()
	if err != nil {
		t.Errorf("should not have seen an error on an invalid iterator")
	}
	it.Seek([]byte("bat"))
	if !it.Valid() {
		t.Errorf("Iterator should be valid again")
	}
}

func CheckGet(t *testing.T, where string, db *DB, roptions *ReadOptions, key, expected []byte) {
	getValue, err := db.Get(roptions, key)

	if err != nil {
		t.Errorf("%s, Get failed: %v", where, err)
	}
	if !bytes.Equal(getValue, expected) {
		t.Errorf("%s, expected Get value %v, got %v", where, expected, getValue)
	}
}

func WBIterCheckEqual(t *testing.T, where string, which string, pos int, expected, given []byte) {
	if !bytes.Equal(expected, given) {
		t.Errorf("%s at pos %d, %s expected: %v, got: %v", where, pos, which, expected, given)
	}
}

func CheckIter(t *testing.T, it *Iterator, key, value []byte) {
	if !bytes.Equal(key, it.Key()) {
		t.Errorf("Iterator: expected key %v, got %v", key, it.Key())
	}
	if !bytes.Equal(value, it.Value()) {
		t.Errorf("Iterator: expected value %v, got %v", value, it.Value())
	}
}

func deleteDBDirectory(t *testing.T, dirPath string) {
	err := os.RemoveAll(dirPath)
	if err != nil {
		t.Errorf("Unable to remove database directory: %s", dirPath)
	}
}

func tempDir(t *testing.T) string {
	bottom := fmt.Sprintf("levigo-test-%d", rand.Int())
	path := filepath.Join(os.TempDir(), bottom)
	deleteDBDirectory(t, path)
	return path
}
<commit_msg>sync writes for accurate tests<commit_after>package levigo

import (
	"bytes"
	"fmt"
	"math/rand"
	"os"
	"path/filepath"
	"testing"
	"time"
)

func init() {
	rand.Seed(int64(time.Now().Nanosecond()))
}

// This testcase is a port of leveldb's c_test.c.
func TestC(t *testing.T) {
	if GetLevelDBMajorVersion() <= 0 {
		t.Errorf("Major version cannot be less than zero")
	}

	dbname := tempDir(t)
	defer deleteDBDirectory(t, dbname)
	env := NewDefaultEnv()
	cache := NewLRUCache(1 << 20)

	options := NewOptions()
	// options.SetComparator(cmp)
	options.SetErrorIfExists(true)
	options.SetCache(cache)
	options.SetEnv(env)
	options.SetInfoLog(nil)
	options.SetWriteBufferSize(1 << 20)
	options.SetParanoidChecks(true)
	options.SetMaxOpenFiles(10)
	options.SetBlockSize(1024)
	options.SetBlockRestartInterval(8)
	options.SetCompression(NoCompression)

	roptions := NewReadOptions()
	roptions.SetVerifyChecksums(true)
	roptions.SetFillCache(false)

	woptions := NewWriteOptions()
	woptions.SetSync(true)

	_ = DestroyDatabase(dbname, options)

	db, err := Open(dbname, options)
	if err == nil {
		t.Errorf("Open on missing db should have failed")
	}

	options.SetCreateIfMissing(true)
	db, err = Open(dbname, options)
	if err != nil {
		t.Fatalf("Open failed: %v", err)
	}

	putKey := []byte("foo")
	putValue := []byte("hello")
	err = db.Put(woptions, putKey, putValue)
	if err != nil {
		t.Errorf("Put failed: %v", err)
	}

	CheckGet(t, "after Put", db, roptions, putKey, putValue)

	wb := NewWriteBatch()
	wb.Put([]byte("foo"), []byte("a"))
	wb.Clear()
	wb.Put([]byte("bar"), []byte("b"))
	wb.Put([]byte("box"), []byte("c"))
	wb.Delete([]byte("bar"))
	err = db.Write(woptions, wb)
	if err != nil {
		t.Errorf("Write batch failed: %v", err)
	}
	CheckGet(t, "after WriteBatch", db, roptions, []byte("foo"), []byte("hello"))
	CheckGet(t, "after WriteBatch", db, roptions, []byte("bar"), nil)
	CheckGet(t, "after WriteBatch", db, roptions, []byte("box"), []byte("c"))
	// TODO: WriteBatch iteration isn't easy. Suffers same problems as
	// Comparator.
	// wbiter := &TestWBIter{t: t}
	// wb.Iterate(wbiter)
	// if wbiter.pos != 3 {
	// 	t.Errorf("After Iterate, on the wrong pos: %d", wbiter.pos)
	// }
	wb.Close()

	iter := db.NewIterator(roptions)
	if iter.Valid() {
		t.Errorf("Read iterator should not be valid, yet")
	}
	iter.SeekToFirst()
	if !iter.Valid() {
		t.Errorf("Read iterator should be valid after seeking to first record")
	}
	CheckIter(t, iter, []byte("box"), []byte("c"))
	iter.Next()
	CheckIter(t, iter, []byte("foo"), []byte("hello"))
	iter.Prev()
	CheckIter(t, iter, []byte("box"), []byte("c"))
	iter.Prev()
	if iter.Valid() {
		t.Errorf("Read iterator should not be valid after go back past the first record")
	}
	iter.SeekToLast()
	CheckIter(t, iter, []byte("foo"), []byte("hello"))
	iter.Seek([]byte("b"))
	CheckIter(t, iter, []byte("box"), []byte("c"))
	if iter.GetError() != nil {
		t.Errorf("Read iterator has an error we didn't expect: %v", iter.GetError())
	}
	iter.Close()

	// approximate sizes
	n := 20000
	for i := 0; i < n; i++ {
		keybuf := []byte(fmt.Sprintf("k%020d", i))
		valbuf := []byte(fmt.Sprintf("v%020d", i))
		err := db.Put(woptions, keybuf, valbuf)
		if err != nil {
			t.Errorf("Put error in approximate size test: %v", err)
		}
	}

	ranges := []Range{
		{[]byte("a"), []byte("k00000000000000010000")},
		{[]byte("k00000000000000010000"), []byte("z")},
	}
	sizes := db.GetApproximateSizes(ranges)
	if len(sizes) == 2 {
		if sizes[0] <= 0 {
			t.Errorf("First size range was %d", sizes[0])
		}
		if sizes[1] <= 0 {
			t.Errorf("Second size range was %d", sizes[1])
		}
	} else {
		t.Errorf("Expected 2 approx. sizes back, got %d", len(sizes))
	}

	// property
	prop := db.PropertyValue("nosuchprop")
	if prop != "" {
		t.Errorf("property nosuchprop should not have a value")
	}
	prop = db.PropertyValue("leveldb.stats")
	if prop == "" {
		t.Errorf("property leveldb.stats should have a value")
	}

	// snapshot
	snap := db.NewSnapshot()
	err = db.Delete(woptions, []byte("foo"))
	if err != nil {
		t.Errorf("Delete during snapshot test errored: %v", err)
	}
	roptions.SetSnapshot(snap)
	CheckGet(t, "from snapshot", db, roptions, []byte("foo"), []byte("hello"))
	roptions.SetSnapshot(nil)
	CheckGet(t, "from snapshot", db, roptions, []byte("foo"), nil)
	db.ReleaseSnapshot(snap)

	// repair
	db.Close()
	options.SetCreateIfMissing(false)
	options.SetErrorIfExists(false)
	err = RepairDatabase(dbname, options)
	if err != nil {
		t.Errorf("Repairing db failed: %v", err)
	}
	db, err = Open(dbname, options)
	if err != nil {
		t.Errorf("Unable to open repaired db: %v", err)
	}
	CheckGet(t, "repair", db, roptions, []byte("foo"), nil)
	CheckGet(t, "repair", db, roptions, []byte("bar"), nil)
	CheckGet(t, "repair", db, roptions, []byte("box"), []byte("c"))
	options.SetCreateIfMissing(true)
	options.SetErrorIfExists(true)

	// filter
	policy := NewBloomFilter(10)
	db.Close()
	DestroyDatabase(dbname, options)
	options.SetFilterPolicy(policy)
	db, err = Open(dbname, options)
	if err != nil {
		t.Fatalf("Unable to recreate db for filter tests: %v", err)
	}
	err = db.Put(woptions, []byte("foo"), []byte("foovalue"))
	if err != nil {
		t.Errorf("Unable to put 'foo' with filter: %v", err)
	}
	err = db.Put(woptions, []byte("bar"), []byte("barvalue"))
	if err != nil {
		t.Errorf("Unable to put 'bar' with filter: %v", err)
	}
	db.CompactRange(Range{nil, nil})
	CheckGet(t, "filter", db, roptions, []byte("foo"), []byte("foovalue"))
	CheckGet(t, "filter", db, roptions, []byte("bar"), []byte("barvalue"))
	options.SetFilterPolicy(nil)
	policy.Close()

	// cleanup
	db.Close()
	options.Close()
	roptions.Close()
	woptions.Close()
	cache.Close()
	// DestroyComparator(cmp)
	env.Close()
}

func TestNilSlicesInDb(t *testing.T) {
	dbname := tempDir(t)
	defer deleteDBDirectory(t, dbname)
	options := NewOptions()
	options.SetErrorIfExists(true)
	options.SetCreateIfMissing(true)
	ro := NewReadOptions()
	_ = DestroyDatabase(dbname, options)
	db, err := Open(dbname, options)
	if err != nil {
		t.Fatalf("Database could not be opened: %v", err)
	}
	defer db.Close()
	val, err := db.Get(ro, []byte("missing"))
	if err != nil {
		t.Errorf("Get failed: %v", err)
	}
	if val != nil {
		t.Errorf("A key not in the db should return nil, not %v", val)
	}
	wo := NewWriteOptions()
	db.Put(wo, nil, []byte("love"))
	val, err = db.Get(ro, nil)
	if !bytes.Equal([]byte("love"), val) {
		t.Errorf("Get should see the nil key: %v", val)
	}
	val, err = db.Get(ro, []byte{})
	if !bytes.Equal([]byte("love"), val) {
		t.Errorf("Get shouldn't distinguish between nil key and empty slice key: %v", val)
	}

	err = db.Put(wo, []byte("nilvalue"), nil)
	if err != nil {
		t.Errorf("nil value Put errored: %v", err)
	}
	// Compare with the []byte("missing") case. We expect Get to return a
	// []byte{} here, but expect a nil returned there.
	CheckGet(t, "nil value Put", db, ro, []byte("nilvalue"), []byte{})

	err = db.Put(wo, []byte("emptyvalue"), []byte{})
	if err != nil {
		t.Errorf("empty value Put errored: %v", err)
	}
	CheckGet(t, "empty value Put", db, ro, []byte("emptyvalue"), []byte{})

	err = db.Delete(wo, nil)
	if err != nil {
		t.Errorf("nil key Delete errored: %v", err)
	}
	err = db.Delete(wo, []byte{})
	if err != nil {
		t.Errorf("empty slice key Delete errored: %v", err)
	}

}

func TestIterationValidityLimits(t *testing.T) {
	dbname := tempDir(t)
	defer deleteDBDirectory(t, dbname)
	options := NewOptions()
	options.SetErrorIfExists(true)
	options.SetCreateIfMissing(true)
	ro := NewReadOptions()
	wo := NewWriteOptions()
	_ = DestroyDatabase(dbname, options)
	db, err := Open(dbname, options)
	if err != nil {
		t.Fatalf("Database could not be opened: %v", err)
	}
	defer db.Close()
	db.Put(wo, []byte("bat"), []byte("somedata"))
	db.Put(wo, []byte("done"), []byte("somedata"))
	it := db.NewIterator(ro)
	defer it.Close()
	if it.Valid() {
		t.Errorf("new Iterator was valid")
	}
	it.Seek([]byte("bat"))
	if !it.Valid() {
		t.Errorf("Seek to %#v failed.", []byte("bat"))
	}
	if !bytes.Equal([]byte("bat"), it.Key()) {
		t.Errorf("did not seek to []byte(\"bat\")")
	}
	key := it.Key()
	it.Next()
	if bytes.Equal(key, it.Key()) {
		t.Errorf("key should be a copy of last key")
	}
	it.Next()
	if it.Valid() {
		t.Errorf("iterating off the db should result in an invalid iterator")
	}
	err = it.GetError()
	if err != nil {
		t.Errorf("should not have seen an error on an invalid iterator")
	}
	it.Seek([]byte("bat"))
	if !it.Valid() {
		t.Errorf("Iterator should be valid again")
	}
}

func CheckGet(t *testing.T, where string, db *DB, roptions *ReadOptions, key, expected []byte) {
	getValue, err := db.Get(roptions, key)

	if err != nil {
		t.Errorf("%s, Get failed: %v", where, err)
	}
	if !bytes.Equal(getValue, expected) {
		t.Errorf("%s, expected Get value %v, got %v", where, expected, getValue)
	}
}

func WBIterCheckEqual(t *testing.T, where string, which string, pos int, expected, given []byte) {
	if !bytes.Equal(expected, given) {
		t.Errorf("%s at pos %d, %s expected: %v, got: %v", where, pos, which, expected, given)
	}
}

func CheckIter(t *testing.T, it *Iterator, key, value []byte) {
	if !bytes.Equal(key, it.Key()) {
		t.Errorf("Iterator: expected key %v, got %v", key, it.Key())
	}
	if !bytes.Equal(value, it.Value()) {
		t.Errorf("Iterator: expected value %v, got %v", value, it.Value())
	}
}

func deleteDBDirectory(t *testing.T, dirPath string) {
	err := os.RemoveAll(dirPath)
	if err != nil {
		t.Errorf("Unable to remove database directory: %s", dirPath)
	}
}

func tempDir(t *testing.T) string {
	bottom := fmt.Sprintf("levigo-test-%d", rand.Int())
	path := filepath.Join(os.TempDir(), bottom)
	deleteDBDirectory(t, path)
	return path
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"fmt"
	"reflect"
	"strings"
)

// Decide on the name of the given op.
func opName(op interface{}) string {
	// We expect all ops to be pointers.
	t := reflect.TypeOf(op).Elem()

	// Strip the "Op" from "FooOp".
	return strings.TrimSuffix(t.Name(), "Op")
}

func describeRequest(op interface{}) (s string) {
	v := reflect.ValueOf(op).Elem()

	// We will set up a comma-separated list of components.
	var components []string
	addComponent := func(format string, v ...interface{}) {
		components = append(components, fmt.Sprintf(format, v...))
	}

	// Include an inode number, if available.
	if f := v.FieldByName("Inode"); f.IsValid() {
		addComponent("inode %v", f.Interface())
	}

	// Handle special cases.
	switch typed := op.(type) {
	case *interruptOp:
		addComponent("fuseid 0x%08x", typed.FuseID)
	}

	// Use just the name if there is no extra info.
	if len(components) == 0 {
		return opName(op)
	}

	// Otherwise, include the extra info.
	return fmt.Sprintf("%s (%s)", opName(op), strings.Join(components, ", "))
}
<commit_msg>Added special cases for ReadFile and WriteFile.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"fmt"
	"reflect"
	"strings"

	"github.com/jacobsa/fuse/fuseops"
)

// Decide on the name of the given op.
func opName(op interface{}) string {
	// We expect all ops to be pointers.
	t := reflect.TypeOf(op).Elem()

	// Strip the "Op" from "FooOp".
	return strings.TrimSuffix(t.Name(), "Op")
}

func describeRequest(op interface{}) (s string) {
	v := reflect.ValueOf(op).Elem()

	// We will set up a comma-separated list of components.
	var components []string
	addComponent := func(format string, v ...interface{}) {
		components = append(components, fmt.Sprintf(format, v...))
	}

	// Include an inode number, if available.
	if f := v.FieldByName("Inode"); f.IsValid() {
		addComponent("inode %v", f.Interface())
	}

	// Handle special cases.
	switch typed := op.(type) {
	case *interruptOp:
		addComponent("fuseid 0x%08x", typed.FuseID)

	case *fuseops.ReadFileOp:
		addComponent("handle %d", typed.Handle)
		addComponent("offset %d", typed.Offset)
		addComponent("%d bytes", len(typed.Dst))

	case *fuseops.WriteFileOp:
		addComponent("handle %d", typed.Handle)
		addComponent("offset %d", typed.Offset)
		addComponent("%d bytes", len(typed.Data))
	}

	// Use just the name if there is no extra info.
	if len(components) == 0 {
		return opName(op)
	}

	// Otherwise, include the extra info.
	return fmt.Sprintf("%s (%s)", opName(op), strings.Join(components, ", "))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/infinityworksltd/prometheus-rancher-exporter/measure"
	"github.com/prometheus/client_golang/prometheus"
)

// Data is used to store data from all the relevant endpoints in the API
type Data struct {
	Data []struct {
		HealthState string `json:"healthState"`
		Name        string `json:"name"`
		State       string `json:"state"`
		System      bool   `json:"system"`
		Scale       int    `json:"scale"`
		HostName    string `json:"hostname"`
		ID          string `json:"id"`
		StackID     string `json:"stackId"`
		BaseType    string `json:"basetype"`
	} `json:"data"`
}

// processMetrics - Collects the data from the API, returns data object
func (e *Exporter) processMetrics(data *Data, endpoint string, hideSys bool, ch chan<- prometheus.Metric) error {

	// Metrics - range through the data object
	for _, x := range data.Data {

		// If system services have been ignored, the loop simply skips them
		if hideSys == true && x.System == true {
			continue
		}

		// Checks the metric is of the expected type
		if checkMetric(endpoint, x.BaseType) == false {
			log.Error("Invalid Metric Type detected from API, failed check Metric")
		}

		log.Debug("Processing metrics for %s", endpoint)

		if endpoint == "hosts" {

			if err := e.setHostMetrics(x.HostName, x.State); err != nil {
				log.Errorf("Error processing host metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s ", x.HostName, x.State)

				continue
			}

		} else if endpoint == "stacks" {

			// Used to create a map of stackID and stackName
			// Later used as a dimension in service metrics
			stackRef = storeStackRef(x.ID, x.Name)

			if err := e.setStackMetrics(x.Name, x.State, x.HealthState); err != nil {
				log.Errorf("Error processing stack metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s, %s ", x.Name, x.State, x.HealthState)
				continue
			}

		} else if endpoint == "services" {

			// Retrieves the stack Name from the previous values stored.
			stackName := retrieveStackRef(x.StackID)

			if err := e.setServiceMetrics(x.Name, stackName, x.State, x.HealthState, x.Scale); err != nil {
				log.Errorf("Error processing service metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s, %s, %s, %d", x.Name, stackName, x.State, x.HealthState, x.Scale)
				continue
			}

			e.setServiceMetrics(x.Name, stackName, x.State, x.HealthState, x.Scale)
		}

	}

	return nil
}

// gatherData - Collects the data from thw API, invokes functions to transform that data into metrics
func (e *Exporter) gatherData(rancherURL string, accessKey string, secretKey string, endpoint string, ch chan<- prometheus.Metric) (*Data, error) {

	// Check API version and return the correct URL path
	url := setEndpoint(rancherURL, endpoint)

	// Create new data slice from Struct
	var data = new(Data)

	// Scrape EndPoint for JSON Data
	err := getJSON(url, accessKey, secretKey, &data)
	if err != nil {
		log.Error("Error getting JSON from URL ", endpoint)
		return nil, err
	}
	log.Debugf("JSON Fetched for: "+endpoint+": ", data)

	return data, err
}

// getJSON return json from server, return the formatted JSON
func getJSON(url string, accessKey string, secretKey string, target interface{}) error {

	start := time.Now()

	// Counter for internal exporter metrics
	measure.FunctionCountTotal.With(prometheus.Labels{"pkg": "main", "fnc": "getJSON"}).Inc()

	log.Info("Scraping: ", url)

	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	req.SetBasicAuth(accessKey, secretKey)
	resp, err := client.Do(req)

	if err != nil {
		log.Error("Error Collecting JSON from API: ", err)
		panic(err)
	}

	respFormatted := json.NewDecoder(resp.Body).Decode(target)

	// Timings recorded as part of internal metrics
	elapsed := float64((time.Since(start)) / time.Microsecond)
	measure.FunctionDurations.WithLabelValues("hosts", "getJSON").Observe(elapsed)

	// Close the response body, the underlying Transport should then close the connection.
	resp.Body.Close()

	// return formatted JSON
	return respFormatted
}

// setEndpoint - Determines the correct URL endpoint to use, gives us backwards compatibility
func setEndpoint(rancherURL string, component string) string {

	var endpoint string

	if strings.Contains(component, "services") {
		endpoint = (rancherURL + "/services/")
	} else if strings.Contains(component, "hosts") {
		endpoint = (rancherURL + "/hosts/")
	} else if strings.Contains(component, "stacks") {
		if strings.Contains(rancherURL, "v1") {
			log.Info("Version 1 API detected, using legacy API fields")
			endpoint = (rancherURL + "/environments/")
		} else if strings.Contains(rancherURL, "v2") {
			endpoint = (rancherURL + "/stacks/")
		} else {
			log.Info("Unknown API version detected, defaulting to /stacks/")
			endpoint = (rancherURL + "/stacks/")
		}
	}

	return endpoint
}

// storeStackRef stores the stackID and stack name for use as a label elsewhere
func storeStackRef(stackID string, stackName string) map[string]string {

	stackRef[stackID] = stackName

	return stackRef
}

// retrieveStackRef returns the stack name, when sending the stackID
func retrieveStackRef(stackID string) string {

	for key, value := range stackRef {
		log.Debugf("Key is %s, Value is %s StackID is %s", key, value, stackID)
		if stackID == key {
			return value
		}
	}
	// returns unkown if no match was found
	return "unknown"
}
<commit_msg>Resolved small Typo on return value<commit_after>package main

import (
	"encoding/json"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/infinityworksltd/prometheus-rancher-exporter/measure"
	"github.com/prometheus/client_golang/prometheus"
)

// Data is used to store data from all the relevant endpoints in the API
type Data struct {
	Data []struct {
		HealthState string `json:"healthState"`
		Name        string `json:"name"`
		State       string `json:"state"`
		System      bool   `json:"system"`
		Scale       int    `json:"scale"`
		HostName    string `json:"hostname"`
		ID          string `json:"id"`
		StackID     string `json:"stackId"`
		BaseType    string `json:"basetype"`
	} `json:"data"`
}

// processMetrics - Collects the data from the API, returns data object
func (e *Exporter) processMetrics(data *Data, endpoint string, hideSys bool, ch chan<- prometheus.Metric) error {

	// Metrics - range through the data object
	for _, x := range data.Data {

		// If system services have been ignored, the loop simply skips them
		if hideSys == true && x.System == true {
			continue
		}

		// Checks the metric is of the expected type
		if checkMetric(endpoint, x.BaseType) == false {
			log.Error("Invalid Metric Type detected from API, failed check Metric")
		}

		log.Debug("Processing metrics for %s", endpoint)

		if endpoint == "hosts" {

			if err := e.setHostMetrics(x.HostName, x.State); err != nil {
				log.Errorf("Error processing host metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s ", x.HostName, x.State)

				continue
			}

		} else if endpoint == "stacks" {

			// Used to create a map of stackID and stackName
			// Later used as a dimension in service metrics
			stackRef = storeStackRef(x.ID, x.Name)

			if err := e.setStackMetrics(x.Name, x.State, x.HealthState); err != nil {
				log.Errorf("Error processing stack metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s, %s ", x.Name, x.State, x.HealthState)
				continue
			}

		} else if endpoint == "services" {

			// Retrieves the stack Name from the previous values stored.
			stackName := retrieveStackRef(x.StackID)

			if err := e.setServiceMetrics(x.Name, stackName, x.State, x.HealthState, x.Scale); err != nil {
				log.Errorf("Error processing service metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s, %s, %s, %d", x.Name, stackName, x.State, x.HealthState, x.Scale)
				continue
			}

			e.setServiceMetrics(x.Name, stackName, x.State, x.HealthState, x.Scale)
		}

	}

	return nil
}

// gatherData - Collects the data from thw API, invokes functions to transform that data into metrics
func (e *Exporter) gatherData(rancherURL string, accessKey string, secretKey string, endpoint string, ch chan<- prometheus.Metric) (*Data, error) {

	// Check API version and return the correct URL path
	url := setEndpoint(rancherURL, endpoint)

	// Create new data slice from Struct
	var data = new(Data)

	// Scrape EndPoint for JSON Data
	err := getJSON(url, accessKey, secretKey, &data)
	if err != nil {
		log.Error("Error getting JSON from URL ", endpoint)
		return nil, err
	}
	log.Debugf("JSON Fetched for: "+endpoint+": ", data)

	return data, err
}

// getJSON return json from server, return the formatted JSON
func getJSON(url string, accessKey string, secretKey string, target interface{}) error {

	start := time.Now()

	// Counter for internal exporter metrics
	measure.FunctionCountTotal.With(prometheus.Labels{"pkg": "main", "fnc": "getJSON"}).Inc()

	log.Info("Scraping: ", url)

	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	req.SetBasicAuth(accessKey, secretKey)
	resp, err := client.Do(req)

	if err != nil {
		log.Error("Error Collecting JSON from API: ", err)
		panic(err)
	}

	respFormatted := json.NewDecoder(resp.Body).Decode(target)

	// Timings recorded as part of internal metrics
	elapsed := float64((time.Since(start)) / time.Microsecond)
	measure.FunctionDurations.WithLabelValues("hosts", "getJSON").Observe(elapsed)

	// Close the response body, the underlying Transport should then close the connection.
	resp.Body.Close()

	// return formatted JSON
	return respFormatted
}

// setEndpoint - Determines the correct URL endpoint to use, gives us backwards compatibility
func setEndpoint(rancherURL string, component string) string {

	var endpoint string

	if strings.Contains(component, "services") {
		endpoint = (rancherURL + "/services/")
	} else if strings.Contains(component, "hosts") {
		endpoint = (rancherURL + "/hosts/")
	} else if strings.Contains(component, "stacks") {
		if strings.Contains(rancherURL, "v1") {
			log.Info("Version 1 API detected, using legacy API fields")
			endpoint = (rancherURL + "/environments/")
		} else if strings.Contains(rancherURL, "v2") {
			endpoint = (rancherURL + "/stacks/")
		} else {
			log.Info("Unknown API version detected, defaulting to /stacks/")
			endpoint = (rancherURL + "/stacks/")
		}
	}

	return endpoint
}

// storeStackRef stores the stackID and stack name for use as a label elsewhere
func storeStackRef(stackID string, stackName string) map[string]string {

	stackRef[stackID] = stackName

	return stackRef
}

// retrieveStackRef returns the stack name, when sending the stackID
func retrieveStackRef(stackID string) string {

	for key, value := range stackRef {
		log.Debugf("Key is %s, Value is %s StackID is %s", key, value, stackID)
		if stackID == key {
			return value
		}
	}
	// returns unknown if no match was found
	return "unknown"
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"strings"

	"github.com/Sirupsen/logrus"
	is "github.com/containers/image/storage"
	"github.com/containers/image/types"
	"github.com/containers/storage"
	"github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/projectatomic/buildah"
	"github.com/projectatomic/buildah/docker"
)

func main() {
	expectedManifestType := ""
	expectedConfigType := ""

	storeOptions := storage.DefaultStoreOptions
	root := flag.String("root", storeOptions.GraphRoot, "storage root directory")
	runroot := flag.String("runroot", storeOptions.RunRoot, "storage runtime directory")
	driver := flag.String("storage-driver", storeOptions.GraphDriverName, "storage driver")
	opts := flag.String("storage-opts", "", "storage option list (comma separated)")
	policy := flag.String("signature-policy", "", "signature policy file")
	mtype := flag.String("expected-manifest-type", buildah.OCIv1ImageManifest, "expected manifest type")
	showm := flag.Bool("show-manifest", false, "output the manifest JSON")
	showc := flag.Bool("show-config", false, "output the configuration JSON")
	flag.Parse()
	switch *mtype {
	case buildah.OCIv1ImageManifest:
		expectedManifestType = *mtype
		expectedConfigType = v1.MediaTypeImageConfig
	case buildah.Dockerv2ImageManifest:
		expectedManifestType = *mtype
		expectedConfigType = docker.V2S2MediaTypeImageConfig
	case "*":
		expectedManifestType = ""
		expectedConfigType = ""
	default:
		logrus.Fatalf("unknown -expected-manifest-type value, expected either %q or %q or %q",
			buildah.OCIv1ImageManifest, buildah.Dockerv2ImageManifest, "*")
	}
	if root != nil {
		storeOptions.GraphRoot = *root
	}
	if runroot != nil {
		storeOptions.RunRoot = *runroot
	}
	if driver != nil {
		storeOptions.GraphDriverName = *driver
	}
	if opts != nil && *opts != "" {
		storeOptions.GraphDriverOptions = strings.Split(*opts, ",")
	}
	systemContext := &types.SystemContext{
		SignaturePolicyPath: *policy,
	}
	args := flag.Args()
	if len(args) == 0 {
		flag.Usage()
		return
	}
	store, err := storage.GetStore(storeOptions)
	if err != nil {
		logrus.Fatalf("error opening storage: %v", err)
	}
	defer store.Shutdown(false)

	for _, image := range args {
		oImage := v1.Image{}
		dImage := docker.V2Image{}
		oManifest := v1.Manifest{}
		dManifest := docker.V2S2Manifest{}
		manifestType := ""
		configType := ""

		ref, err := is.Transport.ParseStoreReference(store, image)
		if err != nil {
			logrus.Fatalf("error parsing reference %q: %v", image, err)
		}

		src, err := ref.NewImageSource(systemContext, []string{expectedManifestType})
		if err != nil {
			logrus.Fatalf("error opening source image %q: %v", image, err)
		}
		defer src.Close()

		manifest, manifestType, err := src.GetManifest()
		if err != nil {
			logrus.Fatalf("error reading manifest from %q: %v", image, err)
		}

		img, err := ref.NewImage(systemContext)
		if err != nil {
			logrus.Fatalf("error opening image %q: %v", image, err)
		}
		defer img.Close()

		config, err := img.ConfigBlob()
		if err != nil {
			logrus.Fatalf("error reading configuration from %q: %v", image, err)
		}

		switch expectedManifestType {
		case buildah.OCIv1ImageManifest:
			err = json.Unmarshal(manifest, &oManifest)
			if err != nil {
				logrus.Fatalf("error parsing manifest from %q: %v", image, err)
			}
			err = json.Unmarshal(config, &oImage)
			if err != nil {
				logrus.Fatalf("error parsing config from %q: %v", image, err)
			}
			manifestType = v1.MediaTypeImageManifest
			configType = oManifest.Config.MediaType
		case buildah.Dockerv2ImageManifest:
			err = json.Unmarshal(manifest, &dManifest)
			if err != nil {
				logrus.Fatalf("error parsing manifest from %q: %v", image, err)
			}
			err = json.Unmarshal(config, &dImage)
			if err != nil {
				logrus.Fatalf("error parsing config from %q: %v", image, err)
			}
			manifestType = dManifest.MediaType
			configType = dManifest.Config.MediaType
		}
		if expectedManifestType != "" && manifestType != expectedManifestType {
			logrus.Fatalf("expected manifest type %q in %q, got %q", expectedManifestType, image, manifestType)
		}
		if expectedConfigType != "" && configType != expectedConfigType {
			logrus.Fatalf("expected config type %q in %q, got %q", expectedConfigType, image, configType)
		}
		if showm != nil && *showm {
			fmt.Println(string(manifest))
		}
		if showc != nil && *showc {
			fmt.Println(string(config))
		}
	}
}
<commit_msg>imgtype: add debugging, reexec, an optimization<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"strings"

	"github.com/Sirupsen/logrus"
	is "github.com/containers/image/storage"
	"github.com/containers/image/types"
	"github.com/containers/storage"
	"github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/projectatomic/buildah"
	"github.com/projectatomic/buildah/docker"
)

func main() {
	if buildah.InitReexec() {
		return
	}

	expectedManifestType := ""
	expectedConfigType := ""

	storeOptions := storage.DefaultStoreOptions
	debug := flag.Bool("debug", false, "turn on debug logging")
	root := flag.String("root", storeOptions.GraphRoot, "storage root directory")
	runroot := flag.String("runroot", storeOptions.RunRoot, "storage runtime directory")
	driver := flag.String("storage-driver", storeOptions.GraphDriverName, "storage driver")
	opts := flag.String("storage-opts", "", "storage option list (comma separated)")
	policy := flag.String("signature-policy", "", "signature policy file")
	mtype := flag.String("expected-manifest-type", buildah.OCIv1ImageManifest, "expected manifest type")
	showm := flag.Bool("show-manifest", false, "output the manifest JSON")
	showc := flag.Bool("show-config", false, "output the configuration JSON")
	flag.Parse()
	logrus.SetLevel(logrus.ErrorLevel)
	if debug != nil && *debug {
		logrus.SetLevel(logrus.DebugLevel)
	}
	switch *mtype {
	case buildah.OCIv1ImageManifest:
		expectedManifestType = *mtype
		expectedConfigType = v1.MediaTypeImageConfig
	case buildah.Dockerv2ImageManifest:
		expectedManifestType = *mtype
		expectedConfigType = docker.V2S2MediaTypeImageConfig
	case "*":
		expectedManifestType = ""
		expectedConfigType = ""
	default:
		logrus.Fatalf("unknown -expected-manifest-type value, expected either %q or %q or %q",
			buildah.OCIv1ImageManifest, buildah.Dockerv2ImageManifest, "*")
	}
	if root != nil {
		storeOptions.GraphRoot = *root
	}
	if runroot != nil {
		storeOptions.RunRoot = *runroot
	}
	if driver != nil {
		storeOptions.GraphDriverName = *driver
	}
	if opts != nil && *opts != "" {
		storeOptions.GraphDriverOptions = strings.Split(*opts, ",")
	}
	systemContext := &types.SystemContext{
		SignaturePolicyPath: *policy,
	}
	args := flag.Args()
	if len(args) == 0 {
		flag.Usage()
		return
	}
	store, err := storage.GetStore(storeOptions)
	if err != nil {
		logrus.Fatalf("error opening storage: %v", err)
	}
	defer store.Shutdown(false)

	for _, image := range args {
		oImage := v1.Image{}
		dImage := docker.V2Image{}
		oManifest := v1.Manifest{}
		dManifest := docker.V2S2Manifest{}
		manifestType := ""
		configType := ""

		ref, err := is.Transport.ParseStoreReference(store, image)
		if err != nil {
			logrus.Fatalf("error parsing reference %q: %v", image, err)
		}

		img, err := ref.NewImage(systemContext)
		if err != nil {
			logrus.Fatalf("error opening image %q: %v", image, err)
		}
		defer img.Close()

		config, err := img.ConfigBlob()
		if err != nil {
			logrus.Fatalf("error reading configuration from %q: %v", image, err)
		}

		manifest, manifestType, err := img.Manifest()
		if err != nil {
			logrus.Fatalf("error reading manifest from %q: %v", image, err)
		}

		switch expectedManifestType {
		case buildah.OCIv1ImageManifest:
			err = json.Unmarshal(manifest, &oManifest)
			if err != nil {
				logrus.Fatalf("error parsing manifest from %q: %v", image, err)
			}
			err = json.Unmarshal(config, &oImage)
			if err != nil {
				logrus.Fatalf("error parsing config from %q: %v", image, err)
			}
			manifestType = v1.MediaTypeImageManifest
			configType = oManifest.Config.MediaType
		case buildah.Dockerv2ImageManifest:
			err = json.Unmarshal(manifest, &dManifest)
			if err != nil {
				logrus.Fatalf("error parsing manifest from %q: %v", image, err)
			}
			err = json.Unmarshal(config, &dImage)
			if err != nil {
				logrus.Fatalf("error parsing config from %q: %v", image, err)
			}
			manifestType = dManifest.MediaType
			configType = dManifest.Config.MediaType
		}
		if expectedManifestType != "" && manifestType != expectedManifestType {
			logrus.Fatalf("expected manifest type %q in %q, got %q", expectedManifestType, image, manifestType)
		}
		if expectedConfigType != "" && configType != expectedConfigType {
			logrus.Fatalf("expected config type %q in %q, got %q", expectedConfigType, image, configType)
		}
		if showm != nil && *showm {
			fmt.Println(string(manifest))
		}
		if showc != nil && *showc {
			fmt.Println(string(config))
		}
	}
}
<|endoftext|>