<commit_before>package tests

import (
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/google/goexpect"
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/log"
	ktests "kubevirt.io/kubevirt/tests"
)

type Result struct {
	cmd           string
	verb          string
	resourceType  string
	resourceName  string
	resourceLabel string
	filePath      string
	nameSpace     string
	query         string
	expectOut     string
	actualOut     string
	params        []string
}

const (
	UsernameTestUser     = "kubevirt-test-user"
	NamespaceTestDefault = "kubevirt-test-default"
	UsernameAdminUser    = "test_admin"
)

const (
	CDI_LABEL_KEY      = "app"
	CDI_LABEL_VALUE    = "containerized-data-importer"
	CDI_LABEL_SELECTOR = CDI_LABEL_KEY + "=" + CDI_LABEL_VALUE
	paramFlag          = "-p"
)

func CreateNamespaces() {
	virtCli, err := kubecli.GetKubevirtClient()
	ktests.PanicOnError(err)

	testNamespaces := []string{ktests.NamespaceTestDefault, ktests.NamespaceTestAlternative}
	// Create a Test Namespaces
	for _, namespace := range testNamespaces {
		ns := &k8sv1.Namespace{
			ObjectMeta: metav1.ObjectMeta{
				Name: namespace,
			},
		}
		_, err = virtCli.CoreV1().Namespaces().Create(ns)
		if !errors.IsAlreadyExists(err) {
			ktests.PanicOnError(err)
		}
	}
}

func RemoveNamespaces() {
	virtCli, err := kubecli.GetKubevirtClient()
	ktests.PanicOnError(err)
	testNamespaces := []string{ktests.NamespaceTestDefault, ktests.NamespaceTestAlternative}

	// First send an initial delete to every namespace
	for _, namespace := range testNamespaces {
		err := virtCli.CoreV1().Namespaces().Delete(namespace, nil)
		if !errors.IsNotFound(err) {
			ktests.PanicOnError(err)
		}
	}
	// Wait until the namespaces are terminated
	fmt.Println("")
	for _, namespace := range testNamespaces {
		fmt.Printf("Removing the %s namespace. It can take some time...\n", namespace)
		Eventually(func() bool { return errors.IsNotFound(virtCli.CoreV1().Namespaces().Delete(namespace, nil)) }, 180*time.Second, 1*time.Second).
			Should(BeTrue())
	}
}

func ProcessTemplateWithParameters(srcFilePath, dstFilePath string, params ...string) string {
	By(fmt.Sprintf("Overriding the template from %s to %s", srcFilePath, dstFilePath))
	out := execute(Result{cmd: "oc", verb: "process", filePath: srcFilePath, params: params})
	filePath, err := writeJson(dstFilePath, out)
	Expect(err).ToNot(HaveOccurred())
	return filePath
}

func CreateResourceWithFilePathTestNamespace(filePath string) {
	By("Creating resource from the json file with the oc-create command")
	execute(Result{cmd: "oc", verb: "create", filePath: filePath, nameSpace: NamespaceTestDefault})
}

func DeleteResourceWithLabelTestNamespace(resourceType, resourceLabel string) {
	By(fmt.Sprintf("Deleting %s:%s from the json file with the oc-delete command", resourceType, resourceLabel))
	execute(Result{cmd: "oc", verb: "delete", resourceType: resourceType, resourceLabel: resourceLabel, nameSpace: NamespaceTestDefault})
}

func WaitUntilResourceReadyByNameTestNamespace(resourceType, resourceName, query, expectOut string) {
	By(fmt.Sprintf("Wait until %s with name %s ready", resourceType, resourceName))
	execute(Result{cmd: "oc", verb: "get", resourceType: resourceType, resourceName: resourceName, query: query, expectOut: expectOut, nameSpace: NamespaceTestDefault})
}

func WaitUntilResourceReadyByLabelTestNamespace(resourceType, label, query, expectOut string) {
	By(fmt.Sprintf("Wait until resource %s with label=%s ready", resourceType, label))
	execute(Result{cmd: "oc", verb: "get", resourceType: resourceType, resourceLabel: label, query: query, expectOut: expectOut, nameSpace: NamespaceTestDefault})
}

func CreateUser(username string) {
	By(fmt.Sprintf("Wait until user %s is created ", username))
	execute(Result{cmd: "oc", verb: "create", resourceType: "user", resourceName: username})
}

func DeleteUser(username string) {
	By(fmt.Sprintf("Wait until user %s is deleted", username))
	execute(Result{cmd: "oc", verb: "delete", resourceType: "user", resourceName: username})
}

func VNCConnection(namespace, vmname string) (string, error) {
	virtClient, err := kubecli.GetKubevirtClient()
	ktests.PanicOnError(err)
	pipeInReader, _ := io.Pipe()
	pipeOutReader, pipeOutWriter := io.Pipe()
	k8ResChan := make(chan error)
	readStop := make(chan string)

	go func() {
		GinkgoRecover()
		vnc, err := virtClient.VirtualMachineInstance(namespace).VNC(vmname)
		if err != nil {
			k8ResChan <- err
			return
		}

		k8ResChan <- vnc.Stream(kubecli.StreamOptions{
			In:  pipeInReader,
			Out: pipeOutWriter,
		})
	}()

	go func() {
		GinkgoRecover()
		buf := make([]byte, 1024, 1024)
		n, err := pipeOutReader.Read(buf)
		if err != nil && err != io.EOF {
			return
		}
		if n == 0 && err == io.EOF {
			log.Log.Info("zero bytes read from vnc socket.")
			return
		}
		readStop <- strings.TrimSpace(string(buf[0:n]))
	}()
	response := ""
	select {
	case response = <-readStop:
	case err = <-k8ResChan:
	case <-time.After(45 * time.Second):
		Fail("Timout reached while waiting for valid VNC server response")
	}
	return response, err
}

func execute(r Result) string {
	var err error
	if r.verb == "" {
		Expect(fmt.Errorf("verb can not be empty"))
	}
	cmd := []string{r.verb}
	if r.filePath == "" {
		if r.resourceType == "" {
			Expect(fmt.Errorf("resourceType can not be empty"))
		}
		cmd = append(cmd, r.resourceType)
	}
	if r.resourceName != "" {
		cmd = append(cmd, r.resourceName)
	}
	if r.filePath != "" {
		cmd = append(cmd, "-f", r.filePath)
	}
	if r.resourceLabel != "" {
		cmd = append(cmd, "-l", r.resourceLabel)
	}
	if r.query != "" {
		cmd = append(cmd, r.query)
	}
	if r.nameSpace != "" {
		cmd = append(cmd, "-n", r.nameSpace)
	}
	if len(r.params) > 0 {
		for _, v := range r.params {
			cmd = append(cmd, paramFlag, v)
		}
	}
	if r.expectOut != "" {
		Eventually(func() bool {
			r.actualOut, _, err = ktests.RunCommand(r.cmd, cmd...)
			Expect(err).ToNot(HaveOccurred())
			return strings.Contains(r.actualOut, r.expectOut)
		}, time.Duration(2)*time.Minute).Should(BeTrue(), fmt.Sprintf("Timed out waiting for %s to appear", r.resourceType))
	} else {
		r.actualOut, _, err = ktests.RunCommand(r.cmd, cmd...)
		Expect(err).ToNot(HaveOccurred())
	}
	return r.actualOut
}

func writeJson(jsonFile string, json string) (string, error) {
	err := ioutil.WriteFile(jsonFile, []byte(json), 0644)
	if err != nil {
		return "", fmt.Errorf("failed to write the json file %s", jsonFile)
	}
	return jsonFile, nil
}

func RunOcDescribeCommand(resourceType, resourceName string) string {
	fmt.Printf("Getting 'oc describe' with: %s ", resourceName)
	return execute(Result{cmd: "oc", verb: "describe", resourceType: resourceType, resourceName: resourceName, nameSpace: NamespaceTestDefault})
}

func OpenConsole(virtCli kubecli.KubevirtClient, vmiName string, vmiNamespace string, timeout time.Duration , consoleType string, opts ...expect.Option) (expect.Expecter, <-chan error, error) {
	vmiReader, vmiWriter := io.Pipe()
	expecterReader, expecterWriter := io.Pipe()
	resCh := make(chan error)
	var con kubecli.StreamInterface
	var err error
	startTime := time.Now()
	if consoleType == "serial" {
		con, err = virtCli.VirtualMachineInstance(vmiNamespace).SerialConsole(vmiName, timeout)
	} else if consoleType == "vnc" {
		con, err = virtCli.VirtualMachineInstance(vmiNamespace).VNC(vmiName)
	}
	if err != nil {
		return nil, nil, err
	}
	timeout = timeout - time.Now().Sub(startTime)

	go func() {
		resCh <- con.Stream(kubecli.StreamOptions{
			In:  vmiReader,
			Out: expecterWriter,
		})
	}()

	return expect.SpawnGeneric(&expect.GenOptions{
		In:  vmiWriter,
		Out: expecterReader,
		Wait: func() error {
			return <-resCh
		},
		Close: func() error {
			expecterWriter.Close()
			vmiReader.Close()
			return nil
		},
		Check: func() bool { return true },
	}, timeout, opts...)
}
<commit_msg>Increase timeout when validating oc cmd results<commit_after>package tests

import (
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/google/goexpect"
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/log"
	ktests "kubevirt.io/kubevirt/tests"
)

type Result struct {
	cmd           string
	verb          string
	resourceType  string
	resourceName  string
	resourceLabel string
	filePath      string
	nameSpace     string
	query         string
	expectOut     string
	actualOut     string
	params        []string
}

const (
	UsernameTestUser     = "kubevirt-test-user"
	NamespaceTestDefault = "kubevirt-test-default"
	UsernameAdminUser    = "test_admin"
)

const (
	CDI_LABEL_KEY      = "app"
	CDI_LABEL_VALUE    = "containerized-data-importer"
	CDI_LABEL_SELECTOR = CDI_LABEL_KEY + "=" + CDI_LABEL_VALUE
	paramFlag          = "-p"
)

const (
	ShortTimeout       = time.Duration(2) * time.Minute
	LongTimeout        = time.Duration(4) * time.Minute
)

func CreateNamespaces() {
	virtCli, err := kubecli.GetKubevirtClient()
	ktests.PanicOnError(err)

	testNamespaces := []string{ktests.NamespaceTestDefault, ktests.NamespaceTestAlternative}
	// Create a Test Namespaces
	for _, namespace := range testNamespaces {
		ns := &k8sv1.Namespace{
			ObjectMeta: metav1.ObjectMeta{
				Name: namespace,
			},
		}
		_, err = virtCli.CoreV1().Namespaces().Create(ns)
		if !errors.IsAlreadyExists(err) {
			ktests.PanicOnError(err)
		}
	}
}

func RemoveNamespaces() {
	virtCli, err := kubecli.GetKubevirtClient()
	ktests.PanicOnError(err)
	testNamespaces := []string{ktests.NamespaceTestDefault, ktests.NamespaceTestAlternative}

	// First send an initial delete to every namespace
	for _, namespace := range testNamespaces {
		err := virtCli.CoreV1().Namespaces().Delete(namespace, nil)
		if !errors.IsNotFound(err) {
			ktests.PanicOnError(err)
		}
	}
	// Wait until the namespaces are terminated
	fmt.Println("")
	for _, namespace := range testNamespaces {
		fmt.Printf("Removing the %s namespace. It can take some time...\n", namespace)
		Eventually(func() bool { return errors.IsNotFound(virtCli.CoreV1().Namespaces().Delete(namespace, nil)) }, 180*time.Second, 1*time.Second).
			Should(BeTrue())
	}
}

func ProcessTemplateWithParameters(srcFilePath, dstFilePath string, params ...string) string {
	By(fmt.Sprintf("Overriding the template from %s to %s", srcFilePath, dstFilePath))
	out := execute(Result{cmd: "oc", verb: "process", filePath: srcFilePath, params: params})
	filePath, err := writeJson(dstFilePath, out)
	Expect(err).ToNot(HaveOccurred())
	return filePath
}

func CreateResourceWithFilePathTestNamespace(filePath string) {
	By("Creating resource from the json file with the oc-create command")
	execute(Result{cmd: "oc", verb: "create", filePath: filePath, nameSpace: NamespaceTestDefault})
}

func DeleteResourceWithLabelTestNamespace(resourceType, resourceLabel string) {
	By(fmt.Sprintf("Deleting %s:%s from the json file with the oc-delete command", resourceType, resourceLabel))
	execute(Result{cmd: "oc", verb: "delete", resourceType: resourceType, resourceLabel: resourceLabel, nameSpace: NamespaceTestDefault})
}

func WaitUntilResourceReadyByNameTestNamespace(resourceType, resourceName, query, expectOut string) {
	By(fmt.Sprintf("Wait until %s with name %s ready", resourceType, resourceName))
	execute(Result{cmd: "oc", verb: "get", resourceType: resourceType, resourceName: resourceName, query: query, expectOut: expectOut, nameSpace: NamespaceTestDefault})
}

func WaitUntilResourceReadyByLabelTestNamespace(resourceType, label, query, expectOut string) {
	By(fmt.Sprintf("Wait until resource %s with label=%s ready", resourceType, label))
	execute(Result{cmd: "oc", verb: "get", resourceType: resourceType, resourceLabel: label, query: query, expectOut: expectOut, nameSpace: NamespaceTestDefault})
}

func CreateUser(username string) {
	By(fmt.Sprintf("Wait until user %s is created ", username))
	execute(Result{cmd: "oc", verb: "create", resourceType: "user", resourceName: username})
}

func DeleteUser(username string) {
	By(fmt.Sprintf("Wait until user %s is deleted", username))
	execute(Result{cmd: "oc", verb: "delete", resourceType: "user", resourceName: username})
}

func VNCConnection(namespace, vmname string) (string, error) {
	virtClient, err := kubecli.GetKubevirtClient()
	ktests.PanicOnError(err)
	pipeInReader, _ := io.Pipe()
	pipeOutReader, pipeOutWriter := io.Pipe()
	k8ResChan := make(chan error)
	readStop := make(chan string)

	go func() {
		GinkgoRecover()
		vnc, err := virtClient.VirtualMachineInstance(namespace).VNC(vmname)
		if err != nil {
			k8ResChan <- err
			return
		}

		k8ResChan <- vnc.Stream(kubecli.StreamOptions{
			In:  pipeInReader,
			Out: pipeOutWriter,
		})
	}()

	go func() {
		GinkgoRecover()
		buf := make([]byte, 1024, 1024)
		n, err := pipeOutReader.Read(buf)
		if err != nil && err != io.EOF {
			return
		}
		if n == 0 && err == io.EOF {
			log.Log.Info("zero bytes read from vnc socket.")
			return
		}
		readStop <- strings.TrimSpace(string(buf[0:n]))
	}()
	response := ""
	select {
	case response = <-readStop:
	case err = <-k8ResChan:
	case <-time.After(45 * time.Second):
		Fail("Timout reached while waiting for valid VNC server response")
	}
	return response, err
}

func execute(r Result) string {
	return executeWithCustomTimeout(r, LongTimeout)
}

func executeWithCustomTimeout(r Result, timeout time.Duration) string {
	var err error
	if r.verb == "" {
		Expect(fmt.Errorf("verb can not be empty"))
	}
	cmd := []string{r.verb}
	if r.filePath == "" {
		if r.resourceType == "" {
			Expect(fmt.Errorf("resourceType can not be empty"))
		}
		cmd = append(cmd, r.resourceType)
	}
	if r.resourceName != "" {
		cmd = append(cmd, r.resourceName)
	}
	if r.filePath != "" {
		cmd = append(cmd, "-f", r.filePath)
	}
	if r.resourceLabel != "" {
		cmd = append(cmd, "-l", r.resourceLabel)
	}
	if r.query != "" {
		cmd = append(cmd, r.query)
	}
	if r.nameSpace != "" {
		cmd = append(cmd, "-n", r.nameSpace)
	}
	if len(r.params) > 0 {
		for _, v := range r.params {
			cmd = append(cmd, paramFlag, v)
		}
	}
	if r.expectOut != "" {
		Eventually(func() bool {
			r.actualOut, _, err = ktests.RunCommand(r.cmd, cmd...)
			Expect(err).ToNot(HaveOccurred())
			return strings.Contains(r.actualOut, r.expectOut)
		}, timeout).Should(BeTrue(), fmt.Sprintf("Timed out waiting for %s to appear", r.resourceType))
	} else {
		r.actualOut, _, err = ktests.RunCommand(r.cmd, cmd...)
		Expect(err).ToNot(HaveOccurred())
	}
	return r.actualOut
}

func writeJson(jsonFile string, json string) (string, error) {
	err := ioutil.WriteFile(jsonFile, []byte(json), 0644)
	if err != nil {
		return "", fmt.Errorf("failed to write the json file %s", jsonFile)
	}
	return jsonFile, nil
}

func RunOcDescribeCommand(resourceType, resourceName string) string {
	fmt.Printf("Getting 'oc describe' with: %s ", resourceName)
	return execute(Result{cmd: "oc", verb: "describe", resourceType: resourceType, resourceName: resourceName, nameSpace: NamespaceTestDefault})
}

func OpenConsole(virtCli kubecli.KubevirtClient, vmiName string, vmiNamespace string, timeout time.Duration , consoleType string, opts ...expect.Option) (expect.Expecter, <-chan error, error) {
	vmiReader, vmiWriter := io.Pipe()
	expecterReader, expecterWriter := io.Pipe()
	resCh := make(chan error)
	var con kubecli.StreamInterface
	var err error
	startTime := time.Now()
	if consoleType == "serial" {
		con, err = virtCli.VirtualMachineInstance(vmiNamespace).SerialConsole(vmiName, timeout)
	} else if consoleType == "vnc" {
		con, err = virtCli.VirtualMachineInstance(vmiNamespace).VNC(vmiName)
	}
	if err != nil {
		return nil, nil, err
	}
	timeout = timeout - time.Now().Sub(startTime)

	go func() {
		resCh <- con.Stream(kubecli.StreamOptions{
			In:  vmiReader,
			Out: expecterWriter,
		})
	}()

	return expect.SpawnGeneric(&expect.GenOptions{
		In:  vmiWriter,
		Out: expecterReader,
		Wait: func() error {
			return <-resCh
		},
		Close: func() error {
			expecterWriter.Close()
			vmiReader.Close()
			return nil
		},
		Check: func() bool { return true },
	}, timeout, opts...)
}
<|endoftext|>


<commit_before>package abstract_sql

import (
	"context"
	"database/sql"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"strings"
)

type AbstractSqlStore struct {
	DB                      *sql.DB
	SqlInsert               string
	SqlUpdate               string
	SqlFind                 string
	SqlDelete               string
	SqlDeleteFolderChildren string
	SqlListExclusive        string
	SqlListInclusive        string
}

type TxOrDB interface {
	ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
	QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row
	QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
}

func (store *AbstractSqlStore) BeginTransaction(ctx context.Context) (context.Context, error) {
	tx, err := store.DB.BeginTx(ctx, &sql.TxOptions{
		Isolation: sql.LevelReadCommitted,
		ReadOnly:  false,
	})
	if err != nil {
		return ctx, err
	}

	return context.WithValue(ctx, "tx", tx), nil
}
func (store *AbstractSqlStore) CommitTransaction(ctx context.Context) error {
	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
		return tx.Commit()
	}
	return nil
}
func (store *AbstractSqlStore) RollbackTransaction(ctx context.Context) error {
	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
		return tx.Rollback()
	}
	return nil
}

func (store *AbstractSqlStore) getTxOrDB(ctx context.Context) TxOrDB {
	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
		return tx
	}
	return store.DB
}

func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {

	dir, name := entry.FullPath.DirAndName()
	meta, err := entry.EncodeAttributesAndChunks()
	if err != nil {
		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
	}

	if len(entry.Chunks) > 50 {
		meta = util.MaybeGzipData(meta)
	}

	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlInsert, util.HashStringToLong(dir), name, dir, meta)
	if err == nil {
		return
	}

	if !strings.Contains(strings.ToLower(err.Error()), "duplicate") {
		// return fmt.Errorf("insert: %s", err)
		// skip this since the error can be in a different language
	}

	// now the insert failed possibly due to duplication constraints
	glog.V(1).Infof("insert %s falls back to update: %v", entry.FullPath, err)

	res, err = store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir)
	if err != nil {
		return fmt.Errorf("upsert %s: %s", entry.FullPath, err)
	}

	_, err = res.RowsAffected()
	if err != nil {
		return fmt.Errorf("upsert %s but no rows affected: %s", entry.FullPath, err)
	}
	return nil

}

func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {

	dir, name := entry.FullPath.DirAndName()
	meta, err := entry.EncodeAttributesAndChunks()
	if err != nil {
		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
	}

	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir)
	if err != nil {
		return fmt.Errorf("update %s: %s", entry.FullPath, err)
	}

	_, err = res.RowsAffected()
	if err != nil {
		return fmt.Errorf("update %s but no rows affected: %s", entry.FullPath, err)
	}
	return nil
}

func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath util.FullPath) (*filer.Entry, error) {

	dir, name := fullpath.DirAndName()
	row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, util.HashStringToLong(dir), name, dir)

	var data []byte
	if err := row.Scan(&data); err != nil {
		if err == sql.ErrNoRows {
			return nil, filer_pb.ErrNotFound
		}
		return nil, fmt.Errorf("find %s: %v", fullpath, err)
	}

	entry := &filer.Entry{
		FullPath: fullpath,
	}
	if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
	}

	return entry, nil
}

func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error {

	dir, name := fullpath.DirAndName()

	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDelete, util.HashStringToLong(dir), name, dir)
	if err != nil {
		return fmt.Errorf("delete %s: %s", fullpath, err)
	}

	_, err = res.RowsAffected()
	if err != nil {
		return fmt.Errorf("delete %s but no rows affected: %s", fullpath, err)
	}

	return nil
}

func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {

	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDeleteFolderChildren, util.HashStringToLong(string(fullpath)), fullpath)
	if err != nil {
		return fmt.Errorf("deleteFolderChildren %s: %s", fullpath, err)
	}

	_, err = res.RowsAffected()
	if err != nil {
		return fmt.Errorf("deleteFolderChildren %s but no rows affected: %s", fullpath, err)
	}

	return nil
}

func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) {
	sqlText := store.SqlListExclusive
	if inclusive {
		sqlText = store.SqlListInclusive
	}

	rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(fullpath)), startFileName, string(fullpath), prefix+"%", limit)
	if err != nil {
		return nil, fmt.Errorf("list %s : %v", fullpath, err)
	}
	defer rows.Close()

	for rows.Next() {
		var name string
		var data []byte
		if err = rows.Scan(&name, &data); err != nil {
			glog.V(0).Infof("scan %s : %v", fullpath, err)
			return nil, fmt.Errorf("scan %s: %v", fullpath, err)
		}

		entry := &filer.Entry{
			FullPath: util.NewFullPath(string(fullpath), name),
		}
		if err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
			glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err)
			return nil, fmt.Errorf("scan decode %s : %v", entry.FullPath, err)
		}

		entries = append(entries, entry)
	}

	return entries, nil
}

func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer.Entry, err error) {
	return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "")
}

func (store *AbstractSqlStore) Shutdown() {
	store.DB.Close()
}
<commit_msg>implement sql changes<commit_after>package abstract_sql

import (
	"context"
	"database/sql"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"strings"
)

type AbstractSqlStore struct {
	DB                      *sql.DB
	SqlInsert               string
	SqlUpdate               string
	SqlFind                 string
	SqlDelete               string
	SqlDeleteFolderChildren string
	SqlListExclusive        string
	SqlListInclusive        string
}

type TxOrDB interface {
	ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
	QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row
	QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
}

func (store *AbstractSqlStore) BeginTransaction(ctx context.Context) (context.Context, error) {
	tx, err := store.DB.BeginTx(ctx, &sql.TxOptions{
		Isolation: sql.LevelReadCommitted,
		ReadOnly:  false,
	})
	if err != nil {
		return ctx, err
	}

	return context.WithValue(ctx, "tx", tx), nil
}
func (store *AbstractSqlStore) CommitTransaction(ctx context.Context) error {
	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
		return tx.Commit()
	}
	return nil
}
func (store *AbstractSqlStore) RollbackTransaction(ctx context.Context) error {
	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
		return tx.Rollback()
	}
	return nil
}

func (store *AbstractSqlStore) getTxOrDB(ctx context.Context) TxOrDB {
	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
		return tx
	}
	return store.DB
}

func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {

	dir, name := entry.FullPath.DirAndName()
	meta, err := entry.EncodeAttributesAndChunks()
	if err != nil {
		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
	}

	if len(entry.Chunks) > 50 {
		meta = util.MaybeGzipData(meta)
	}

	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlInsert, util.HashStringToLong(dir), name, dir, meta)
	if err == nil {
		return
	}

	if !strings.Contains(strings.ToLower(err.Error()), "duplicate") {
		// return fmt.Errorf("insert: %s", err)
		// skip this since the error can be in a different language
	}

	// now the insert failed possibly due to duplication constraints
	glog.V(1).Infof("insert %s falls back to update: %v", entry.FullPath, err)

	res, err = store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir)
	if err != nil {
		return fmt.Errorf("upsert %s: %s", entry.FullPath, err)
	}

	_, err = res.RowsAffected()
	if err != nil {
		return fmt.Errorf("upsert %s but no rows affected: %s", entry.FullPath, err)
	}
	return nil

}

func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {

	dir, name := entry.FullPath.DirAndName()
	meta, err := entry.EncodeAttributesAndChunks()
	if err != nil {
		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
	}

	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir)
	if err != nil {
		return fmt.Errorf("update %s: %s", entry.FullPath, err)
	}

	_, err = res.RowsAffected()
	if err != nil {
		return fmt.Errorf("update %s but no rows affected: %s", entry.FullPath, err)
	}
	return nil
}

func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath util.FullPath) (*filer.Entry, error) {

	dir, name := fullpath.DirAndName()
	row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, util.HashStringToLong(dir), name, dir)

	var data []byte
	if err := row.Scan(&data); err != nil {
		if err == sql.ErrNoRows {
			return nil, filer_pb.ErrNotFound
		}
		return nil, fmt.Errorf("find %s: %v", fullpath, err)
	}

	entry := &filer.Entry{
		FullPath: fullpath,
	}
	if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
	}

	return entry, nil
}

func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error {

	dir, name := fullpath.DirAndName()

	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDelete, util.HashStringToLong(dir), name, dir)
	if err != nil {
		return fmt.Errorf("delete %s: %s", fullpath, err)
	}

	_, err = res.RowsAffected()
	if err != nil {
		return fmt.Errorf("delete %s but no rows affected: %s", fullpath, err)
	}

	return nil
}

func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {

	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDeleteFolderChildren, util.HashStringToLong(string(fullpath)), fullpath)
	if err != nil {
		return fmt.Errorf("deleteFolderChildren %s: %s", fullpath, err)
	}

	_, err = res.RowsAffected()
	if err != nil {
		return fmt.Errorf("deleteFolderChildren %s but no rows affected: %s", fullpath, err)
	}

	return nil
}

func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
	sqlText := store.SqlListExclusive
	if includeStartFile {
		sqlText = store.SqlListInclusive
	}

	rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(dirPath)), startFileName, string(dirPath), prefix+"%", limit+1)
	if err != nil {
		return nil, false, fmt.Errorf("list %s : %v", dirPath, err)
	}
	defer rows.Close()

	for rows.Next() {
		var name string
		var data []byte
		if err = rows.Scan(&name, &data); err != nil {
			glog.V(0).Infof("scan %s : %v", dirPath, err)
			return nil, false, fmt.Errorf("scan %s: %v", dirPath, err)
		}

		entry := &filer.Entry{
			FullPath: util.NewFullPath(string(dirPath), name),
		}
		if err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
			glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err)
			return nil, false, fmt.Errorf("scan decode %s : %v", entry.FullPath, err)
		}

		entries = append(entries, entry)
	}

	hasMore = len(entries) == limit + 1
	if hasMore {
		entries = entries[:limit]
	}

	return entries, hasMore, nil
}

func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) {
	return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "")
}

func (store *AbstractSqlStore) Shutdown() {
	store.DB.Close()
}
<|endoftext|>


<commit_before><commit_msg>Adjust behavior of time bindings for immutable predicates (or objects that are immutable predicates) to depend on the type of the clause (optional or not)<commit_after><|endoftext|>


<commit_before>package calcium

import (
	"bytes"
	"context"
	"sync"

	"github.com/projecteru2/core/cluster"
	enginetypes "github.com/projecteru2/core/engine/types"
	"github.com/projecteru2/core/types"
	log "github.com/sirupsen/logrus"
)

// ControlContainer control containers status
func (c *Calcium) ControlContainer(ctx context.Context, IDs []string, t string, force bool) (chan *types.ControlContainerMessage, error) {
	ch := make(chan *types.ControlContainerMessage)

	go func() {
		defer close(ch)
		wg := sync.WaitGroup{}
		for _, ID := range IDs {
			container, err := c.GetContainer(ctx, ID)
			if err != nil {
				ch <- &types.ControlContainerMessage{
					ContainerID: ID,
					Error:       err,
				}
				continue
			}

			wg.Add(1)
			go func(container *types.Container) {
				defer wg.Done()
				var err error
				var message []*bytes.Buffer
				defer func() {
					if err == nil {
						log.Infof("[ControlContainer] Control container %s %s", container.ID, t)
						log.Infof("[ControlContainer] Output:\n %s", string(types.HookOutput(message)))
					}
					ch <- &types.ControlContainerMessage{
						ContainerID: container.ID,
						Error:       err,
						Hook:        message,
					}
				}()

				containerInfo, err := container.Inspect(ctx)
				if err != nil {
					return
				}

				switch t {
				case cluster.ContainerStop:
					message, err = c.doStopContainer(ctx, container, containerInfo, force)
					return
				case cluster.ContainerStart:
					message, err = c.doStartContainer(ctx, container, containerInfo, force)
					return
				case cluster.ContainerRestart:
					message, err = c.doStopContainer(ctx, container, containerInfo, force)
					if err != nil {
						return
					}
					m2, e2 := c.doStartContainer(ctx, container, containerInfo, force)
					message = append(message, m2...)
					if e2 != nil {
						err = e2
					}
					return
				default:
					err = types.ErrUnknownControlType
				}
			}(container)
		}
		wg.Wait()
	}()

	return ch, nil
}

func (c *Calcium) doStartContainer(ctx context.Context, container *types.Container, containerInfo *enginetypes.VirtualizationInfo, force bool) ([]*bytes.Buffer, error) {
	var message []*bytes.Buffer
	var err error

	startCtx, cancel := context.WithTimeout(ctx, c.config.GlobalTimeout)
	defer cancel()
	if err = container.Start(startCtx); err != nil {
		return message, err
	}
	// TODO healthcheck
	if container.Hook != nil && len(container.Hook.AfterStart) > 0 {
		message, err = c.doHook(
			ctx,
			container.ID, containerInfo.User,
			container.Hook.AfterStart, containerInfo.Env,
			container.Hook.Force, container.Privileged,
			force, container.Engine,
		)
	}
	return message, err
}

func (c *Calcium) doStopContainer(ctx context.Context, container *types.Container, containerInfo *enginetypes.VirtualizationInfo, force bool) ([]*bytes.Buffer, error) {
	var message []*bytes.Buffer
	var err error

	if container.Hook != nil && len(container.Hook.BeforeStop) > 0 {
		message, err = c.doHook(
			ctx,
			container.ID, containerInfo.User,
			container.Hook.BeforeStop, containerInfo.Env,
			container.Hook.Force, container.Privileged,
			force, container.Engine,
		)
		if err != nil {
			return message, err
		}
	}

	// 这里 block 的问题很严重，按照目前的配置是 5 分钟一级的 block
	// 一个简单的处理方法是相信 ctx 不相信 engine 自身的处理
	// 另外我怀疑 engine 自己的 timeout 实现是完全的等 timeout 而非结束了就退出
	stopCtx, cancel := context.WithTimeout(ctx, c.config.GlobalTimeout)
	defer cancel()
	if err = container.Stop(stopCtx); err != nil {
		message = append(message, bytes.NewBufferString(err.Error()))
	}
	return message, err
}
<commit_msg>inspect container beforce control it<commit_after>package calcium

import (
	"bytes"
	"context"
	"sync"

	"github.com/projecteru2/core/cluster"
	enginetypes "github.com/projecteru2/core/engine/types"
	"github.com/projecteru2/core/types"
	log "github.com/sirupsen/logrus"
)

// ControlContainer control containers status
func (c *Calcium) ControlContainer(ctx context.Context, IDs []string, t string, force bool) (chan *types.ControlContainerMessage, error) {
	ch := make(chan *types.ControlContainerMessage)

	go func() {
		defer close(ch)
		wg := sync.WaitGroup{}
		for _, ID := range IDs {
			container, err := c.GetContainer(ctx, ID)
			if err != nil {
				ch <- &types.ControlContainerMessage{
					ContainerID: ID,
					Error:       err,
				}
				continue
			}

			wg.Add(1)
			go func(container *types.Container) {
				defer wg.Done()
				var err error
				var message []*bytes.Buffer
				defer func() {
					if err == nil {
						log.Infof("[ControlContainer] Control container %s %s", container.ID, t)
						log.Infof("[ControlContainer] Output:\n %s", string(types.HookOutput(message)))
					}
					ch <- &types.ControlContainerMessage{
						ContainerID: container.ID,
						Error:       err,
						Hook:        message,
					}
				}()

				containerInfo, err := container.Inspect(ctx)
				if err != nil {
					return
				}

				switch t {
				case cluster.ContainerStop:
					message, err = c.doStopContainer(ctx, container, containerInfo, force)
					return
				case cluster.ContainerStart:
					message, err = c.doStartContainer(ctx, container, containerInfo, force)
					return
				case cluster.ContainerRestart:
					if containerInfo, err := container.Inspect(ctx); err == nil && containerInfo.Running {
						message, err = c.doStopContainer(ctx, container, containerInfo, force)
						if err != nil {
							return
						}

					} else if err != nil {
						log.Errorf("[ControlContainer] Inspect container %s failed %v", container.ID, err)
					} else if !containerInfo.Running {
						message = append(message, bytes.NewBufferString("container stopped, can't run hook\n"))
					}
					m2, e2 := c.doStartContainer(ctx, container, containerInfo, force)
					message = append(message, m2...)
					if e2 != nil {
						err = e2
					}
					return
				default:
					err = types.ErrUnknownControlType
				}
			}(container)
		}
		wg.Wait()
	}()

	return ch, nil
}

func (c *Calcium) doStartContainer(ctx context.Context, container *types.Container, containerInfo *enginetypes.VirtualizationInfo, force bool) ([]*bytes.Buffer, error) {
	var message []*bytes.Buffer
	var err error

	startCtx, cancel := context.WithTimeout(ctx, c.config.GlobalTimeout)
	defer cancel()
	if err = container.Start(startCtx); err != nil {
		return message, err
	}
	// TODO healthcheck
	if container.Hook != nil && len(container.Hook.AfterStart) > 0 {
		message, err = c.doHook(
			ctx,
			container.ID, containerInfo.User,
			container.Hook.AfterStart, containerInfo.Env,
			container.Hook.Force, container.Privileged,
			force, container.Engine,
		)
	}
	return message, err
}

func (c *Calcium) doStopContainer(ctx context.Context, container *types.Container, containerInfo *enginetypes.VirtualizationInfo, force bool) ([]*bytes.Buffer, error) {
	var message []*bytes.Buffer
	var err error

	if container.Hook != nil && len(container.Hook.BeforeStop) > 0 {
		message, err = c.doHook(
			ctx,
			container.ID, containerInfo.User,
			container.Hook.BeforeStop, containerInfo.Env,
			container.Hook.Force, container.Privileged,
			force, container.Engine,
		)
		if err != nil {
			return message, err
		}
	}

	// 这里 block 的问题很严重，按照目前的配置是 5 分钟一级的 block
	// 一个简单的处理方法是相信 ctx 不相信 engine 自身的处理
	// 另外我怀疑 engine 自己的 timeout 实现是完全的等 timeout 而非结束了就退出
	stopCtx, cancel := context.WithTimeout(ctx, c.config.GlobalTimeout)
	defer cancel()
	if err = container.Stop(stopCtx); err != nil {
		message = append(message, bytes.NewBufferString(err.Error()))
	}
	return message, err
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/filegen/client"
	"github.com/Symantec/Dominator/lib/mdb"
	"github.com/Symantec/Dominator/lib/mdb/mdbd"
	"github.com/Symantec/Dominator/lib/objectserver/memory"
	"github.com/Symantec/Dominator/lib/srpc/setupclient"
	proto "github.com/Symantec/Dominator/proto/filegenerator"
	"io"
	"log"
	"os"
	"time"
)

var (
	benchmark = flag.Bool("benchmark", false,
		"If true, perform benchmark timing")
	debug = flag.Bool("debug", false,
		"If true, show debugging output")
	mdbFile = flag.String("mdbFile", "/var/lib/Dominator/mdb",
		"File to read MDB data from (default format is JSON)")

	numMachines int
)

func printUsage() {
	fmt.Fprintln(os.Stderr,
		"Usage: filegen-client [flags...] pathname source")
	fmt.Fprintln(os.Stderr, "Common flags:")
	flag.PrintDefaults()
}

func showMdb(mdb *mdb.Mdb) {
	mdb.DebugWrite(os.Stdout)
	fmt.Println()
}

type messageType struct {
	hostname  string
	fileInfos []proto.FileInfo
}

func benchmarkMessageHandler(messageChannel <-chan messageType) {
	numMessages := 0
	startTime := time.Now()
	for message := range messageChannel {
		numMessages++
		if numMessages == numMachines {
			fmt.Printf("Time taken: %s\n", time.Since(startTime))
		} else if numMessages > numMachines {
			fmt.Printf("Extra message for machine: %s\n", message.hostname)
		}
	}
}

func displayMessageHandler(messageChannel <-chan messageType,
	objSrv *memory.ObjectServer) {
	for message := range messageChannel {
		fmt.Printf("For machine: %s:\n", message.hostname)
		for _, fileInfo := range message.fileInfos {
			fmt.Printf("  pathname: %s\n    hash=%x\n    contents:\n",
				fileInfo.Pathname, fileInfo.Hash)
			if _, reader, err := objSrv.GetObject(fileInfo.Hash); err != nil {
				fmt.Println(err)
			} else {
				io.Copy(os.Stdout, reader)
				fmt.Println("\n-----------------------------------------------")
			}
		}
	}
}

func handleUpdates(hostname string, updateChannel <-chan []proto.FileInfo,
	messageChannel chan<- messageType) {
	for fileInfos := range updateChannel {
		messageChannel <- messageType{hostname, fileInfos}
	}
}

func main() {
	flag.Usage = printUsage
	flag.Parse()
	if flag.NArg() != 2 {
		printUsage()
		os.Exit(2)
	}
	if err := setupclient.SetupTls(true); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	objectServer := memory.NewObjectServer()
	logger := log.New(os.Stdout, "", log.LstdFlags)
	manager := client.New(objectServer, logger)
	mdbChannel := mdbd.StartMdbDaemon(*mdbFile, logger)
	machines := make(map[string]struct{})
	computedFiles := make([]client.ComputedFile, 1)
	computedFiles[0].Pathname = flag.Arg(0)
	computedFiles[0].Source = flag.Arg(1)
	messageChannel := make(chan messageType, 1)
	if *benchmark {
		go benchmarkMessageHandler(messageChannel)
	} else {
		go displayMessageHandler(messageChannel, objectServer)
	}
	for {
		select {
		case mdb := <-mdbChannel:
			if *debug {
				showMdb(mdb)
			}
			numMachines = len(mdb.Machines)
			machinesToDelete := make(map[string]struct{}, len(machines))
			for hostname := range machines {
				machinesToDelete[hostname] = struct{}{}
			}
			for _, mdbEntry := range mdb.Machines {
				delete(machinesToDelete, mdbEntry.Hostname)
				machine := client.Machine{mdbEntry, computedFiles}
				if _, ok := machines[mdbEntry.Hostname]; !ok {
					machines[mdbEntry.Hostname] = struct{}{}
					go handleUpdates(mdbEntry.Hostname, manager.Add(machine, 1),
						messageChannel)
				} else {
					manager.Update(client.Machine{mdbEntry, computedFiles})
				}
			}
			for hostname := range machinesToDelete {
				manager.Remove(hostname)
				delete(machines, hostname)
			}
		}
	}
}
<commit_msg>Change filegen-client to use lib/log/cmdlogger.<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/filegen/client"
	"github.com/Symantec/Dominator/lib/log/cmdlogger"
	"github.com/Symantec/Dominator/lib/mdb"
	"github.com/Symantec/Dominator/lib/mdb/mdbd"
	"github.com/Symantec/Dominator/lib/objectserver/memory"
	"github.com/Symantec/Dominator/lib/srpc/setupclient"
	proto "github.com/Symantec/Dominator/proto/filegenerator"
	"io"
	"os"
	"time"
)

var (
	benchmark = flag.Bool("benchmark", false,
		"If true, perform benchmark timing")
	debug = flag.Bool("debug", false,
		"If true, show debugging output")
	mdbFile = flag.String("mdbFile", "/var/lib/Dominator/mdb",
		"File to read MDB data from (default format is JSON)")

	numMachines int
)

func printUsage() {
	fmt.Fprintln(os.Stderr,
		"Usage: filegen-client [flags...] pathname source")
	fmt.Fprintln(os.Stderr, "Common flags:")
	flag.PrintDefaults()
}

func showMdb(mdb *mdb.Mdb) {
	mdb.DebugWrite(os.Stdout)
	fmt.Println()
}

type messageType struct {
	hostname  string
	fileInfos []proto.FileInfo
}

func benchmarkMessageHandler(messageChannel <-chan messageType) {
	numMessages := 0
	startTime := time.Now()
	for message := range messageChannel {
		numMessages++
		if numMessages == numMachines {
			fmt.Printf("Time taken: %s\n", time.Since(startTime))
		} else if numMessages > numMachines {
			fmt.Printf("Extra message for machine: %s\n", message.hostname)
		}
	}
}

func displayMessageHandler(messageChannel <-chan messageType,
	objSrv *memory.ObjectServer) {
	for message := range messageChannel {
		fmt.Printf("For machine: %s:\n", message.hostname)
		for _, fileInfo := range message.fileInfos {
			fmt.Printf("  pathname: %s\n    hash=%x\n    contents:\n",
				fileInfo.Pathname, fileInfo.Hash)
			if _, reader, err := objSrv.GetObject(fileInfo.Hash); err != nil {
				fmt.Println(err)
			} else {
				io.Copy(os.Stdout, reader)
				fmt.Println("\n-----------------------------------------------")
			}
		}
	}
}

func handleUpdates(hostname string, updateChannel <-chan []proto.FileInfo,
	messageChannel chan<- messageType) {
	for fileInfos := range updateChannel {
		messageChannel <- messageType{hostname, fileInfos}
	}
}

func main() {
	flag.Usage = printUsage
	flag.Parse()
	if flag.NArg() != 2 {
		printUsage()
		os.Exit(2)
	}
	if err := setupclient.SetupTls(true); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	objectServer := memory.NewObjectServer()
	logger := cmdlogger.New()
	manager := client.New(objectServer, logger)
	mdbChannel := mdbd.StartMdbDaemon(*mdbFile, logger)
	machines := make(map[string]struct{})
	computedFiles := make([]client.ComputedFile, 1)
	computedFiles[0].Pathname = flag.Arg(0)
	computedFiles[0].Source = flag.Arg(1)
	messageChannel := make(chan messageType, 1)
	if *benchmark {
		go benchmarkMessageHandler(messageChannel)
	} else {
		go displayMessageHandler(messageChannel, objectServer)
	}
	for {
		select {
		case mdb := <-mdbChannel:
			if *debug {
				showMdb(mdb)
			}
			numMachines = len(mdb.Machines)
			machinesToDelete := make(map[string]struct{}, len(machines))
			for hostname := range machines {
				machinesToDelete[hostname] = struct{}{}
			}
			for _, mdbEntry := range mdb.Machines {
				delete(machinesToDelete, mdbEntry.Hostname)
				machine := client.Machine{mdbEntry, computedFiles}
				if _, ok := machines[mdbEntry.Hostname]; !ok {
					machines[mdbEntry.Hostname] = struct{}{}
					go handleUpdates(mdbEntry.Hostname, manager.Add(machine, 1),
						messageChannel)
				} else {
					manager.Update(client.Machine{mdbEntry, computedFiles})
				}
			}
			for hostname := range machinesToDelete {
				manager.Remove(hostname)
				delete(machines, hostname)
			}
		}
	}
}
<|endoftext|>


<commit_before>// jekyllbot is the server which controls @jekyllbot on GitHub.
package main

import (
	"flag"
	"log"
	"net/http"

	"github.com/google/go-github/github"
	"github.com/parkr/auto-reply/ctx"
	"github.com/parkr/auto-reply/hooks"
	"github.com/parkr/auto-reply/jekyll"
)

var context *ctx.Context

func main() {
	var port string
	flag.StringVar(&port, "port", "8080", "The port to serve to")
	flag.Parse()
	context = ctx.NewDefaultContext()

	http.HandleFunc("/_ping", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/plain")
		w.Write([]byte("ok\n"))
	}))

	jekyllOrgHandler := jekyll.NewJekyllOrgHandler(context)
	http.Handle("/_github/jekyll", jekyllOrgHandler)

	log.Printf("Listening on :%s", port)
	log.Fatal(http.ListenAndServe(":"+port, nil))
}

func verifyPayload(secret []byte, handler hooks.HookHandler) http.HandlerFunc {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/plain")
		payload, err := github.ValidatePayload(r, secret)
		if err != nil {
			log.Println("received invalid signature:", err)
			http.Error(w, err.Error(), http.StatusForbidden)
			return
		}
		handler.HandlePayload(w, r, payload)
	})
}
<commit_msg>cmd/jekyllbot: remove unused method<commit_after>// jekyllbot is the server which controls @jekyllbot on GitHub.
package main

import (
	"flag"
	"log"
	"net/http"

	"github.com/parkr/auto-reply/ctx"
	"github.com/parkr/auto-reply/jekyll"
)

var context *ctx.Context

func main() {
	var port string
	flag.StringVar(&port, "port", "8080", "The port to serve to")
	flag.Parse()
	context = ctx.NewDefaultContext()

	http.HandleFunc("/_ping", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/plain")
		w.Write([]byte("ok\n"))
	}))

	jekyllOrgHandler := jekyll.NewJekyllOrgHandler(context)
	http.Handle("/_github/jekyll", jekyllOrgHandler)

	log.Printf("Listening on :%s", port)
	log.Fatal(http.ListenAndServe(":"+port, nil))
}
<|endoftext|>


<commit_before>// Command juggler-callee implements a testing callee that provides
// simple URI functions:
//     - test.echo (string) : returns the received string
//     - test.reverse (string) : reverses each rune in the received string
//     - test.delay (string) : sleeps for the duration received as string, converted to number (in ms)
//
package main

import (
	"encoding/json"
	"log"
	"strconv"
	"time"

	"github.com/PuerkitoBio/exp/juggler/broker/redisbroker"
	"github.com/PuerkitoBio/exp/juggler/callee"
	"github.com/PuerkitoBio/exp/juggler/msg"
	"github.com/garyburd/redigo/redis"
)

func main() {
	pool := newRedisPool(":6379")
	broker := &redisbroker.Broker{
		Pool:      pool,
		Dial:      pool.Dial,
		ResultCap: 100,
	}

	c := &callee.Callee{Broker: broker}

	log.Printf("listening for call requests")
	if err := c.Listen(map[string]callee.Thunk{
		"test.echo":    logWrapThunk(echoThunk),
		"test.reverse": logWrapThunk(reverseThunk),
		"test.delay":   logWrapThunk(delayThunk),
	}); err != nil {
		log.Fatalf("Listen failed: %v", err)
	}
}

func logWrapThunk(t callee.Thunk) callee.Thunk {
	return func(cp *msg.CallPayload) (interface{}, error) {
		log.Printf("received call for %s from %v", cp.URI, cp.MsgUUID)
		v, err := t(cp)
		log.Printf("sending result for %s from %v", cp.URI, cp.MsgUUID)
		return v, err
	}
}

func delayThunk(cp *msg.CallPayload) (interface{}, error) {
	var s string
	if err := json.Unmarshal(cp.Args, &s); err != nil {
		return nil, err
	}
	i, err := strconv.Atoi(s)
	if err != nil {
		return nil, err
	}
	return delay(i), nil
}

func delay(i int) int {
	time.Sleep(time.Duration(i) * time.Millisecond)
	return i
}

func reverseThunk(cp *msg.CallPayload) (interface{}, error) {
	var s string
	if err := json.Unmarshal(cp.Args, &s); err != nil {
		return nil, err
	}
	return reverse(s), nil
}

func reverse(s string) string {
	chars := []rune(s)
	for i, j := 0, len(chars)-1; i < j; i, j = i+1, j-1 {
		chars[i], chars[j] = chars[j], chars[i]
	}
	return string(chars)
}

func echoThunk(cp *msg.CallPayload) (interface{}, error) {
	var s string
	if err := json.Unmarshal(cp.Args, &s); err != nil {
		return nil, err
	}
	return echo(s), nil
}

func echo(s string) string {
	return s
}

func newRedisPool(addr string) *redis.Pool {
	return &redis.Pool{
		MaxIdle:     10,
		MaxActive:   10000,
		IdleTimeout: 2 * time.Minute,
		Dial: func() (redis.Conn, error) {
			c, err := redis.Dial("tcp", addr)
			if err != nil {
				return nil, err
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}
<commit_msg>juggler/cmd: fix juggler-callee package comment<commit_after>// Command juggler-callee implements a testing callee that provides
// simple URI functions.
//
//     - test.echo (string) : returns the received string
//     - test.reverse (string) : reverses each rune in the received string
//     - test.delay (string) : sleeps for the duration received as string, converted to number (in ms)
//
package main

import (
	"encoding/json"
	"log"
	"strconv"
	"time"

	"github.com/PuerkitoBio/exp/juggler/broker/redisbroker"
	"github.com/PuerkitoBio/exp/juggler/callee"
	"github.com/PuerkitoBio/exp/juggler/msg"
	"github.com/garyburd/redigo/redis"
)

func main() {
	pool := newRedisPool(":6379")
	broker := &redisbroker.Broker{
		Pool:      pool,
		Dial:      pool.Dial,
		ResultCap: 100,
	}

	c := &callee.Callee{Broker: broker}

	log.Printf("listening for call requests")
	if err := c.Listen(map[string]callee.Thunk{
		"test.echo":    logWrapThunk(echoThunk),
		"test.reverse": logWrapThunk(reverseThunk),
		"test.delay":   logWrapThunk(delayThunk),
	}); err != nil {
		log.Fatalf("Listen failed: %v", err)
	}
}

func logWrapThunk(t callee.Thunk) callee.Thunk {
	return func(cp *msg.CallPayload) (interface{}, error) {
		log.Printf("received call for %s from %v", cp.URI, cp.MsgUUID)
		v, err := t(cp)
		log.Printf("sending result for %s from %v", cp.URI, cp.MsgUUID)
		return v, err
	}
}

func delayThunk(cp *msg.CallPayload) (interface{}, error) {
	var s string
	if err := json.Unmarshal(cp.Args, &s); err != nil {
		return nil, err
	}
	i, err := strconv.Atoi(s)
	if err != nil {
		return nil, err
	}
	return delay(i), nil
}

func delay(i int) int {
	time.Sleep(time.Duration(i) * time.Millisecond)
	return i
}

func reverseThunk(cp *msg.CallPayload) (interface{}, error) {
	var s string
	if err := json.Unmarshal(cp.Args, &s); err != nil {
		return nil, err
	}
	return reverse(s), nil
}

func reverse(s string) string {
	chars := []rune(s)
	for i, j := 0, len(chars)-1; i < j; i, j = i+1, j-1 {
		chars[i], chars[j] = chars[j], chars[i]
	}
	return string(chars)
}

func echoThunk(cp *msg.CallPayload) (interface{}, error) {
	var s string
	if err := json.Unmarshal(cp.Args, &s); err != nil {
		return nil, err
	}
	return echo(s), nil
}

func echo(s string) string {
	return s
}

func newRedisPool(addr string) *redis.Pool {
	return &redis.Pool{
		MaxIdle:     10,
		MaxActive:   10000,
		IdleTimeout: 2 * time.Minute,
		Dial: func() (redis.Conn, error) {
			c, err := redis.Dial("tcp", addr)
			if err != nil {
				return nil, err
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"path/filepath"
	"runtime"

	"github.com/openshift/origin/pkg/cmd/openshift"
	"github.com/openshift/origin/pkg/cmd/util/serviceability"
)

func main() {
	defer serviceability.BehaviorOnPanic(os.Getenv("OPENSHIFT_ON_PANIC"))()
	defer serviceability.Profile(os.Getenv("OPENSHIFT_PROFILE")).Stop()

	if len(os.Getenv("GOMAXPROCS")) == 0 {
		runtime.GOMAXPROCS(runtime.NumCPU())
	}

	basename := filepath.Base(os.Args[0])
	command := openshift.CommandFor(basename)
	if err := command.Execute(); err != nil {
		os.Exit(1)
	}
}
<commit_msg>Introducing syntax error for infra testing<commit_after>package main

import (
	"os"
	"path/filepath"
	"runtime"

	"github.com/openshift/origin/pkg/cmd/openshift"
	"github.com/openshift/origin/pkg/cmd/util/serviceability"
)

unc main() {
	defer serviceability.BehaviorOnPanic(os.Getenv("OPENSHIFT_ON_PANIC"))()
	defer serviceability.Profile(os.Getenv("OPENSHIFT_PROFILE")).Stop()

	if len(os.Getenv("GOMAXPROCS")) == 0 {
		runtime.GOMAXPROCS(runtime.NumCPU())
	}

	basename := filepath.Base(os.Args[0])
	command := openshift.CommandFor(basename)
	if err := command.Execute(); err != nil {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package geo_skeleton

import "fmt"

import "github.com/sjsafranek/DiffDB/diff_store"
import "github.com/sjsafranek/DiffDB/diff_db"

var diffDb diff_db.DiffDb

func init() {
	diffDb = diff_db.NewDiffDb("diff.db")
	diffDb.Init()
}

func update_timeseries_datasource(datasource_id string, value []byte) {

	update_value := string(value)
	var ddata diff_store.DiffStore
	data, err := diffDb.Load(datasource_id)
	if nil != err {
		if err.Error() == "Not found" {
			// create new diffstore if key not found in database
			ddata = diff_store.NewDiffStore(datasource_id)
		} else {
			panic(err)
		}
	} else {
		ddata.Decode(data)
	}

	// update diffstore
	ddata.Update(update_value)

	// save to database
	enc, err := ddata.Encode()
	if nil != err {
		panic(err)
	}

	err = diffDb.Save(ddata.Name, enc)
	fmt.Println(err)

}
<commit_msg>diff db added<commit_after>package geo_skeleton

import "fmt"

import "github.com/sjsafranek/DiffDB/diff_store"
import "github.com/sjsafranek/DiffDB/diff_db"

var diffDb diff_db.DiffDb

func init() {
	diffDb = diff_db.NewDiffDb("diff.db")
	diffDb.Init()
}

func update_timeseries_datasource(datasource_id string, value []byte) {

	update_value := string(value)
	var ddata diff_store.DiffStore
	data, err := diffDb.Load(datasource_id)
	if nil != err {
		if err.Error() == "Not found" {
			// create new diffstore if key not found in database
			ddata = diff_store.NewDiffStore(datasource_id)
		} else {
			panic(err)
		}
	} else {
		ddata.Decode(data)
	}

	// update diffstore
	ddata.Update(update_value)

	// save to database
	enc, err := ddata.Encode()
	if nil != err {
		panic(err)
	}

	err = diffDb.Save(ddata.Name, enc)
	fmt.Println(datasource_id, ddata.Name, err)

}
<|endoftext|>


<commit_before>package rancher

import (
	"context"
	"io/ioutil"
	"os"
	"sync"

	"github.com/rancher/rancher/pkg/agent/cluster"
	"github.com/rancher/rancher/pkg/features"
	"github.com/rancher/rancher/pkg/namespace"
	"github.com/rancher/rancher/pkg/rancher"
	"github.com/rancher/wrangler/pkg/apply"
	corefactory "github.com/rancher/wrangler/pkg/generated/controllers/core"
	corecontrollers "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
	"github.com/rancher/wrangler/pkg/kubeconfig"
	"github.com/sirupsen/logrus"
	corev1 "k8s.io/api/core/v1"
	apierror "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/rest"
)

func Run(ctx context.Context) error {
	if err := setupSteveAggregation(); err != nil {
		return err
	}

	if !features.MCMAgent.Enabled() {
		return nil
	}

	cfg, err := kubeconfig.GetNonInteractiveClientConfig("").ClientConfig()
	if err != nil {
		return err
	}

	core, err := corefactory.NewFactoryFromConfig(cfg)
	if err != nil {
		return err
	}

	h := handler{
		ctx:          ctx,
		serviceCache: core.Core().V1().Service().Cache(),
	}

	core.Core().V1().Service().OnChange(ctx, "rancher-installed", h.OnChange)
	return core.Start(ctx, 1)
}

type handler struct {
	lock            sync.Mutex
	ctx             context.Context
	rancherNotFound *bool
	serviceCache    corecontrollers.ServiceCache
}

func (h *handler) startRancher() {
	clientConfig := kubeconfig.GetNonInteractiveClientConfig("")
	server, err := rancher.New(h.ctx, clientConfig, &rancher.Options{
		HTTPListenPort:  80,
		HTTPSListenPort: 443,
		Features:        os.Getenv("CATTLE_FEATURES"),
		AddLocal:        "true",
	})
	if err != nil {
		logrus.Fatalf("Embedded rancher failed to initialize: %v", err)
	}
	go func() {
		err = server.ListenAndServe(h.ctx)
		logrus.Fatalf("Embedded rancher failed to start: %v", err)
	}()
}

func (h *handler) OnChange(key string, service *corev1.Service) (*corev1.Service, error) {
	h.lock.Lock()
	defer h.lock.Unlock()
	if h.rancherNotFound == nil {
		_, err := h.serviceCache.Get(namespace.System, "rancher")
		if notFound := apierror.IsNotFound(err); notFound {
			h.rancherNotFound = &notFound
			h.startRancher()
		} else if err != nil {
			return nil, err
		} else {
			h.rancherNotFound = &notFound
		}
	}

	if service == nil {
		if key == namespace.System+"/rancher" {
			logrus.Info("Rancher has been uninstalled, restarting")
			os.Exit(0)
		}
	} else if service.Namespace == namespace.System && service.Name == "rancher" && *h.rancherNotFound {
		logrus.Info("Rancher has been installed, restarting")
		os.Exit(0)
	}

	return service, nil
}

func setupSteveAggregation() error {
	c, err := rest.InClusterConfig()
	if err != nil {
		return err
	}

	apply, err := apply.NewForConfig(c)
	if err != nil {
		return err
	}

	token, url, err := cluster.TokenAndURL()
	if err != nil {
		return err
	}

	data := map[string][]byte{
		"CATTLE_SERVER":      []byte(url),
		"CATTLE_TOKEN":       []byte(token),
		"CATTLE_CA_CHECKSUM": []byte(cluster.CAChecksum()),
		"url":                []byte(url + "/v3/connect"),
		"token":              []byte("stv-cluster-" + token),
	}

	ca, err := ioutil.ReadFile("/etc/kubernetes/ssl/certs/serverca")
	if os.IsNotExist(err) {
	} else if err != nil {
		return err
	} else {
		data["ca.crt"] = ca
	}

	return apply.
		WithDynamicLookup().
		WithSetID("rancher-stv-aggregation").
		WithListerNamespace(namespace.System).
		ApplyObjects(&corev1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Namespace: namespace.System,
				Name:      "stv-aggregation",
			},
			Data: data,
		})
}
<commit_msg>Don't start another embedded rancher on reconnect<commit_after>package rancher

import (
	"context"
	"io/ioutil"
	"os"
	"sync"

	"github.com/rancher/rancher/pkg/agent/cluster"
	"github.com/rancher/rancher/pkg/features"
	"github.com/rancher/rancher/pkg/namespace"
	"github.com/rancher/rancher/pkg/rancher"
	"github.com/rancher/wrangler/pkg/apply"
	corefactory "github.com/rancher/wrangler/pkg/generated/controllers/core"
	corecontrollers "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
	"github.com/rancher/wrangler/pkg/kubeconfig"
	"github.com/sirupsen/logrus"
	corev1 "k8s.io/api/core/v1"
	apierror "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/rest"
)

var (
	started bool
)

func Run(ctx context.Context) error {
	if err := setupSteveAggregation(); err != nil {
		return err
	}

	if started {
		return nil
	}

	if !features.MCMAgent.Enabled() {
		return nil
	}

	cfg, err := kubeconfig.GetNonInteractiveClientConfig("").ClientConfig()
	if err != nil {
		return err
	}

	core, err := corefactory.NewFactoryFromConfig(cfg)
	if err != nil {
		return err
	}

	h := handler{
		ctx:          ctx,
		serviceCache: core.Core().V1().Service().Cache(),
	}

	core.Core().V1().Service().OnChange(ctx, "rancher-installed", h.OnChange)
	if err := core.Start(ctx, 1); err != nil {
		return err
	}
	started = true
	return nil
}

type handler struct {
	lock            sync.Mutex
	ctx             context.Context
	rancherNotFound *bool
	serviceCache    corecontrollers.ServiceCache
}

func (h *handler) startRancher() {
	clientConfig := kubeconfig.GetNonInteractiveClientConfig("")
	server, err := rancher.New(h.ctx, clientConfig, &rancher.Options{
		HTTPListenPort:  80,
		HTTPSListenPort: 443,
		Features:        os.Getenv("CATTLE_FEATURES"),
		AddLocal:        "true",
	})
	if err != nil {
		logrus.Fatalf("Embedded rancher failed to initialize: %v", err)
	}
	go func() {
		err = server.ListenAndServe(h.ctx)
		logrus.Fatalf("Embedded rancher failed to start: %v", err)
	}()
}

func (h *handler) OnChange(key string, service *corev1.Service) (*corev1.Service, error) {
	h.lock.Lock()
	defer h.lock.Unlock()
	if h.rancherNotFound == nil {
		_, err := h.serviceCache.Get(namespace.System, "rancher")
		if notFound := apierror.IsNotFound(err); notFound {
			h.rancherNotFound = &notFound
			h.startRancher()
		} else if err != nil {
			return nil, err
		} else {
			h.rancherNotFound = &notFound
		}
	}

	if service == nil {
		if key == namespace.System+"/rancher" {
			logrus.Info("Rancher has been uninstalled, restarting")
			os.Exit(0)
		}
	} else if service.Namespace == namespace.System && service.Name == "rancher" && *h.rancherNotFound {
		logrus.Info("Rancher has been installed, restarting")
		os.Exit(0)
	}

	return service, nil
}

func setupSteveAggregation() error {
	c, err := rest.InClusterConfig()
	if err != nil {
		return err
	}

	apply, err := apply.NewForConfig(c)
	if err != nil {
		return err
	}

	token, url, err := cluster.TokenAndURL()
	if err != nil {
		return err
	}

	data := map[string][]byte{
		"CATTLE_SERVER":      []byte(url),
		"CATTLE_TOKEN":       []byte(token),
		"CATTLE_CA_CHECKSUM": []byte(cluster.CAChecksum()),
		"url":                []byte(url + "/v3/connect"),
		"token":              []byte("stv-cluster-" + token),
	}

	ca, err := ioutil.ReadFile("/etc/kubernetes/ssl/certs/serverca")
	if os.IsNotExist(err) {
	} else if err != nil {
		return err
	} else {
		data["ca.crt"] = ca
	}

	return apply.
		WithDynamicLookup().
		WithSetID("rancher-stv-aggregation").
		WithListerNamespace(namespace.System).
		ApplyObjects(&corev1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Namespace: namespace.System,
				Name:      "stv-aggregation",
			},
			Data: data,
		})
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package blobserver

import (
	"camlistore.org/pkg/blob"
	"camlistore.org/pkg/context"
)

const buffered = 8

// MergedEnumerate implements the BlobEnumerator interface by
// merge-joining 0 or more sources.
func MergedEnumerate(ctx *context.Context, dest chan<- blob.SizedRef, sources []BlobEnumerator, after string, limit int) error {
	return mergedEnumerate(ctx, dest, len(sources), func(i int) BlobEnumerator { return sources[i] }, after, limit)
}

// MergedEnumerateStorage implements the BlobEnumerator interface by
// merge-joining 0 or more sources.
//
// In this version, the sources implement the Storage interface, even
// though only the BlobEnumerator interface is used.
func MergedEnumerateStorage(ctx *context.Context, dest chan<- blob.SizedRef, sources []Storage, after string, limit int) error {
	return mergedEnumerate(ctx, dest, len(sources), func(i int) BlobEnumerator { return sources[i] }, after, limit)
}

func mergedEnumerate(ctx *context.Context, dest chan<- blob.SizedRef, nsrc int, getSource func(int) BlobEnumerator, after string, limit int) error {
	defer close(dest)

	startEnum := func(source BlobEnumerator) (*blob.ChanPeeker, <-chan error) {
		ch := make(chan blob.SizedRef, buffered)
		errch := make(chan error, 1)
		go func() {
			errch <- source.EnumerateBlobs(ctx, ch, after, limit)
		}()
		return &blob.ChanPeeker{Ch: ch}, errch
	}

	peekers := make([]*blob.ChanPeeker, 0, nsrc)
	errs := make([]<-chan error, 0, nsrc)
	for i := 0; i < nsrc; i++ {
		peeker, errch := startEnum(getSource(i))
		peekers = append(peekers, peeker)
		errs = append(errs, errch)
	}

	nSent := 0
	lastSent := ""
	for nSent < limit {
		lowestIdx := -1
		var lowest blob.SizedRef
		for idx, peeker := range peekers {
			for !peeker.Closed() && peeker.MustPeek().Ref.String() <= lastSent {
				peeker.Take()
			}
			if peeker.Closed() {
				continue
			}
			sb := peeker.MustPeek()                                       // can't be nil if not Closed
			if lowestIdx == -1 || sb.Ref.String() < lowest.Ref.String() { // TODO: add cheaper Ref comparison function, avoiding String
				lowestIdx = idx
				lowest = sb
			}
		}
		if lowestIdx == -1 {
			// all closed
			break
		}

		dest <- lowest
		nSent++
		lastSent = lowest.Ref.String()
	}

	// Once we've gotten enough, ignore the rest of whatever's
	// coming in.
	for _, peeker := range peekers {
		go peeker.ConsumeAll()
	}

	// If any part returns an error, we return an error.
	var retErr error
	for _, errch := range errs {
		if err := <-errch; err != nil {
			retErr = err
		}
	}
	return retErr
}
<commit_msg>blobserver: used blob.Ref.Less in MergedEnumerate<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package blobserver

import (
	"camlistore.org/pkg/blob"
	"camlistore.org/pkg/context"
)

const buffered = 8

// MergedEnumerate implements the BlobEnumerator interface by
// merge-joining 0 or more sources.
func MergedEnumerate(ctx *context.Context, dest chan<- blob.SizedRef, sources []BlobEnumerator, after string, limit int) error {
	return mergedEnumerate(ctx, dest, len(sources), func(i int) BlobEnumerator { return sources[i] }, after, limit)
}

// MergedEnumerateStorage implements the BlobEnumerator interface by
// merge-joining 0 or more sources.
//
// In this version, the sources implement the Storage interface, even
// though only the BlobEnumerator interface is used.
func MergedEnumerateStorage(ctx *context.Context, dest chan<- blob.SizedRef, sources []Storage, after string, limit int) error {
	return mergedEnumerate(ctx, dest, len(sources), func(i int) BlobEnumerator { return sources[i] }, after, limit)
}

func mergedEnumerate(ctx *context.Context, dest chan<- blob.SizedRef, nsrc int, getSource func(int) BlobEnumerator, after string, limit int) error {
	defer close(dest)

	startEnum := func(source BlobEnumerator) (*blob.ChanPeeker, <-chan error) {
		ch := make(chan blob.SizedRef, buffered)
		errch := make(chan error, 1)
		go func() {
			errch <- source.EnumerateBlobs(ctx, ch, after, limit)
		}()
		return &blob.ChanPeeker{Ch: ch}, errch
	}

	peekers := make([]*blob.ChanPeeker, 0, nsrc)
	errs := make([]<-chan error, 0, nsrc)
	for i := 0; i < nsrc; i++ {
		peeker, errch := startEnum(getSource(i))
		peekers = append(peekers, peeker)
		errs = append(errs, errch)
	}

	nSent := 0
	lastSent := ""
	for nSent < limit {
		lowestIdx := -1
		var lowest blob.SizedRef
		for idx, peeker := range peekers {
			for !peeker.Closed() && peeker.MustPeek().Ref.String() <= lastSent {
				peeker.Take()
			}
			if peeker.Closed() {
				continue
			}
			sb := peeker.MustPeek() // can't be nil if not Closed
			if lowestIdx == -1 || sb.Ref.Less(lowest.Ref) {
				lowestIdx = idx
				lowest = sb
			}
		}
		if lowestIdx == -1 {
			// all closed
			break
		}

		dest <- lowest
		nSent++
		lastSent = lowest.Ref.String()
	}

	// Once we've gotten enough, ignore the rest of whatever's
	// coming in.
	for _, peeker := range peekers {
		go peeker.ConsumeAll()
	}

	// If any part returns an error, we return an error.
	var retErr error
	for _, errch := range errs {
		if err := <-errch; err != nil {
			retErr = err
		}
	}
	return retErr
}
<|endoftext|>


<commit_before>package clideployment

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"git.containerum.net/ch/kube-client/pkg/model"
	"github.com/containerum/chkit/pkg/context"
	"github.com/containerum/chkit/pkg/model/container"
	"github.com/containerum/chkit/pkg/model/deployment"
	"github.com/containerum/chkit/pkg/model/deployment/deplactive"
	"github.com/containerum/chkit/pkg/util/activekit"
	"github.com/containerum/chkit/pkg/util/angel"
	"github.com/containerum/chkit/pkg/util/namegen"
	"github.com/containerum/chkit/pkg/util/pairs"
	"github.com/containerum/chkit/pkg/util/text"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

func Create(ctx *context.Context) *cobra.Command {
	var file string
	var force bool
	var flagCont container.Container
	var flagDepl deployment.Deployment
	var envs string
	command := &cobra.Command{
		Use:     "deployment",
		Aliases: aliases,
		Short:   "create new deployment",
		Long: `Creates new deployment.
Has an one-line mode, suitable for integration with other tools, and an interactive wizard mode`,
		Run: func(cmd *cobra.Command, args []string) {
			depl := deplactive.DefaultDeployment()
			if cmd.Flag("file").Changed {
				var err error
				depl, err = deplactive.FromFile(file)
				if err != nil {
					logrus.WithError(err).Errorf("unable to load deployment data from file %s", file)
					fmt.Printf("Unable to load deployment data from file :(\n%v", err)
					os.Exit(1)
				}
			} else if cmd.Flag("force").Changed {
				if cmd.Flag("env").Changed {
					envMap, err := pairs.ParseMap(envs, ":")
					if err != nil {
						fmt.Printf("invalid env flag\n")
						os.Exit(1)
					}
					for k, v := range envMap {
						flagCont.Env = append(flagCont.Env, model.Env{
							Name:  k,
							Value: v,
						})
					}
				}
				if flagCont.Name == "" {
					flagCont.Name = namegen.Aster() + "-" + flagCont.Image
				}
				flagDepl.Containers = []container.Container{flagCont}
				depl = flagDepl
			}
			if cmd.Flag("force").Changed {
				if err := deplactive.ValidateDeployment(depl); err != nil {
					fmt.Println(err)
					os.Exit(1)
				}
				fmt.Println(depl.RenderTable())
				if err := ctx.Client.CreateDeployment(ctx.Namespace, depl); err != nil {
					fmt.Println(err)
					os.Exit(1)
				}
				fmt.Println("OK")
				return
			}
			depl, err := deplactive.Wizard(deplactive.Config{
				Deployment: &depl,
			})
			if err != nil {
				logrus.WithError(err).Errorf("unable to create deployment")
				fmt.Println(err)
				os.Exit(1)
			}
			if activekit.YesNo("Are you sure?") {
				if err := ctx.Client.CreateDeployment(ctx.Namespace, depl); err != nil {
					logrus.WithError(err).Errorf("unable to create deployment %q", depl.Name)
					activekit.Attention(err.Error())
					os.Exit(1)
				}
				fmt.Printf("Congratulations! Deployment %q created!\n", depl.Name)
			}
			for {
				_, err := (&activekit.Menu{
					Items: []*activekit.MenuItem{
						{
							Label: "Push changes to server",
							Action: func() error {
								if activekit.YesNo("Are you sure?") {
									err := ctx.Client.ReplaceDeployment(ctx.Namespace, depl)
									if err != nil {
										logrus.WithError(err).Errorf("unable to update deployment %q", depl.Name)
										fmt.Println(err)
										return nil
									}
									fmt.Printf("Congratulations! Deployment %q updated!\n", depl.Name)
								}
								return nil
							},
						},
						{
							Label: "Edit deployment",
							Action: func() error {
								var err error
								depl, err = deplactive.ReplaceWizard(deplactive.Config{
									Deployment: &depl,
								})
								if err != nil {
									logrus.WithError(err).Errorf("unable to create deployment")
									fmt.Println(err)
									os.Exit(1)
								}
								return nil
							},
						},
						{
							Label: "Print to terminal",
							Action: activekit.ActionWithErr(func() error {
								if data, err := depl.RenderYAML(); err != nil {
									return err
								} else {
									upBorders := strings.Repeat("_", text.Width(data))
									downBorders := strings.Repeat("_", text.Width(data))
									fmt.Printf("%s\n\n%s\n%s\n", upBorders, data, downBorders)
								}
								return nil
							}),
						},
						{
							Label: "Save to file",
							Action: func() error {
								filename, _ := activekit.AskLine("Print filename: ")
								data, err := depl.RenderJSON()
								if err != nil {
									return err
								}
								if err := ioutil.WriteFile(filename, []byte(data), os.ModePerm); err != nil {
									logrus.WithError(err).Errorf("unable to save deployment %q to file", depl.Name)
									fmt.Printf("Unable to save deployment to file :(\n%v", err)
									return nil
								}
								fmt.Printf("OK\n")
								return nil
							},
						},
						{
							Label: "Exit",
							Action: func() error {
								if yes, _ := activekit.Yes("Are you sure you want to exit?"); yes {
									os.Exit(0)
								}
								return nil
							},
						},
					},
				}).Run()
				if err != nil {
					logrus.WithError(err).Errorf("error while menu execution")
					angel.Angel(ctx, err)
					os.Exit(1)
				}
			}
		},
	}
	command.PersistentFlags().
		StringVar(&file, "file", "", "create deployment from file")
	command.PersistentFlags().
		BoolVarP(&force, "force", "f", false, "suppress confirmation")

	command.PersistentFlags().
		StringVar(&flagDepl.Name, "deployment-name", namegen.Color()+"-"+namegen.Aster(), "deployment name, optional")
	command.PersistentFlags().
		IntVar(&flagDepl.Replicas, "replicas", 1, "replicas, optional")
	command.PersistentFlags().
		StringVar(&flagCont.Name, "container-name", "", "container name, equal to image name by default")
	command.PersistentFlags().
		StringVar(&flagCont.Image, "image", "", "container image, required")
	command.PersistentFlags().
		UintVar(&flagCont.Limits.Memory, "memory", 256, "container memory limit im Mb, optional")
	command.PersistentFlags().
		UintVar(&flagCont.Limits.CPU, "cpu", 200, "container CPU limit in mCPU, optional")
	command.PersistentFlags().
		StringSliceVar(&flagCont.Commands, "commands", nil, "container commands")
	command.PersistentFlags().
		StringVar(&envs, "env", "", "container env variable in KEY0:VALUE0 KEY1:VALUE1 format")
	return command
}
<commit_msg>Add dynamic menu creation: on Confirm changes first menu item to "Push changes on server" on Yes and to "Create deployment on server" on No<commit_after>package clideployment

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"git.containerum.net/ch/kube-client/pkg/model"
	"github.com/containerum/chkit/pkg/context"
	"github.com/containerum/chkit/pkg/model/container"
	"github.com/containerum/chkit/pkg/model/deployment"
	"github.com/containerum/chkit/pkg/model/deployment/deplactive"
	"github.com/containerum/chkit/pkg/util/activekit"
	"github.com/containerum/chkit/pkg/util/namegen"
	"github.com/containerum/chkit/pkg/util/pairs"
	"github.com/containerum/chkit/pkg/util/text"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

func Create(ctx *context.Context) *cobra.Command {
	var file string
	var force bool
	var flagCont container.Container
	var flagDepl deployment.Deployment
	var envs string
	command := &cobra.Command{
		Use:     "deployment",
		Aliases: aliases,
		Short:   "create new deployment",
		Long: `Creates new deployment.
Has an one-line mode, suitable for integration with other tools, and an interactive wizard mode`,
		Run: func(cmd *cobra.Command, args []string) {
			depl := deplactive.DefaultDeployment()
			if cmd.Flag("file").Changed {
				var err error
				depl, err = deplactive.FromFile(file)
				if err != nil {
					logrus.WithError(err).Errorf("unable to load deployment data from file %s", file)
					fmt.Printf("Unable to load deployment data from file :(\n%v", err)
					os.Exit(1)
				}
			} else if cmd.Flag("force").Changed {
				if cmd.Flag("env").Changed {
					envMap, err := pairs.ParseMap(envs, ":")
					if err != nil {
						fmt.Printf("invalid env flag\n")
						os.Exit(1)
					}
					for k, v := range envMap {
						flagCont.Env = append(flagCont.Env, model.Env{
							Name:  k,
							Value: v,
						})
					}
				}
				if flagCont.Name == "" {
					flagCont.Name = namegen.Aster() + "-" + flagCont.Image
				}
				flagDepl.Containers = []container.Container{flagCont}
				depl = flagDepl
			}
			if cmd.Flag("force").Changed {
				if err := deplactive.ValidateDeployment(depl); err != nil {
					fmt.Println(err)
					os.Exit(1)
				}
				fmt.Println(depl.RenderTable())
				if err := ctx.Client.CreateDeployment(ctx.Namespace, depl); err != nil {
					fmt.Println(err)
					os.Exit(1)
				}
				fmt.Println("OK")
				return
			}
			depl, err := deplactive.Wizard(deplactive.Config{
				Deployment: &depl,
			})
			if err != nil {
				logrus.WithError(err).Errorf("unable to create deployment")
				fmt.Println(err)
				os.Exit(1)
			}

			var firstItem *activekit.MenuItem
			var created = false
			if activekit.YesNo("Are you sure?") {
				if err := ctx.Client.CreateDeployment(ctx.Namespace, depl); err != nil {
					logrus.WithError(err).Errorf("unable to create deployment %q", depl.Name)
					activekit.Attention(err.Error())
					os.Exit(1)
				}
				fmt.Printf("Congratulations! Deployment %q created!\n", depl.Name)
				created = true
				firstItem = &activekit.MenuItem{
					Label: "Push changes to server",
					Action: func() error {
						if activekit.YesNo("Are you sure?") {
							err := ctx.Client.ReplaceDeployment(ctx.Namespace, depl)
							if err != nil {
								logrus.WithError(err).Errorf("unable to update deployment %q", depl.Name)
								fmt.Println(err)
								return nil
							}
							fmt.Printf("Congratulations! Deployment %q updated!\n", depl.Name)
						}
						return nil
					},
				}
			} else {
				firstItem = &activekit.MenuItem{
					Label: "Create deployment on server",
					Action: func() error {
						if activekit.YesNo("Are you sure?") {
							err := ctx.Client.CreateDeployment(ctx.Namespace, depl)
							if err != nil {
								logrus.WithError(err).Errorf("unable to update deployment %q", depl.Name)
								fmt.Println(err)
								return nil
							}
							fmt.Printf("Congratulations! Deployment %q created!\n", depl.Name)
							created = true
						}
						return nil
					},
				}
			}
			for {
				deploymentMenu(ctx, depl, firstItem).Run()
				if created {
					firstItem = &activekit.MenuItem{
						Label: "Push changes to server",
						Action: func() error {
							if activekit.YesNo("Are you sure?") {
								err := ctx.Client.ReplaceDeployment(ctx.Namespace, depl)
								if err != nil {
									logrus.WithError(err).Errorf("unable to update deployment %q", depl.Name)
									fmt.Println(err)
									return nil
								}
								fmt.Printf("Congratulations! Deployment %q updated!\n", depl.Name)
							}
							return nil
						},
					}
				}
			}
		},
	}
	command.PersistentFlags().
		StringVar(&file, "file", "", "create deployment from file")
	command.PersistentFlags().
		BoolVarP(&force, "force", "f", false, "suppress confirmation")

	command.PersistentFlags().
		StringVar(&flagDepl.Name, "deployment-name", namegen.Color()+"-"+namegen.Aster(), "deployment name, optional")
	command.PersistentFlags().
		IntVar(&flagDepl.Replicas, "replicas", 1, "replicas, optional")
	command.PersistentFlags().
		StringVar(&flagCont.Name, "container-name", "", "container name, equal to image name by default")
	command.PersistentFlags().
		StringVar(&flagCont.Image, "image", "", "container image, required")
	command.PersistentFlags().
		UintVar(&flagCont.Limits.Memory, "memory", 256, "container memory limit im Mb, optional")
	command.PersistentFlags().
		UintVar(&flagCont.Limits.CPU, "cpu", 200, "container CPU limit in mCPU, optional")
	command.PersistentFlags().
		StringSliceVar(&flagCont.Commands, "commands", nil, "container commands")
	command.PersistentFlags().
		StringVar(&envs, "env", "", "container env variable in KEY0:VALUE0 KEY1:VALUE1 format")
	return command
}

func deploymentMenu(ctx *context.Context, depl deployment.Deployment, firstItem *activekit.MenuItem) *activekit.Menu {
	return &activekit.Menu{
		Items: []*activekit.MenuItem{
			firstItem,
			{
				Label: "Edit deployment",
				Action: func() error {
					var err error
					depl, err = deplactive.ReplaceWizard(deplactive.Config{
						Deployment: &depl,
					})
					if err != nil {
						logrus.WithError(err).Errorf("unable to create deployment")
						fmt.Println(err)
						os.Exit(1)
					}
					return nil
				},
			},
			{
				Label: "Print to terminal",
				Action: activekit.ActionWithErr(func() error {
					if data, err := depl.RenderYAML(); err != nil {
						return err
					} else {
						upBorders := strings.Repeat("_", text.Width(data))
						downBorders := strings.Repeat("_", text.Width(data))
						fmt.Printf("%s\n\n%s\n%s\n", upBorders, data, downBorders)
					}
					return nil
				}),
			},
			{
				Label: "Save to file",
				Action: func() error {
					filename, _ := activekit.AskLine("Print filename: ")
					data, err := depl.RenderJSON()
					if err != nil {
						return err
					}
					if err := ioutil.WriteFile(filename, []byte(data), os.ModePerm); err != nil {
						logrus.WithError(err).Errorf("unable to save deployment %q to file", depl.Name)
						fmt.Printf("Unable to save deployment to file :(\n%v", err)
						return nil
					}
					fmt.Printf("OK\n")
					return nil
				},
			},
			{
				Label: "Exit",
				Action: func() error {
					if yes, _ := activekit.Yes("Are you sure you want to exit?"); yes {
						os.Exit(0)
					}
					return nil
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>package mlxscraper

import (
	"fmt"
	"log"
	"time"

	"github.com/gbl08ma/disturbancesmlx/dataobjects"
	"github.com/mmcdole/gofeed"
)

// RSSScraper is an announcement scraper for the Metro de Lisboa website
// It reads the RSS feed from the official website
type RSSScraper struct {
	ticker              *time.Ticker
	stopChan            chan struct{}
	log                 *log.Logger
	newAnnCallback      func(announcement *dataobjects.Announcement)
	initialDataCallback func(anns []*dataobjects.Announcement)
	firstUpdate         bool
	fp                  *gofeed.Parser
	announcements       []*dataobjects.Announcement

	URL     string
	Network *dataobjects.Network
	Period  time.Duration
}

// Begin starts the scraper
func (sc *RSSScraper) Begin(log *log.Logger,
	newAnnCallback func(announcement *dataobjects.Announcement),
	initialDataCallback func(anns []*dataobjects.Announcement)) {
	sc.stopChan = make(chan struct{})
	sc.ticker = time.NewTicker(sc.Period)
	sc.log = log
	sc.newAnnCallback = newAnnCallback
	sc.initialDataCallback = initialDataCallback
	sc.firstUpdate = true
	sc.fp = gofeed.NewParser()

	sc.log.Println("RSSScraper starting")
	sc.update()
	sc.firstUpdate = false
	sc.log.Println("RSSScraper completed first fetch")
	initialDataCallback(sc.copyAnnouncements())
	go sc.scrape()
}

// End stops the scraper
func (sc *RSSScraper) End() {
	sc.ticker.Stop()
	close(sc.stopChan)
}

func (sc *RSSScraper) copyAnnouncements() []*dataobjects.Announcement {
	c := make([]*dataobjects.Announcement, len(sc.announcements))
	for i, annPointer := range sc.announcements {
		ann := *annPointer
		c[i] = &ann
	}
	return c
}

func (sc *RSSScraper) scrape() {
	sc.update()
	sc.log.Println("RSSScraper completed second fetch")
	for {
		select {
		case <-sc.ticker.C:
			sc.update()
			sc.log.Println("RSSScraper fetch complete")
		case <-sc.stopChan:
			return
		}
	}
}

func (sc *RSSScraper) update() {
	feed, _ := sc.fp.ParseURL(sc.URL)

	if sc.announcements == nil {
		sc.announcements = []*dataobjects.Announcement{}
		for _, item := range feed.Items {
			fmt.Println(item.Title)
			fmt.Println(item.Description)
			fmt.Println(item.PublishedParsed)
			ann := dataobjects.Announcement{
				Time:    *item.PublishedParsed,
				Network: sc.Network,
				Title:   item.Title,
				Body:    item.Description,
				URL:     item.Link,
				Source:  "pt-ml-rss",
			}
			sc.announcements = append(sc.announcements, &ann)
		}
	}
}

// Networks returns the networks monitored by this scraper
func (sc *RSSScraper) Networks() []*dataobjects.Network {
	return []*dataobjects.Network{sc.Network}
}
<commit_msg>More work on the RSS feed scraper<commit_after>package mlxscraper

import (
	"fmt"
	"log"
	"sort"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"

	"github.com/gbl08ma/disturbancesmlx/dataobjects"
	"github.com/mmcdole/gofeed"
)

// RSSScraper is an announcement scraper for the Metro de Lisboa website
// It reads the RSS feed from the official website
type RSSScraper struct {
	ticker              *time.Ticker
	stopChan            chan struct{}
	log                 *log.Logger
	newAnnCallback      func(announcement *dataobjects.Announcement)
	initialDataCallback func(anns []*dataobjects.Announcement)
	firstUpdate         bool
	fp                  *gofeed.Parser
	announcements       []*dataobjects.Announcement

	URL     string
	Network *dataobjects.Network
	Period  time.Duration
}

// Begin starts the scraper
func (sc *RSSScraper) Begin(log *log.Logger,
	newAnnCallback func(announcement *dataobjects.Announcement),
	initialDataCallback func(anns []*dataobjects.Announcement)) {
	sc.stopChan = make(chan struct{})
	sc.ticker = time.NewTicker(sc.Period)
	sc.log = log
	sc.newAnnCallback = newAnnCallback
	sc.initialDataCallback = initialDataCallback
	sc.firstUpdate = true
	sc.fp = gofeed.NewParser()

	sc.log.Println("RSSScraper starting")
	sc.update()
	sc.firstUpdate = false
	sc.log.Println("RSSScraper completed first fetch")
	initialDataCallback(sc.copyAnnouncements())
	go sc.scrape()
}

// End stops the scraper
func (sc *RSSScraper) End() {
	sc.ticker.Stop()
	close(sc.stopChan)
}

func (sc *RSSScraper) copyAnnouncements() []*dataobjects.Announcement {
	c := make([]*dataobjects.Announcement, len(sc.announcements))
	for i, annPointer := range sc.announcements {
		ann := *annPointer
		c[i] = &ann
	}
	return c
}

func (sc *RSSScraper) scrape() {
	sc.update()
	sc.log.Println("RSSScraper completed second fetch")
	for {
		select {
		case <-sc.ticker.C:
			sc.update()
			sc.log.Println("RSSScraper fetch complete")
		case <-sc.stopChan:
			return
		}
	}
}

func (sc *RSSScraper) update() {
	feed, _ := sc.fp.ParseURL(sc.URL)

	announcements := []*dataobjects.Announcement{}
	for _, item := range feed.Items {
		ann := dataobjects.Announcement{
			Time:    *item.PublishedParsed,
			Network: sc.Network,
			Title:   item.Title,
			Body:    sc.adaptPostBody(item.Description),
			URL:     item.Link,
			Source:  "pt-ml-rss",
		}
		announcements = append(announcements, &ann)
	}

	sort.SliceStable(announcements, func(i, j int) bool {
		return announcements[i].Time.Before(announcements[j].Time)
	})

	for _, ann := range announcements {
		fmt.Println(ann.Title)
		fmt.Println(ann.Body)
		fmt.Println(ann.URL)
		fmt.Println(ann.Time)
	}

	if !sc.firstUpdate && len(announcements) > 0 {
		isNew := false
		curLast := announcements[len(announcements)-1]
		if len(sc.announcements) == 0 {
			isNew = true
		} else {
			// decide if an announcement is new by looking only into the published date
			prevLast := sc.announcements[len(sc.announcements)-1]
			if curLast.Time.After(prevLast.Time) {
				isNew = true
			}
		}
		if isNew {
			sc.newAnnCallback(curLast)
		}
	}
	sc.announcements = announcements
}

func (sc *RSSScraper) adaptPostBody(original string) string {
	doc, err := goquery.NewDocumentFromReader(strings.NewReader(original))
	if err != nil {
		sc.log.Println(err)
		return ""
	}
	return doc.Find("p").First().Text()
}

// Networks returns the networks monitored by this scraper
func (sc *RSSScraper) Networks() []*dataobjects.Network {
	return []*dataobjects.Network{sc.Network}
}
<|endoftext|>


<commit_before>package datasource

import (
	"context"
	"errors"
	"github.com/alexanderzobnin/grafana-zabbix/pkg/httpclient"
	"github.com/alexanderzobnin/grafana-zabbix/pkg/settings"
	"github.com/alexanderzobnin/grafana-zabbix/pkg/zabbix"
	"github.com/alexanderzobnin/grafana-zabbix/pkg/zabbixapi"
	"github.com/grafana/grafana-plugin-sdk-go/backend"
	"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
	"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
	"github.com/grafana/grafana-plugin-sdk-go/backend/log"
)

var (
	ErrFunctionsNotSupported      = errors.New("zabbix queries with functions are not supported")
	ErrNonMetricQueryNotSupported = errors.New("non-metrics queries are not supported")
)

type ZabbixDatasource struct {
	im     instancemgmt.InstanceManager
	logger log.Logger
}

// ZabbixDatasourceInstance stores state about a specific datasource
// and provides methods to make requests to the Zabbix API
type ZabbixDatasourceInstance struct {
	zabbix   *zabbix.Zabbix
	dsInfo   *backend.DataSourceInstanceSettings
	Settings *settings.ZabbixDatasourceSettings
	logger   log.Logger
}

func NewZabbixDatasource() *ZabbixDatasource {
	im := datasource.NewInstanceManager(newZabbixDatasourceInstance)
	return &ZabbixDatasource{
		im:     im,
		logger: log.New(),
	}
}

// newZabbixDatasourceInstance returns an initialized zabbix datasource instance
func newZabbixDatasourceInstance(dsSettings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
	logger := log.New()
	logger.Debug("Initializing new data source instance")

	zabbixSettings, err := settings.ReadZabbixSettings(&dsSettings)
	if err != nil {
		logger.Error("Error parsing Zabbix settings", "error", err)
		return nil, err
	}

	client, err := httpclient.New(&dsSettings, zabbixSettings.Timeout)
	if err != nil {
		logger.Error("Error initializing HTTP client", "error", err)
		return nil, err
	}

	zabbixAPI, err := zabbixapi.New(dsSettings.URL, client)
	if err != nil {
		logger.Error("Error initializing Zabbix API", "error", err)
		return nil, err
	}

	zabbixClient, err := zabbix.New(&dsSettings, zabbixSettings, zabbixAPI)
	if err != nil {
		logger.Error("Error initializing Zabbix client", "error", err)
		return nil, err
	}

	return &ZabbixDatasourceInstance{
		dsInfo:   &dsSettings,
		zabbix:   zabbixClient,
		Settings: zabbixSettings,
		logger:   logger,
	}, nil
}

// CheckHealth checks if the plugin is running properly
func (ds *ZabbixDatasource) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
	res := &backend.CheckHealthResult{}

	dsInstance, err := ds.getDSInstance(req.PluginContext)
	if err != nil {
		res.Status = backend.HealthStatusError
		res.Message = "Error getting datasource instance"
		ds.logger.Error("Error getting datasource instance", "err", err)
		return res, nil
	}

	message, err := dsInstance.TestConnection(ctx)
	if err != nil {
		res.Status = backend.HealthStatusError
		res.Message = err.Error()
		ds.logger.Error("Error connecting Zabbix server", "err", err)
		return res, nil
	}

	res.Status = backend.HealthStatusOk
	res.Message = message
	return res, nil
}

func (ds *ZabbixDatasource) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
	ds.logger.Debug("QueryData()")
	qdr := backend.NewQueryDataResponse()

	zabbixDS, err := ds.getDSInstance(req.PluginContext)
	if err != nil {
		return nil, err
	}

	for _, q := range req.Queries {
		res := backend.DataResponse{}
		query, err := ReadQuery(q)
		ds.logger.Debug("DS query", "query", q)
		if err != nil {
			res.Error = err
		} else if query.QueryType == MODE_METRICS {
			frames, err := zabbixDS.queryNumericItems(ctx, &query)
			if err != nil {
				res.Error = err
			} else {
				res.Frames = append(res.Frames, frames...)
			}
		} else if query.QueryType == MODE_ITEMID {
			frames, err := zabbixDS.queryItemIdData(ctx, &query)
			if err != nil {
				res.Error = err
			} else {
				res.Frames = append(res.Frames, frames...)
			}
		} else {
			res.Error = ErrNonMetricQueryNotSupported
		}
		qdr.Responses[q.RefID] = res
	}

	return qdr, nil
}

// getDSInstance Returns cached datasource or creates new one
func (ds *ZabbixDatasource) getDSInstance(pluginContext backend.PluginContext) (*ZabbixDatasourceInstance, error) {
	instance, err := ds.im.Get(pluginContext)
	if err != nil {
		return nil, err
	}
	return instance.(*ZabbixDatasourceInstance), nil
}
<commit_msg>Remove unused debug logging<commit_after>package datasource

import (
	"context"
	"errors"

	"github.com/alexanderzobnin/grafana-zabbix/pkg/httpclient"
	"github.com/alexanderzobnin/grafana-zabbix/pkg/settings"
	"github.com/alexanderzobnin/grafana-zabbix/pkg/zabbix"
	"github.com/alexanderzobnin/grafana-zabbix/pkg/zabbixapi"
	"github.com/grafana/grafana-plugin-sdk-go/backend"
	"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
	"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
	"github.com/grafana/grafana-plugin-sdk-go/backend/log"
)

var (
	ErrFunctionsNotSupported      = errors.New("zabbix queries with functions are not supported")
	ErrNonMetricQueryNotSupported = errors.New("non-metrics queries are not supported")
)

type ZabbixDatasource struct {
	im     instancemgmt.InstanceManager
	logger log.Logger
}

// ZabbixDatasourceInstance stores state about a specific datasource
// and provides methods to make requests to the Zabbix API
type ZabbixDatasourceInstance struct {
	zabbix   *zabbix.Zabbix
	dsInfo   *backend.DataSourceInstanceSettings
	Settings *settings.ZabbixDatasourceSettings
	logger   log.Logger
}

func NewZabbixDatasource() *ZabbixDatasource {
	im := datasource.NewInstanceManager(newZabbixDatasourceInstance)
	return &ZabbixDatasource{
		im:     im,
		logger: log.New(),
	}
}

// newZabbixDatasourceInstance returns an initialized zabbix datasource instance
func newZabbixDatasourceInstance(dsSettings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
	logger := log.New()
	logger.Debug("Initializing new data source instance")

	zabbixSettings, err := settings.ReadZabbixSettings(&dsSettings)
	if err != nil {
		logger.Error("Error parsing Zabbix settings", "error", err)
		return nil, err
	}

	client, err := httpclient.New(&dsSettings, zabbixSettings.Timeout)
	if err != nil {
		logger.Error("Error initializing HTTP client", "error", err)
		return nil, err
	}

	zabbixAPI, err := zabbixapi.New(dsSettings.URL, client)
	if err != nil {
		logger.Error("Error initializing Zabbix API", "error", err)
		return nil, err
	}

	zabbixClient, err := zabbix.New(&dsSettings, zabbixSettings, zabbixAPI)
	if err != nil {
		logger.Error("Error initializing Zabbix client", "error", err)
		return nil, err
	}

	return &ZabbixDatasourceInstance{
		dsInfo:   &dsSettings,
		zabbix:   zabbixClient,
		Settings: zabbixSettings,
		logger:   logger,
	}, nil
}

// CheckHealth checks if the plugin is running properly
func (ds *ZabbixDatasource) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
	res := &backend.CheckHealthResult{}

	dsInstance, err := ds.getDSInstance(req.PluginContext)
	if err != nil {
		res.Status = backend.HealthStatusError
		res.Message = "Error getting datasource instance"
		ds.logger.Error("Error getting datasource instance", "err", err)
		return res, nil
	}

	message, err := dsInstance.TestConnection(ctx)
	if err != nil {
		res.Status = backend.HealthStatusError
		res.Message = err.Error()
		ds.logger.Error("Error connecting Zabbix server", "err", err)
		return res, nil
	}

	res.Status = backend.HealthStatusOk
	res.Message = message
	return res, nil
}

func (ds *ZabbixDatasource) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
	qdr := backend.NewQueryDataResponse()

	zabbixDS, err := ds.getDSInstance(req.PluginContext)
	if err != nil {
		return nil, err
	}

	for _, q := range req.Queries {
		res := backend.DataResponse{}
		query, err := ReadQuery(q)
		ds.logger.Debug("DS query", "query", q)
		if err != nil {
			res.Error = err
		} else if query.QueryType == MODE_METRICS {
			frames, err := zabbixDS.queryNumericItems(ctx, &query)
			if err != nil {
				res.Error = err
			} else {
				res.Frames = append(res.Frames, frames...)
			}
		} else if query.QueryType == MODE_ITEMID {
			frames, err := zabbixDS.queryItemIdData(ctx, &query)
			if err != nil {
				res.Error = err
			} else {
				res.Frames = append(res.Frames, frames...)
			}
		} else {
			res.Error = ErrNonMetricQueryNotSupported
		}
		qdr.Responses[q.RefID] = res
	}

	return qdr, nil
}

// getDSInstance Returns cached datasource or creates new one
func (ds *ZabbixDatasource) getDSInstance(pluginContext backend.PluginContext) (*ZabbixDatasourceInstance, error) {
	instance, err := ds.im.Get(pluginContext)
	if err != nil {
		return nil, err
	}
	return instance.(*ZabbixDatasourceInstance), nil
}
<|endoftext|>