<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package bootstrap

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/utils"
	"github.com/juju/utils/series"
	"github.com/juju/utils/ssh"

	"github.com/juju/juju/cloudconfig/instancecfg"
	"github.com/juju/juju/constraints"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/imagemetadata"
	"github.com/juju/juju/environs/simplestreams"
	"github.com/juju/juju/environs/storage"
	"github.com/juju/juju/environs/sync"
	"github.com/juju/juju/environs/tools"
	"github.com/juju/juju/network"
	coretools "github.com/juju/juju/tools"
	"github.com/juju/juju/version"
)

const noToolsMessage = `Juju cannot bootstrap because no tools are available for your environment.
You may want to use the 'agent-metadata-url' configuration setting to specify the tools location.
`

var (
	logger = loggo.GetLogger("juju.environs.bootstrap")
)

// BootstrapParams holds the parameters for bootstrapping an environment.
type BootstrapParams struct {
	// Constraints are used to choose the initial instance specification,
	// and will be stored in the new environment's state.
	Constraints constraints.Value

	// Placement, if non-empty, holds an environment-specific placement
	// directive used to choose the initial instance.
	Placement string

	// UploadTools reports whether we should upload the local tools and
	// override the environment's specified agent-version.
	UploadTools bool

	// MetadataDir is an optional path to a local directory containing
	// tools and/or image metadata.
	MetadataDir string

	// AgentVersion, if set, determines the exact tools version that
	// will be used to start the Juju agents.
	AgentVersion *version.Number
}

// Bootstrap bootstraps the given environment. The supplied constraints are
// used to provision the instance, and are also set within the bootstrapped
// environment.
func Bootstrap(ctx environs.BootstrapContext, environ environs.Environ, args BootstrapParams) error {
	cfg := environ.Config()
	network.SetPreferIPv6(cfg.PreferIPv6())
	if secret := cfg.AdminSecret(); secret == "" {
		return errors.Errorf("environment configuration has no admin-secret")
	}
	if authKeys := ssh.SplitAuthorisedKeys(cfg.AuthorizedKeys()); len(authKeys) == 0 {
		// Apparently this can never happen, so it's not tested. But, one day,
		// Config will act differently (it's pretty crazy that, AFAICT, the
		// authorized-keys are optional config settings... but it's impossible
		// to actually *create* a config without them)... and when it does,
		// we'll be here to catch this problem early.
		return errors.Errorf("environment configuration has no authorized-keys")
	}
	if _, hasCACert := cfg.CACert(); !hasCACert {
		return errors.Errorf("environment configuration has no ca-cert")
	}
	if _, hasCAKey := cfg.CAPrivateKey(); !hasCAKey {
		return errors.Errorf("environment configuration has no ca-private-key")
	}

	// Set default tools metadata source, add image metadata source,
	// then verify constraints. Providers may rely on image metadata
	// for constraint validation.
	var customImageMetadata []*imagemetadata.ImageMetadata
	if args.MetadataDir != "" {
		var err error
		customImageMetadata, err = setPrivateMetadataSources(environ, args.MetadataDir)
		if err != nil {
			return err
		}
	}
	if err := validateConstraints(environ, args.Constraints); err != nil {
		return err
	}

	_, supportsNetworking := environs.SupportsNetworking(environ)

	ctx.Infof("Bootstrapping environment %q", cfg.Name())
	logger.Debugf("environment %q supports service/machine networks: %v", cfg.Name(), supportsNetworking)
	disableNetworkManagement, _ := cfg.DisableNetworkManagement()
	logger.Debugf("network management by juju enabled: %v", !disableNetworkManagement)
	availableTools, err := findAvailableTools(environ, args.AgentVersion, args.Constraints.Arch, args.UploadTools)
	if errors.IsNotFound(err) {
		return errors.New(noToolsMessage)
	} else if err != nil {
		return err
	}
	if lxcMTU, ok := cfg.LXCDefaultMTU(); ok {
		logger.Debugf("using MTU %v for all created LXC containers' network interfaces", lxcMTU)
	}

	// For providers that support make use of simplestreams image metadata,
	// search public image metadata. We need to pass this onto Bootstrap
	// for selecting images.
	var imageMetadata []*imagemetadata.ImageMetadata
	var publicImageMetadata []*imagemetadata.ImageMetadata
	if hasRegion, ok := environ.(simplestreams.HasRegion); ok {
		sources, err := environs.ImageMetadataSources(environ)
		if err != nil {
			return errors.Trace(err)
		}
		region, err := hasRegion.Region()
		if err != nil {
			return errors.Trace(err)
		}
		imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
			CloudSpec: region,
			Series:    availableTools.AllSeries(),
			Arches:    availableTools.Arches(),
			Stream:    environ.Config().ImageStream(),
		})
		publicImageMetadata, _, err = imagemetadata.Fetch(sources, imageConstraint)
		if err != nil {
			return errors.Annotate(err, "searching image metadata")
		}

		// Filter custom image metadata to the ones we're interested in for
		// bootstrapping the initial instance.
		seriesVersions := make([]string, len(imageConstraint.Series))
		for i, constraintSeries := range imageConstraint.Series {
			seriesVersion, err := series.SeriesVersion(constraintSeries)
			if err != nil {
				return errors.Trace(err)
			}
			seriesVersions[i] = seriesVersion
		}
		for _, image := range customImageMetadata {
			if matchImageMetadata(
				image,
				imageConstraint.CloudSpec,
				seriesVersions,
				imageConstraint.Arches,
				imageConstraint.Stream,
			) {
				imageMetadata = append(imageMetadata, image)
			}
		}
		imageMetadata = append(imageMetadata, publicImageMetadata...)
	}

	// If we're uploading, we must override agent-version;
	// if we're not uploading, we want to ensure we have an
	// agent-version set anyway, to appease FinishInstanceConfig.
	// In the latter case, setBootstrapTools will later set
	// agent-version to the correct thing.
	agentVersion := version.Current
	if args.AgentVersion != nil {
		agentVersion = *args.AgentVersion
	}
	if cfg, err = cfg.Apply(map[string]interface{}{
		"agent-version": agentVersion.String(),
	}); err != nil {
		return err
	}
	if err = environ.SetConfig(cfg); err != nil {
		return err
	}

	ctx.Infof("Starting new instance for initial state server")
	result, err := environ.Bootstrap(ctx, environs.BootstrapParams{
		Constraints:    args.Constraints,
		Placement:      args.Placement,
		AvailableTools: availableTools,
		ImageMetadata:  imageMetadata,
	})
	if err != nil {
		return err
	}

	matchingTools, err := availableTools.Match(coretools.Filter{
		Arch:   result.Arch,
		Series: result.Series,
	})
	if err != nil {
		return err
	}
	selectedTools, err := setBootstrapTools(environ, matchingTools)
	if err != nil {
		return err
	}
	if selectedTools.URL == "" {
		if !args.UploadTools {
			logger.Warningf("no prepackaged tools available")
		}
		ctx.Infof("Building tools to upload (%s)", selectedTools.Version)
		builtTools, err := sync.BuildToolsTarball(&selectedTools.Version.Number, cfg.AgentStream())
		if err != nil {
			return errors.Annotate(err, "cannot upload bootstrap tools")
		}
		defer os.RemoveAll(builtTools.Dir)
		filename := filepath.Join(builtTools.Dir, builtTools.StorageName)
		selectedTools.URL = fmt.Sprintf("file://%s", filename)
		selectedTools.Size = builtTools.Size
		selectedTools.SHA256 = builtTools.Sha256Hash
	}

	ctx.Infof("Installing Juju agent on bootstrap instance")
	instanceConfig, err := instancecfg.NewBootstrapInstanceConfig(args.Constraints, result.Series)
	if err != nil {
		return err
	}
	instanceConfig.Tools = selectedTools
	instanceConfig.CustomImageMetadata = customImageMetadata
	if err := result.Finalize(ctx, instanceConfig); err != nil {
		return err
	}
	ctx.Infof("Bootstrap agent installed")
	return nil
}

func matchImageMetadata(
	image *imagemetadata.ImageMetadata,
	cloudSpec simplestreams.CloudSpec,
	seriesVersions []string, arches []string,
	stream string,
) bool {
	return image.Stream == stream &&
		image.RegionName == cloudSpec.Region &&
		image.Endpoint == cloudSpec.Endpoint &&
		sliceContainsString(seriesVersions, image.Version) &&
		sliceContainsString(arches, image.Arch)
}

func sliceContainsString(values []string, search string) bool {
	for _, value := range values {
		if value == search {
			return true
		}
	}
	return false
}

// setBootstrapTools returns the newest tools from the given tools list,
// and updates the agent-version configuration attribute.
func setBootstrapTools(environ environs.Environ, possibleTools coretools.List) (*coretools.Tools, error) {
	if len(possibleTools) == 0 {
		return nil, fmt.Errorf("no bootstrap tools available")
	}
	var newVersion version.Number
	newVersion, toolsList := possibleTools.Newest()
	logger.Infof("newest version: %s", newVersion)
	cfg := environ.Config()
	if agentVersion, _ := cfg.AgentVersion(); agentVersion != newVersion {
		cfg, err := cfg.Apply(map[string]interface{}{
			"agent-version": newVersion.String(),
		})
		if err == nil {
			err = environ.SetConfig(cfg)
		}
		if err != nil {
			return nil, fmt.Errorf("failed to update environment configuration: %v", err)
		}
	}
	bootstrapVersion := newVersion
	// We should only ever bootstrap the exact same version as the client,
	// or we risk bootstrap incompatibility. We still set agent-version to
	// the newest version, so the agent will immediately upgrade itself.
	if !isCompatibleVersion(newVersion, version.Current) {
		compatibleVersion, compatibleTools := findCompatibleTools(possibleTools, version.Current)
		if len(compatibleTools) == 0 {
			logger.Warningf(
				"failed to find %s tools, will attempt to use %s",
				version.Current, newVersion,
			)
		} else {
			bootstrapVersion, toolsList = compatibleVersion, compatibleTools
		}
	}
	logger.Infof("picked bootstrap tools version: %s", bootstrapVersion)
	return toolsList[0], nil
}

// findCompatibleTools finds tools in the list that have the same major, minor
// and patch level as version.Current.
//
// Build number is not important to match; uploaded tools will have
// incremented build number, and we want to match them.
func findCompatibleTools(possibleTools coretools.List, version version.Number) (version.Number, coretools.List) {
	var compatibleTools coretools.List
	for _, tools := range possibleTools {
		if isCompatibleVersion(tools.Version.Number, version) {
			compatibleTools = append(compatibleTools, tools)
		}
	}
	return compatibleTools.Newest()
}

func isCompatibleVersion(v1, v2 version.Number) bool {
	v1.Build = 0
	v2.Build = 0
	return v1.Compare(v2) == 0
}

// setPrivateMetadataSources sets the default tools metadata source
// for tools syncing, and adds an image metadata source after verifying
// the contents.
func setPrivateMetadataSources(env environs.Environ, metadataDir string) ([]*imagemetadata.ImageMetadata, error) {
	logger.Infof("Setting default tools and image metadata sources: %s", metadataDir)
	tools.DefaultBaseURL = metadataDir

	imageMetadataDir := filepath.Join(metadataDir, storage.BaseImagesPath)
	if _, err := os.Stat(imageMetadataDir); err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Annotate(err, "cannot access image metadata")
		}
		return nil, nil
	}

	baseURL := fmt.Sprintf("file://%s", filepath.ToSlash(imageMetadataDir))
	datasource := simplestreams.NewURLDataSource("bootstrap metadata", baseURL, utils.NoVerifySSLHostnames, simplestreams.CUSTOM_CLOUD_DATA, false)

	// Read the image metadata, as we'll want to upload it to the environment.
	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{})
	existingMetadata, _, err := imagemetadata.Fetch([]simplestreams.DataSource{datasource}, imageConstraint)
	if err != nil && !errors.IsNotFound(err) {
		return nil, errors.Annotate(err, "cannot read image metadata")
	}

	// Add an image metadata datasource for constraint validation, etc.
	environs.RegisterUserImageDataSourceFunc("bootstrap metadata", func(environs.Environ) (simplestreams.DataSource, error) {
		return datasource, nil
	})
	logger.Infof("custom image metadata added to search path")
	return existingMetadata, nil
}

func validateConstraints(env environs.Environ, cons constraints.Value) error {
	validator, err := env.ConstraintsValidator()
	if err != nil {
		return err
	}
	unsupported, err := validator.Validate(cons)
	if len(unsupported) > 0 {
		logger.Warningf("unsupported constraints: %v", unsupported)
	}
	return err
}

// EnsureNotBootstrapped returns nil if the environment is not
// bootstrapped, and an error if it is or if the function was not able
// to tell.
func EnsureNotBootstrapped(env environs.Environ) error {
	_, err := env.StateServerInstances()
	// If there is no error determining state server instaces,
	// then we are bootstrapped.
	switch errors.Cause(err) {
	case nil:
		return environs.ErrAlreadyBootstrapped
	case environs.ErrNoInstances:
		// TODO(axw) 2015-02-03 #1417526
		// We should not be relying on this result,
		// as it is possible for there to be no
		// state servers despite the environment
		// being bootstrapped.
		fallthrough
	case environs.ErrNotBootstrapped:
		return nil
	}
	return err
}
<commit_msg>environs/bootstrap: use set.Strings<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package bootstrap

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/utils"
	"github.com/juju/utils/series"
	"github.com/juju/utils/set"
	"github.com/juju/utils/ssh"

	"github.com/juju/juju/cloudconfig/instancecfg"
	"github.com/juju/juju/constraints"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/imagemetadata"
	"github.com/juju/juju/environs/simplestreams"
	"github.com/juju/juju/environs/storage"
	"github.com/juju/juju/environs/sync"
	"github.com/juju/juju/environs/tools"
	"github.com/juju/juju/network"
	coretools "github.com/juju/juju/tools"
	"github.com/juju/juju/version"
)

const noToolsMessage = `Juju cannot bootstrap because no tools are available for your environment.
You may want to use the 'agent-metadata-url' configuration setting to specify the tools location.
`

var (
	logger = loggo.GetLogger("juju.environs.bootstrap")
)

// BootstrapParams holds the parameters for bootstrapping an environment.
type BootstrapParams struct {
	// Constraints are used to choose the initial instance specification,
	// and will be stored in the new environment's state.
	Constraints constraints.Value

	// Placement, if non-empty, holds an environment-specific placement
	// directive used to choose the initial instance.
	Placement string

	// UploadTools reports whether we should upload the local tools and
	// override the environment's specified agent-version.
	UploadTools bool

	// MetadataDir is an optional path to a local directory containing
	// tools and/or image metadata.
	MetadataDir string

	// AgentVersion, if set, determines the exact tools version that
	// will be used to start the Juju agents.
	AgentVersion *version.Number
}

// Bootstrap bootstraps the given environment. The supplied constraints are
// used to provision the instance, and are also set within the bootstrapped
// environment.
func Bootstrap(ctx environs.BootstrapContext, environ environs.Environ, args BootstrapParams) error {
	cfg := environ.Config()
	network.SetPreferIPv6(cfg.PreferIPv6())
	if secret := cfg.AdminSecret(); secret == "" {
		return errors.Errorf("environment configuration has no admin-secret")
	}
	if authKeys := ssh.SplitAuthorisedKeys(cfg.AuthorizedKeys()); len(authKeys) == 0 {
		// Apparently this can never happen, so it's not tested. But, one day,
		// Config will act differently (it's pretty crazy that, AFAICT, the
		// authorized-keys are optional config settings... but it's impossible
		// to actually *create* a config without them)... and when it does,
		// we'll be here to catch this problem early.
		return errors.Errorf("environment configuration has no authorized-keys")
	}
	if _, hasCACert := cfg.CACert(); !hasCACert {
		return errors.Errorf("environment configuration has no ca-cert")
	}
	if _, hasCAKey := cfg.CAPrivateKey(); !hasCAKey {
		return errors.Errorf("environment configuration has no ca-private-key")
	}

	// Set default tools metadata source, add image metadata source,
	// then verify constraints. Providers may rely on image metadata
	// for constraint validation.
	var customImageMetadata []*imagemetadata.ImageMetadata
	if args.MetadataDir != "" {
		var err error
		customImageMetadata, err = setPrivateMetadataSources(environ, args.MetadataDir)
		if err != nil {
			return err
		}
	}
	if err := validateConstraints(environ, args.Constraints); err != nil {
		return err
	}

	_, supportsNetworking := environs.SupportsNetworking(environ)

	ctx.Infof("Bootstrapping environment %q", cfg.Name())
	logger.Debugf("environment %q supports service/machine networks: %v", cfg.Name(), supportsNetworking)
	disableNetworkManagement, _ := cfg.DisableNetworkManagement()
	logger.Debugf("network management by juju enabled: %v", !disableNetworkManagement)
	availableTools, err := findAvailableTools(environ, args.AgentVersion, args.Constraints.Arch, args.UploadTools)
	if errors.IsNotFound(err) {
		return errors.New(noToolsMessage)
	} else if err != nil {
		return err
	}
	if lxcMTU, ok := cfg.LXCDefaultMTU(); ok {
		logger.Debugf("using MTU %v for all created LXC containers' network interfaces", lxcMTU)
	}

	// For providers that support make use of simplestreams image metadata,
	// search public image metadata. We need to pass this onto Bootstrap
	// for selecting images.
	var imageMetadata []*imagemetadata.ImageMetadata
	var publicImageMetadata []*imagemetadata.ImageMetadata
	if hasRegion, ok := environ.(simplestreams.HasRegion); ok {
		sources, err := environs.ImageMetadataSources(environ)
		if err != nil {
			return errors.Trace(err)
		}
		region, err := hasRegion.Region()
		if err != nil {
			return errors.Trace(err)
		}
		imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
			CloudSpec: region,
			Series:    availableTools.AllSeries(),
			Arches:    availableTools.Arches(),
			Stream:    environ.Config().ImageStream(),
		})
		publicImageMetadata, _, err = imagemetadata.Fetch(sources, imageConstraint)
		if err != nil {
			return errors.Annotate(err, "searching image metadata")
		}

		// Filter custom image metadata to the ones we're interested in for
		// bootstrapping the initial instance.
		seriesVersions := set.NewStrings()
		for _, constraintSeries := range imageConstraint.Series {
			seriesVersion, err := series.SeriesVersion(constraintSeries)
			if err != nil {
				return errors.Trace(err)
			}
			seriesVersions.Add(seriesVersion)
		}
		arches := set.NewStrings(imageConstraint.Arches...)
		for _, image := range customImageMetadata {
			if matchImageMetadata(
				image,
				imageConstraint.CloudSpec,
				seriesVersions,
				arches,
				imageConstraint.Stream,
			) {
				imageMetadata = append(imageMetadata, image)
			}
		}
		imageMetadata = append(imageMetadata, publicImageMetadata...)
	}

	// If we're uploading, we must override agent-version;
	// if we're not uploading, we want to ensure we have an
	// agent-version set anyway, to appease FinishInstanceConfig.
	// In the latter case, setBootstrapTools will later set
	// agent-version to the correct thing.
	agentVersion := version.Current
	if args.AgentVersion != nil {
		agentVersion = *args.AgentVersion
	}
	if cfg, err = cfg.Apply(map[string]interface{}{
		"agent-version": agentVersion.String(),
	}); err != nil {
		return err
	}
	if err = environ.SetConfig(cfg); err != nil {
		return err
	}

	ctx.Infof("Starting new instance for initial state server")
	result, err := environ.Bootstrap(ctx, environs.BootstrapParams{
		Constraints:    args.Constraints,
		Placement:      args.Placement,
		AvailableTools: availableTools,
		ImageMetadata:  imageMetadata,
	})
	if err != nil {
		return err
	}

	matchingTools, err := availableTools.Match(coretools.Filter{
		Arch:   result.Arch,
		Series: result.Series,
	})
	if err != nil {
		return err
	}
	selectedTools, err := setBootstrapTools(environ, matchingTools)
	if err != nil {
		return err
	}
	if selectedTools.URL == "" {
		if !args.UploadTools {
			logger.Warningf("no prepackaged tools available")
		}
		ctx.Infof("Building tools to upload (%s)", selectedTools.Version)
		builtTools, err := sync.BuildToolsTarball(&selectedTools.Version.Number, cfg.AgentStream())
		if err != nil {
			return errors.Annotate(err, "cannot upload bootstrap tools")
		}
		defer os.RemoveAll(builtTools.Dir)
		filename := filepath.Join(builtTools.Dir, builtTools.StorageName)
		selectedTools.URL = fmt.Sprintf("file://%s", filename)
		selectedTools.Size = builtTools.Size
		selectedTools.SHA256 = builtTools.Sha256Hash
	}

	ctx.Infof("Installing Juju agent on bootstrap instance")
	instanceConfig, err := instancecfg.NewBootstrapInstanceConfig(args.Constraints, result.Series)
	if err != nil {
		return err
	}
	instanceConfig.Tools = selectedTools
	instanceConfig.CustomImageMetadata = customImageMetadata
	if err := result.Finalize(ctx, instanceConfig); err != nil {
		return err
	}
	ctx.Infof("Bootstrap agent installed")
	return nil
}

func matchImageMetadata(
	image *imagemetadata.ImageMetadata,
	cloudSpec simplestreams.CloudSpec,
	seriesVersions, arches set.Strings,
	stream string,
) bool {
	return image.Stream == stream &&
		image.RegionName == cloudSpec.Region &&
		image.Endpoint == cloudSpec.Endpoint &&
		seriesVersions.Contains(image.Version) &&
		arches.Contains(image.Arch)
}

// setBootstrapTools returns the newest tools from the given tools list,
// and updates the agent-version configuration attribute.
func setBootstrapTools(environ environs.Environ, possibleTools coretools.List) (*coretools.Tools, error) {
	if len(possibleTools) == 0 {
		return nil, fmt.Errorf("no bootstrap tools available")
	}
	var newVersion version.Number
	newVersion, toolsList := possibleTools.Newest()
	logger.Infof("newest version: %s", newVersion)
	cfg := environ.Config()
	if agentVersion, _ := cfg.AgentVersion(); agentVersion != newVersion {
		cfg, err := cfg.Apply(map[string]interface{}{
			"agent-version": newVersion.String(),
		})
		if err == nil {
			err = environ.SetConfig(cfg)
		}
		if err != nil {
			return nil, fmt.Errorf("failed to update environment configuration: %v", err)
		}
	}
	bootstrapVersion := newVersion
	// We should only ever bootstrap the exact same version as the client,
	// or we risk bootstrap incompatibility. We still set agent-version to
	// the newest version, so the agent will immediately upgrade itself.
	if !isCompatibleVersion(newVersion, version.Current) {
		compatibleVersion, compatibleTools := findCompatibleTools(possibleTools, version.Current)
		if len(compatibleTools) == 0 {
			logger.Warningf(
				"failed to find %s tools, will attempt to use %s",
				version.Current, newVersion,
			)
		} else {
			bootstrapVersion, toolsList = compatibleVersion, compatibleTools
		}
	}
	logger.Infof("picked bootstrap tools version: %s", bootstrapVersion)
	return toolsList[0], nil
}

// findCompatibleTools finds tools in the list that have the same major, minor
// and patch level as version.Current.
//
// Build number is not important to match; uploaded tools will have
// incremented build number, and we want to match them.
func findCompatibleTools(possibleTools coretools.List, version version.Number) (version.Number, coretools.List) {
	var compatibleTools coretools.List
	for _, tools := range possibleTools {
		if isCompatibleVersion(tools.Version.Number, version) {
			compatibleTools = append(compatibleTools, tools)
		}
	}
	return compatibleTools.Newest()
}

func isCompatibleVersion(v1, v2 version.Number) bool {
	v1.Build = 0
	v2.Build = 0
	return v1.Compare(v2) == 0
}

// setPrivateMetadataSources sets the default tools metadata source
// for tools syncing, and adds an image metadata source after verifying
// the contents.
func setPrivateMetadataSources(env environs.Environ, metadataDir string) ([]*imagemetadata.ImageMetadata, error) {
	logger.Infof("Setting default tools and image metadata sources: %s", metadataDir)
	tools.DefaultBaseURL = metadataDir

	imageMetadataDir := filepath.Join(metadataDir, storage.BaseImagesPath)
	if _, err := os.Stat(imageMetadataDir); err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Annotate(err, "cannot access image metadata")
		}
		return nil, nil
	}

	baseURL := fmt.Sprintf("file://%s", filepath.ToSlash(imageMetadataDir))
	datasource := simplestreams.NewURLDataSource("bootstrap metadata", baseURL, utils.NoVerifySSLHostnames, simplestreams.CUSTOM_CLOUD_DATA, false)

	// Read the image metadata, as we'll want to upload it to the environment.
	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{})
	existingMetadata, _, err := imagemetadata.Fetch([]simplestreams.DataSource{datasource}, imageConstraint)
	if err != nil && !errors.IsNotFound(err) {
		return nil, errors.Annotate(err, "cannot read image metadata")
	}

	// Add an image metadata datasource for constraint validation, etc.
	environs.RegisterUserImageDataSourceFunc("bootstrap metadata", func(environs.Environ) (simplestreams.DataSource, error) {
		return datasource, nil
	})
	logger.Infof("custom image metadata added to search path")
	return existingMetadata, nil
}

func validateConstraints(env environs.Environ, cons constraints.Value) error {
	validator, err := env.ConstraintsValidator()
	if err != nil {
		return err
	}
	unsupported, err := validator.Validate(cons)
	if len(unsupported) > 0 {
		logger.Warningf("unsupported constraints: %v", unsupported)
	}
	return err
}

// EnsureNotBootstrapped returns nil if the environment is not
// bootstrapped, and an error if it is or if the function was not able
// to tell.
func EnsureNotBootstrapped(env environs.Environ) error {
	_, err := env.StateServerInstances()
	// If there is no error determining state server instaces,
	// then we are bootstrapped.
	switch errors.Cause(err) {
	case nil:
		return environs.ErrAlreadyBootstrapped
	case environs.ErrNoInstances:
		// TODO(axw) 2015-02-03 #1417526
		// We should not be relying on this result,
		// as it is possible for there to be no
		// state servers despite the environment
		// being bootstrapped.
		fallthrough
	case environs.ErrNotBootstrapped:
		return nil
	}
	return err
}
<|endoftext|>


<commit_before>package testevents

// EventListeners are functions that are called when the appropriate event
// is dispatched
//
// Funcs can only be compared to nil, so a listener
// is expected to have a unique numeric ID attached to it.
type EventListener struct {
	Listener func(Event)
	ID       int
}

var listeners map[EventType][]EventListener

func init() {
	listeners = make(map[EventType][]EventListener)
}

// The EventType is exactly what occurred during testing, such as the test passing
// or failing
type EventType uint

const (
	TestStarted EventType = iota
	TestPassed
	TestFinished
	TestFailed
	TestSkipped
)

// An Event is a simple pair of the name of the testing function being executed,
// and the test status (e.g. TestFailed, TestFinished, etc)
type Event struct {
	Name string
	Typ  EventType
}

// Registers a listener for a certain type of test event
//
// If the listener is already registered, this is a no op
func Register(typ EventType, listener EventListener) {
	if IsRegistered(typ, listener) {
		return
	}

	listeners[typ] = append(listeners[typ], listener)
}

// Unregisters a listener for a certain type of event
//
// If the listener is not registered, this is a no op
func Unregister(typ EventType, listener EventListener) {
	index := -1
	for i, l := range listeners[typ] {
		if l.ID == listener.ID {
			index = i
			break
		}
	}

	listeners[typ] = append(listeners[typ][:index], listeners[typ][index+1:]...)
}

// Returns whether or not a listener is registered already
func IsRegistered(typ EventType, listener EventListener) bool {
	for _, l := range listeners[typ] {
		if l.ID == listener.ID {
			return true
		}
	}

	return false
}

// Dispatches an event to all registered listeners
func Dispatch(e Event) {
	for _, l := range listeners[e.Typ] {
		l.Listener(e)
	}
}
<commit_msg>Package documentation<commit_after>// Package testevents is an example for this question on StackOverflow:
// http://stackoverflow.com/questions/23699237/go-lang-test-events-listener/23706120#23706120
//
// It provides a simple wrapper of *testing.T and a basic event manager API to dispatch events on tests.
// An example of its use would be, at the top of your testing file:
//
//    import "github.com/Jragonmiris/testevents"
//    import "testing"
//
//    func init() {
//        testevents.Register(MyListener)
//    }
//
//    func TestXxx(tst *testing.T) {
//        t := testevents.Start(tst, "TestXxx", false)
//        defer t.Done()
//
//        err := DoSomething()
//        if err != nil {
//            t.Error(err)
//        }
//    }
//
// This package is engineered to be as close to the original testing framework as possible.
// Note that the Parallel function is intentionally left unimplemented, as the package is not
// concurrency safe.
package testevents

// EventListeners are functions that are called when the appropriate event
// is dispatched
//
// Funcs can only be compared to nil, so a listener
// is expected to have a unique numeric ID attached to it.
type EventListener struct {
	Listener func(Event)
	ID       int
}

var listeners map[EventType][]EventListener

func init() {
	listeners = make(map[EventType][]EventListener)
}

// The EventType is exactly what occurred during testing, such as the test passing
// or failing
type EventType uint

const (
	TestStarted EventType = iota
	TestPassed
	TestFinished
	TestFailed
	TestSkipped
)

// An Event is a simple pair of the name of the testing function being executed,
// and the test status (e.g. TestFailed, TestFinished, etc)
type Event struct {
	Name string
	Typ  EventType
}

// Registers a listener for a certain type of test event
//
// If the listener is already registered, this is a no op
func Register(typ EventType, listener EventListener) {
	if IsRegistered(typ, listener) {
		return
	}

	listeners[typ] = append(listeners[typ], listener)
}

// Unregisters a listener for a certain type of event
//
// If the listener is not registered, this is a no op
func Unregister(typ EventType, listener EventListener) {
	index := -1
	for i, l := range listeners[typ] {
		if l.ID == listener.ID {
			index = i
			break
		}
	}

	listeners[typ] = append(listeners[typ][:index], listeners[typ][index+1:]...)
}

// Returns whether or not a listener is registered already
func IsRegistered(typ EventType, listener EventListener) bool {
	for _, l := range listeners[typ] {
		if l.ID == listener.ID {
			return true
		}
	}

	return false
}

// Dispatches an event to all registered listeners
func Dispatch(e Event) {
	for _, l := range listeners[e.Typ] {
		l.Listener(e)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"errors"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/g-rath/osv-detector/pkg/lockfile"
	"github.com/jedib0t/go-pretty/v6/table"
	"github.com/jedib0t/go-pretty/v6/text"
	"github.com/package-url/packageurl-go"
	"github.com/urfave/cli/v2"
	"golang.org/x/crypto/ssh/terminal"

	"github.com/google/osv.dev/tools/osv-scanner/internal/grouper"
	"github.com/google/osv.dev/tools/osv-scanner/internal/osv"
	"github.com/google/osv.dev/tools/osv-scanner/internal/sbom"
)

// scanDir walks through the given directory to try to find any relevant files
func scanDir(query *osv.BatchedQuery, dir string) error {
	log.Printf("Scanning dir %s\n", dir)
	return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			log.Printf("Failed to walk %s: %v", path, err)
			return err
		}

		if info.IsDir() && info.Name() == ".git" {
			gitQuery, err := scanGit(filepath.Dir(path))
			if err != nil {
				log.Printf("scan failed for %s: %v\n", path, err)
				return err
			}
			gitQuery.Source = "git:" + filepath.Dir(path)
			query.Queries = append(query.Queries, gitQuery)
		}

		if !info.IsDir() {
			if parser, _ := lockfile.FindParser(path, ""); parser != nil {
				err := scanLockfile(query, path)
				if err != nil {
					log.Println("Attempted to scan lockfile but failed: " + path)
				}
			}
			// No need to check for error
			// If scan fails, it means it isn't a valid SBOM file,
			// so just move onto the next file
			_ = scanSBOMFile(query, path)
		}

		return nil
	})
}

func scanLockfile(query *osv.BatchedQuery, path string) error {
	log.Printf("Scanning file %s\n", path)

	parsedLockfile, err := lockfile.Parse(path, "")
	if err != nil {
		return err
	}
	log.Printf("Scanned %s file with %d packages", parsedLockfile.ParsedAs, len(parsedLockfile.Packages))

	for _, pkgDetail := range parsedLockfile.Packages {
		pkgDetailQuery := osv.MakePkgRequest(pkgDetail)
		pkgDetailQuery.Source = "lockfile:" + path
		query.Queries = append(query.Queries, pkgDetailQuery)
	}
	return nil
}

func scanSBOMFile(query *osv.BatchedQuery, path string) error {
	log.Printf("Scanning file %s\n", path)
	file, err := os.Open(path)
	if err != nil {
		return err
	}

	for _, provider := range sbom.Providers {
		if provider.Name() == "SPDX" &&
			!strings.Contains(strings.ToLower(filepath.Base(path)), ".spdx") {
			// All spdx files should have the .spdx in the filename, even if
			// it's not the extension:  https://spdx.github.io/spdx-spec/v2.3/conformance/
			// Skip if this isn't the case to avoid panics
			continue
		}
		err := provider.GetPackages(file, func(id sbom.Identifier) error {
			purlQuery := osv.MakePURLRequest(id.PURL)
			purlQuery.Source = "sbom:" + path
			query.Queries = append(query.Queries, purlQuery)
			return nil
		})
		if err == nil {
			// Found the right format.
			log.Printf("Scanned %s SBOM", provider.Name())
			return nil
		}

		if errors.Is(err, sbom.InvalidFormat) {
			continue
		}

		return err
	}

	return nil
}

func getCommitSHA(repoDir string) (string, error) {
	cmd := exec.Command("git", "-C", repoDir, "rev-parse", "HEAD")
	var out bytes.Buffer
	cmd.Stdout = &out

	err := cmd.Run()
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(out.String()), nil
}

func scanGit(repoDir string) (*osv.Query, error) {
	commit, err := getCommitSHA(repoDir)
	if err != nil {
		return nil, err
	}

	log.Printf("Scanning %s at commit %s", repoDir, commit)
	return osv.MakeCommitRequest(commit), nil
}

func scanDebianDocker(query *osv.BatchedQuery, dockerImageName string) {
	cmd := exec.Command("docker", "run", "--rm", dockerImageName, "/usr/bin/dpkg-query", "-f", "${Package}###${Version}\\n", "-W")
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatalf("Failed to get stdout: %s", err)
	}
	err = cmd.Start()
	if err != nil {
		log.Fatalf("Failed to start docker image: %s", err)
	}
	defer cmd.Wait()
	if err != nil {
		log.Fatalf("Failed to run docker: %s", err)
	}
	scanner := bufio.NewScanner(stdout)
	for scanner.Scan() {
		text := scanner.Text()
		text = strings.TrimSpace(text)
		if len(text) == 0 {
			continue
		}
		splitText := strings.Split(text, "###")
		if len(splitText) != 2 {
			log.Fatalf("Unexpected output from Debian container: \n\n%s", text)
		}
		pkgDetailsQuery := osv.MakePkgRequest(lockfile.PackageDetails{
			Name:    splitText[0],
			Version: splitText[1],
			// TODO(rexpan): Get and specify exact debian release version
			Ecosystem: "Debian",
		})
		pkgDetailsQuery.Source = "docker:" + dockerImageName
		query.Queries = append(query.Queries, pkgDetailsQuery)
	}
	log.Printf("Scanned docker image")
}

// printResults prints the osv scan results into a human friendly table.
func printResults(query osv.BatchedQuery, resp *osv.HydratedBatchedResponse) {
	outputTable := table.NewWriter()
	outputTable.SetOutputMirror(os.Stdout)
	outputTable.AppendHeader(table.Row{"Source", "Ecosystem", "Affected Package", "Installed Version", "Vulnerability ID", "OSV URL"})

	for i, query := range query.Queries {
		if len(resp.Results[i].Vulns) == 0 {
			continue
		}
		for _, group := range grouper.Group(resp.Results[i].Vulns) {
			outputRow := table.Row{query.Source}
			shouldMerge := false
			if query.Commit != "" {
				outputRow = append(outputRow, "GIT", query.Commit, query.Commit)
				shouldMerge = true
			} else if query.Package.PURL != "" {
				parsedPURL, err := packageurl.FromString(query.Package.PURL)
				if err != nil {
					log.Println("Failed to parse purl")
					continue
				}
				purlVersion := parsedPURL.Version
				parsedPURL.Version = ""
				parsedPURL.Qualifiers = []packageurl.Qualifier{}
				outputRow = append(outputRow, "PURL", parsedPURL.ToString(), purlVersion)
				shouldMerge = true
			} else {
				outputRow = append(outputRow, query.Package.Ecosystem, query.Package.Name, query.Version)
			}

			var ids []string
			var links []string

			for _, vuln := range group {
				ids = append(ids, vuln.ID)
				links = append(links, osv.BaseVulnerabilityURL+vuln.ID)
			}

			outputRow = append(outputRow, strings.Join(ids, "\n"), strings.Join(links, "\n"))
			outputTable.AppendRow(outputRow, table.RowConfig{AutoMerge: shouldMerge})
		}
	}

	outputTable.SetStyle(table.StyleRounded)
	outputTable.Style().Color.Row = text.Colors{text.Reset, text.BgBlack}
	outputTable.Style().Color.RowAlternate = text.Colors{text.Reset, text.Reset}
	// TODO: Leave these here until styling is finalized
	//outputTable.Style().Color.Header = text.Colors{text.FgHiCyan, text.BgBlack}
	//outputTable.Style().Color.Row = text.Colors{text.Reset, text.Reset}
	//outputTable.Style().Options.SeparateRows = true
	//outputTable.Style().Options.SeparateColumns = true
	//outputTable.SetColumnConfigs([]table.ColumnConfig{
	//	{Number: 2, AutoMerge: true, WidthMax: maxCharacters},
	//	{Number: 3, AutoMerge: true, WidthMax: maxCharacters},
	//})

	width, _, err := terminal.GetSize(int(os.Stdout.Fd()))
	if err == nil { // If output is a terminal, set max length to width
		outputTable.SetAllowedRowLength(width)
	} // Otherwise don't set max width (e.g. getting piped to a file)
	if outputTable.Length() == 0 {
		return
	}
	outputTable.Render()
}

// TODO(ochang): Machine readable output format.
func main() {
	var query osv.BatchedQuery

	app := &cli.App{
		Name:  "osv-scanner",
		Usage: "scans various mediums for dependencies and matches it against the OSV database",
		Flags: []cli.Flag{
			&cli.StringSliceFlag{
				Name:      "docker",
				Aliases:   []string{"D"},
				Usage:     "scan docker image with this name",
				TakesFile: false,
			},
			&cli.StringSliceFlag{
				Name:      "lockfile",
				Aliases:   []string{"L"},
				Usage:     "scan package lockfile on this path",
				TakesFile: true,
			},
			&cli.StringSliceFlag{
				Name:      "sbom",
				Aliases:   []string{"S"},
				Usage:     "scan sbom file on this path",
				TakesFile: true,
			},
			&cli.BoolFlag{
				Name:  "json",
				Usage: "sets output to json (WIP)",
			},
		},
		ArgsUsage: "[directory1 directory2...]",
		Action: func(context *cli.Context) error {
			containers := context.StringSlice("docker")
			for _, container := range containers {
				// TODO: Automatically figure out what docker base image
				// and scan appropriately.
				scanDebianDocker(&query, container)
			}

			lockfiles := context.StringSlice("lockfile")
			for _, lockfileElem := range lockfiles {
				err := scanLockfile(&query, lockfileElem)
				if err != nil {
					return err
				}
			}

			sboms := context.StringSlice("sbom")
			for _, sbomElem := range sboms {
				err := scanSBOMFile(&query, sbomElem)
				if err != nil {
					return err
				}
			}

			genericDirs := context.Args().Slice()
			for _, dir := range genericDirs {
				err := scanDir(&query, dir)
				if err != nil {
					return err
				}
			}

			if len(query.Queries) == 0 {
				cli.ShowAppHelpAndExit(context, 1)
			}

			return nil
		},
	}
	if err := app.Run(os.Args); err != nil {
		log.Fatal(err)
	}

	resp, err := osv.MakeRequest(query)
	if err != nil {
		log.Fatalf("Scan failed: %v", err)
	}

	hydratedResp, err := osv.Hydrate(resp)
	if err != nil {
		log.Fatalf("Failed to hydrate OSV response: %v", err)
	}

	printResults(query, hydratedResp)
}
<commit_msg>osv-scanner: Reduce some log spam. (#682)<commit_after>package main

import (
	"bufio"
	"bytes"
	"errors"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/g-rath/osv-detector/pkg/lockfile"
	"github.com/jedib0t/go-pretty/v6/table"
	"github.com/jedib0t/go-pretty/v6/text"
	"github.com/package-url/packageurl-go"
	"github.com/urfave/cli/v2"
	"golang.org/x/crypto/ssh/terminal"

	"github.com/google/osv.dev/tools/osv-scanner/internal/grouper"
	"github.com/google/osv.dev/tools/osv-scanner/internal/osv"
	"github.com/google/osv.dev/tools/osv-scanner/internal/sbom"
)

// scanDir walks through the given directory to try to find any relevant files
func scanDir(query *osv.BatchedQuery, dir string) error {
	log.Printf("Scanning dir %s\n", dir)
	return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			log.Printf("Failed to walk %s: %v", path, err)
			return err
		}

		if info.IsDir() && info.Name() == ".git" {
			gitQuery, err := scanGit(filepath.Dir(path))
			if err != nil {
				log.Printf("scan failed for %s: %v\n", path, err)
				return err
			}
			gitQuery.Source = "git:" + filepath.Dir(path)
			query.Queries = append(query.Queries, gitQuery)
		}

		if !info.IsDir() {
			if parser, _ := lockfile.FindParser(path, ""); parser != nil {
				err := scanLockfile(query, path)
				if err != nil {
					log.Println("Attempted to scan lockfile but failed: " + path)
				}
			}
			// No need to check for error
			// If scan fails, it means it isn't a valid SBOM file,
			// so just move onto the next file
			_ = scanSBOMFile(query, path)
		}

		return nil
	})
}

func scanLockfile(query *osv.BatchedQuery, path string) error {
	parsedLockfile, err := lockfile.Parse(path, "")
	if err != nil {
		return err
	}
	log.Printf("Scanned %s file and found %d packages", path, len(parsedLockfile.Packages))

	for _, pkgDetail := range parsedLockfile.Packages {
		pkgDetailQuery := osv.MakePkgRequest(pkgDetail)
		pkgDetailQuery.Source = "lockfile:" + path
		query.Queries = append(query.Queries, pkgDetailQuery)
	}
	return nil
}

func scanSBOMFile(query *osv.BatchedQuery, path string) error {
	file, err := os.Open(path)
	if err != nil {
		return err
	}

	for _, provider := range sbom.Providers {
		if provider.Name() == "SPDX" &&
			!strings.Contains(strings.ToLower(filepath.Base(path)), ".spdx") {
			// All spdx files should have the .spdx in the filename, even if
			// it's not the extension:  https://spdx.github.io/spdx-spec/v2.3/conformance/
			// Skip if this isn't the case to avoid panics
			continue
		}
		err := provider.GetPackages(file, func(id sbom.Identifier) error {
			purlQuery := osv.MakePURLRequest(id.PURL)
			purlQuery.Source = "sbom:" + path
			query.Queries = append(query.Queries, purlQuery)
			return nil
		})
		if err == nil {
			// Found the right format.
			log.Printf("Scanned %s SBOM", provider.Name())
			return nil
		}

		if errors.Is(err, sbom.InvalidFormat) {
			continue
		}

		return err
	}

	return nil
}

func getCommitSHA(repoDir string) (string, error) {
	cmd := exec.Command("git", "-C", repoDir, "rev-parse", "HEAD")
	var out bytes.Buffer
	cmd.Stdout = &out

	err := cmd.Run()
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(out.String()), nil
}

func scanGit(repoDir string) (*osv.Query, error) {
	commit, err := getCommitSHA(repoDir)
	if err != nil {
		return nil, err
	}

	log.Printf("Scanning %s at commit %s", repoDir, commit)
	return osv.MakeCommitRequest(commit), nil
}

func scanDebianDocker(query *osv.BatchedQuery, dockerImageName string) {
	cmd := exec.Command("docker", "run", "--rm", dockerImageName, "/usr/bin/dpkg-query", "-f", "${Package}###${Version}\\n", "-W")
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatalf("Failed to get stdout: %s", err)
	}
	err = cmd.Start()
	if err != nil {
		log.Fatalf("Failed to start docker image: %s", err)
	}
	defer cmd.Wait()
	if err != nil {
		log.Fatalf("Failed to run docker: %s", err)
	}
	scanner := bufio.NewScanner(stdout)
	for scanner.Scan() {
		text := scanner.Text()
		text = strings.TrimSpace(text)
		if len(text) == 0 {
			continue
		}
		splitText := strings.Split(text, "###")
		if len(splitText) != 2 {
			log.Fatalf("Unexpected output from Debian container: \n\n%s", text)
		}
		pkgDetailsQuery := osv.MakePkgRequest(lockfile.PackageDetails{
			Name:    splitText[0],
			Version: splitText[1],
			// TODO(rexpan): Get and specify exact debian release version
			Ecosystem: "Debian",
		})
		pkgDetailsQuery.Source = "docker:" + dockerImageName
		query.Queries = append(query.Queries, pkgDetailsQuery)
	}
	log.Printf("Scanned docker image")
}

// printResults prints the osv scan results into a human friendly table.
func printResults(query osv.BatchedQuery, resp *osv.HydratedBatchedResponse) {
	outputTable := table.NewWriter()
	outputTable.SetOutputMirror(os.Stdout)
	outputTable.AppendHeader(table.Row{"Source", "Ecosystem", "Affected Package", "Installed Version", "Vulnerability ID", "OSV URL"})

	for i, query := range query.Queries {
		if len(resp.Results[i].Vulns) == 0 {
			continue
		}
		for _, group := range grouper.Group(resp.Results[i].Vulns) {
			outputRow := table.Row{query.Source}
			shouldMerge := false
			if query.Commit != "" {
				outputRow = append(outputRow, "GIT", query.Commit, query.Commit)
				shouldMerge = true
			} else if query.Package.PURL != "" {
				parsedPURL, err := packageurl.FromString(query.Package.PURL)
				if err != nil {
					log.Println("Failed to parse purl")
					continue
				}
				purlVersion := parsedPURL.Version
				parsedPURL.Version = ""
				parsedPURL.Qualifiers = []packageurl.Qualifier{}
				outputRow = append(outputRow, "PURL", parsedPURL.ToString(), purlVersion)
				shouldMerge = true
			} else {
				outputRow = append(outputRow, query.Package.Ecosystem, query.Package.Name, query.Version)
			}

			var ids []string
			var links []string

			for _, vuln := range group {
				ids = append(ids, vuln.ID)
				links = append(links, osv.BaseVulnerabilityURL+vuln.ID)
			}

			outputRow = append(outputRow, strings.Join(ids, "\n"), strings.Join(links, "\n"))
			outputTable.AppendRow(outputRow, table.RowConfig{AutoMerge: shouldMerge})
		}
	}

	outputTable.SetStyle(table.StyleRounded)
	outputTable.Style().Color.Row = text.Colors{text.Reset, text.BgBlack}
	outputTable.Style().Color.RowAlternate = text.Colors{text.Reset, text.Reset}
	// TODO: Leave these here until styling is finalized
	//outputTable.Style().Color.Header = text.Colors{text.FgHiCyan, text.BgBlack}
	//outputTable.Style().Color.Row = text.Colors{text.Reset, text.Reset}
	//outputTable.Style().Options.SeparateRows = true
	//outputTable.Style().Options.SeparateColumns = true
	//outputTable.SetColumnConfigs([]table.ColumnConfig{
	//	{Number: 2, AutoMerge: true, WidthMax: maxCharacters},
	//	{Number: 3, AutoMerge: true, WidthMax: maxCharacters},
	//})

	width, _, err := terminal.GetSize(int(os.Stdout.Fd()))
	if err == nil { // If output is a terminal, set max length to width
		outputTable.SetAllowedRowLength(width)
	} // Otherwise don't set max width (e.g. getting piped to a file)
	if outputTable.Length() == 0 {
		return
	}
	outputTable.Render()
}

// TODO(ochang): Machine readable output format.
func main() {
	var query osv.BatchedQuery

	app := &cli.App{
		Name:  "osv-scanner",
		Usage: "scans various mediums for dependencies and matches it against the OSV database",
		Flags: []cli.Flag{
			&cli.StringSliceFlag{
				Name:      "docker",
				Aliases:   []string{"D"},
				Usage:     "scan docker image with this name",
				TakesFile: false,
			},
			&cli.StringSliceFlag{
				Name:      "lockfile",
				Aliases:   []string{"L"},
				Usage:     "scan package lockfile on this path",
				TakesFile: true,
			},
			&cli.StringSliceFlag{
				Name:      "sbom",
				Aliases:   []string{"S"},
				Usage:     "scan sbom file on this path",
				TakesFile: true,
			},
			&cli.BoolFlag{
				Name:  "json",
				Usage: "sets output to json (WIP)",
			},
		},
		ArgsUsage: "[directory1 directory2...]",
		Action: func(context *cli.Context) error {
			containers := context.StringSlice("docker")
			for _, container := range containers {
				// TODO: Automatically figure out what docker base image
				// and scan appropriately.
				scanDebianDocker(&query, container)
			}

			lockfiles := context.StringSlice("lockfile")
			for _, lockfileElem := range lockfiles {
				err := scanLockfile(&query, lockfileElem)
				if err != nil {
					return err
				}
			}

			sboms := context.StringSlice("sbom")
			for _, sbomElem := range sboms {
				err := scanSBOMFile(&query, sbomElem)
				if err != nil {
					return err
				}
			}

			genericDirs := context.Args().Slice()
			for _, dir := range genericDirs {
				err := scanDir(&query, dir)
				if err != nil {
					return err
				}
			}

			if len(query.Queries) == 0 {
				cli.ShowAppHelpAndExit(context, 1)
			}

			return nil
		},
	}
	if err := app.Run(os.Args); err != nil {
		log.Fatal(err)
	}

	resp, err := osv.MakeRequest(query)
	if err != nil {
		log.Fatalf("Scan failed: %v", err)
	}

	hydratedResp, err := osv.Hydrate(resp)
	if err != nil {
		log.Fatalf("Failed to hydrate OSV response: %v", err)
	}

	printResults(query, hydratedResp)
}
<|endoftext|>


<commit_before>package handlers

import (
	"net/http"
	"time"

	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/user"

	"github.com/icco/natnatnat/models"
	"github.com/pilu/traffic"
)

type RootData struct {
	Posts   interface{}
	IsAdmin bool
}

func RootHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := models.AllPosts(c)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	data := &RootData{Posts: entries, IsAdmin: user.IsAdmin(c)}
	w.Render("index", data)
}

func AboutHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Redirect(w, r.Request, "http://natwelch.com", 301)
}

func UnimplementedHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Error(w, "Sorry, I haven't implemented this yet", 500)
}

func MarkdownHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	err := r.ParseForm()
	if err != nil {
		log.Warningf(c, "Couldn't parse form: %v", r)
		http.Error(w, "Unable to parse request.", 500)
		return
	}

	in := r.Request.FormValue("text")
	md := models.Markdown(in)

	log.Infof(c, "Markdown Recieved: %s", in)
	log.Infof(c, "Markdown Rendered: %s", md)
	w.WriteText(string(md))
}

type ArchiveData struct {
	Years   map[int]Year
	IsAdmin bool
}

// TODO(icco): Rewrite to fix map iteration problems.
type Year map[time.Month]Month
type Month []Day
type Day []models.Entry

var months = [12]time.Month{
	time.January,
	time.February,
	time.March,
	time.April,
	time.May,
	time.June,
	time.July,
	time.August,
	time.September,
	time.October,
	time.November,
	time.December,
}

func ArchiveHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := models.Posts(c, -1, false)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	years := make(map[int]Year)

	oldest := (*entries)[0].Datetime.Year()
	newest := (*entries)[len(*entries)-1].Datetime.Year()

	for year := oldest; year <= newest; year += 1 {
		years[year] = make(Year)

		for _, month := range months {
			years[year][month] = make([]Day, daysIn(month, year))
		}
	}
	log.Infof(c, "Archive Data: %+v", years)

	for _, p := range *entries {
		year := p.Datetime.Year()
		month := p.Datetime.Month()
		day := p.Datetime.Day()

		years[year][month][day] = append(years[year][month][day], p)
	}

	data := &ArchiveData{Years: years, IsAdmin: user.IsAdmin(c)}
	w.Render("archive", data)
}

// daysIn returns the number of days in a month for a given year.
func daysIn(m time.Month, year int) int {
	// This is equivalent to time.daysIn(m, year).
	return time.Date(year, m+1, 0, 0, 0, 0, 0, time.UTC).Day()
}
<commit_msg>try to only show up till today<commit_after>package handlers

import (
	"net/http"
	"time"

	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/user"

	"github.com/icco/natnatnat/models"
	"github.com/pilu/traffic"
)

type RootData struct {
	Posts   interface{}
	IsAdmin bool
}

func RootHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := models.AllPosts(c)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	data := &RootData{Posts: entries, IsAdmin: user.IsAdmin(c)}
	w.Render("index", data)
}

func AboutHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Redirect(w, r.Request, "http://natwelch.com", 301)
}

func UnimplementedHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Error(w, "Sorry, I haven't implemented this yet", 500)
}

func MarkdownHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	err := r.ParseForm()
	if err != nil {
		log.Warningf(c, "Couldn't parse form: %v", r)
		http.Error(w, "Unable to parse request.", 500)
		return
	}

	in := r.Request.FormValue("text")
	md := models.Markdown(in)

	log.Infof(c, "Markdown Recieved: %s", in)
	log.Infof(c, "Markdown Rendered: %s", md)
	w.WriteText(string(md))
}

type ArchiveData struct {
	Years   map[int]Year
	IsAdmin bool
}

// TODO(icco): Rewrite to fix map iteration problems.
type Year map[time.Month]Month
type Month []Day
type Day []models.Entry

var months = [12]time.Month{
	time.January,
	time.February,
	time.March,
	time.April,
	time.May,
	time.June,
	time.July,
	time.August,
	time.September,
	time.October,
	time.November,
	time.December,
}

func ArchiveHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := models.Posts(c, -1, false)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	years := make(map[int]Year)

	oldest := (*entries)[0].Datetime
	newest := (*entries)[len(*entries)-1].Datetime

	for year := oldest.Year(); year <= newest.Year(); year += 1 {
		years[year] = make(Year)

		for _, month := range months {
			if year < newest.Year() || (year == newest.Year() && month <= newest.Month()) {
				years[year][month] = make([]Day, daysIn(month, year))
			}
		}
	}
	log.Infof(c, "Archive Data: %+v", years)

	for _, p := range *entries {
		year := p.Datetime.Year()
		month := p.Datetime.Month()
		day := p.Datetime.Day()

		years[year][month][day] = append(years[year][month][day], p)
	}

	data := &ArchiveData{Years: years, IsAdmin: user.IsAdmin(c)}
	w.Render("archive", data)
}

// daysIn returns the number of days in a month for a given year.
func daysIn(m time.Month, year int) int {
	// This is equivalent to time.daysIn(m, year).
	return time.Date(year, m+1, 0, 0, 0, 0, 0, time.UTC).Day()
}
<|endoftext|>


<commit_before><commit_msg>runtime package usage<commit_after><|endoftext|>


<commit_before>package futures

import (
	"context"
)

// DepthService show depth info
type DepthService struct {
	c      *Client
	symbol string
	limit  *int
}

// Symbol set symbol
func (s *DepthService) Symbol(symbol string) *DepthService {
	s.symbol = symbol
	return s
}

// Limit set limit
func (s *DepthService) Limit(limit int) *DepthService {
	s.limit = &limit
	return s
}

// Do send request
func (s *DepthService) Do(ctx context.Context, opts ...RequestOption) (res *DepthResponse, err error) {
	r := &request{
		method:   "GET",
		endpoint: "/fapi/v1/depth",
	}
	r.setParam("symbol", s.symbol)
	if s.limit != nil {
		r.setParam("limit", *s.limit)
	}
	data, err := s.c.callAPI(ctx, r, opts...)
	if err != nil {
		return nil, err
	}
	j, err := newJSON(data)
	if err != nil {
		return nil, err
	}
	res = new(DepthResponse)
	res.LastUpdateID = j.Get("lastUpdateId").MustInt64()
	bidsLen := len(j.Get("bids").MustArray())
	res.Bids = make([]Bid, bidsLen)
	for i := 0; i < bidsLen; i++ {
		item := j.Get("bids").GetIndex(i)
		res.Bids[i] = Bid{
			Price:    item.GetIndex(0).MustString(),
			Quantity: item.GetIndex(1).MustString(),
		}
	}
	asksLen := len(j.Get("asks").MustArray())
	res.Asks = make([]Ask, asksLen)
	for i := 0; i < asksLen; i++ {
		item := j.Get("asks").GetIndex(i)
		res.Asks[i] = Ask{
			Price:    item.GetIndex(0).MustString(),
			Quantity: item.GetIndex(1).MustString(),
		}
	}
	return res, nil
}

// DepthResponse define depth info with bids and asks
type DepthResponse struct {
	LastUpdateID int64 `json:"lastUpdateId"`
	Bids         []Bid `json:"bids"`
	Asks         []Ask `json:"asks"`
}

// Bid define bid info with price and quantity
type Bid struct {
	Price    string
	Quantity string
}

// Ask define ask info with price and quantity
type Ask struct {
	Price    string
	Quantity string
}
<commit_msg>Until now Bid and Ask were two distinct types for the same struct. (#250)<commit_after>package futures

import (
	"context"
	"strconv"
)

// DepthService show depth info
type DepthService struct {
	c      *Client
	symbol string
	limit  *int
}

// Symbol set symbol
func (s *DepthService) Symbol(symbol string) *DepthService {
	s.symbol = symbol
	return s
}

// Limit set limit
func (s *DepthService) Limit(limit int) *DepthService {
	s.limit = &limit
	return s
}

// Do send request
func (s *DepthService) Do(ctx context.Context, opts ...RequestOption) (res *DepthResponse, err error) {
	r := &request{
		method:   "GET",
		endpoint: "/fapi/v1/depth",
	}
	r.setParam("symbol", s.symbol)
	if s.limit != nil {
		r.setParam("limit", *s.limit)
	}
	data, err := s.c.callAPI(ctx, r, opts...)
	if err != nil {
		return nil, err
	}
	j, err := newJSON(data)
	if err != nil {
		return nil, err
	}
	res = new(DepthResponse)
	res.LastUpdateID = j.Get("lastUpdateId").MustInt64()
	bidsLen := len(j.Get("bids").MustArray())
	res.Bids = make([]Bid, bidsLen)
	for i := 0; i < bidsLen; i++ {
		item := j.Get("bids").GetIndex(i)
		res.Bids[i] = Bid{
			Price:    item.GetIndex(0).MustString(),
			Quantity: item.GetIndex(1).MustString(),
		}
	}
	asksLen := len(j.Get("asks").MustArray())
	res.Asks = make([]Ask, asksLen)
	for i := 0; i < asksLen; i++ {
		item := j.Get("asks").GetIndex(i)
		res.Asks[i] = Ask{
			Price:    item.GetIndex(0).MustString(),
			Quantity: item.GetIndex(1).MustString(),
		}
	}
	return res, nil
}

// DepthResponse define depth info with bids and asks
type DepthResponse struct {
	LastUpdateID int64 `json:"lastUpdateId"`
	Bids         []Bid `json:"bids"`
	Asks         []Ask `json:"asks"`
}

// PriceLevel is a common structure for bids and asks in the
// order book.
type PriceLevel struct {
	Price    string
	Quantity string
}

// Parse parses this PriceLevel's Price and Quantity and
// returns them both.  It also returns an error if either
// fails to parse.
func (p *PriceLevel) Parse() (float64, float64, error) {
	price, err := strconv.ParseFloat(p.Price, 64)
	if err != nil {
		return 0, 0, err
	}
	quantity, err := strconv.ParseFloat(p.Quantity, 64)
	if err != nil {
		return price, 0, err
	}
	return price, quantity, nil
}

// Ask is a type alias for PriceLevel.
type Ask = PriceLevel

// Bid is a tpye alias for PriceLevel.
type Bid = PriceLevel
<|endoftext|>


<commit_before>package db

import (
    "testing"
    "fmt"
    "os"
    "rand"
)

func TestLRUCache(t *testing.T) {
    strategy := NewLRUPeriodFlushStrategy(3, 100, 0)
    storage := NewInMemoryKeyValueStorage(10)
    flusher := &FakeFlusher{}

    cache := NewKeyValueCache(storage, strategy, flusher)
    fmt.Print("Start LRU cache test ...\t")

    for i := 0; i < 10; i++ {
        str := fmt.Sprint(i)
        cache.Show(str, str)
    }

    keys, _ := cache.Keys()
    if !same(convert2string([]int{7,9,8}), keys) {
        t.Errorf("should be [7 8 9], but %v", keys)
    }

    if v, _ := cache.Get("1"); v != nil {
        t.Errorf("%v should not be in cache", v)
    }

    cache.Get("7")
    cache.Show("1", "1")

    if v, _ := cache.Get("8"); v != nil {
        keys, _ := cache.Keys()
        t.Errorf("%v should not be in cache; cache content: %v", v, keys)
    }
    fmt.Print("OK\n")
}

func BenchmarkKeyValueCache(b *testing.B) {
    strategy := NewLRUPeriodFlushStrategy(200, 100, 0)
    storage := NewInMemoryKeyValueStorage(200)
    flusher := &FakeFlusher{}

    cache := NewKeyValueCache(storage, strategy, flusher)
    for i:= 0; i < 100000; i++ {
        key_int := rand.Int() % 1000
        key := fmt.Sprintf("%d", key_int)
        v, _ := cache.Get(key)
        if v == nil {
            cache.Show(key, key)
        }
    }
}
<commit_msg>an error in old test case<commit_after>package db

import (
    "testing"
    "fmt"
    "rand"
)

func TestLRUCache(t *testing.T) {
    strategy := NewLRUPeriodFlushStrategy(3, 100, 0)
    storage := NewInMemoryKeyValueStorage(10)
    flusher := &FakeFlusher{}

    cache := NewKeyValueCache(storage, strategy, flusher)
    fmt.Print("Start LRU cache test ...\t")

    for i := 0; i < 10; i++ {
        str := fmt.Sprint(i)
        cache.Show(str, str)
    }

    keys, _ := cache.Keys()
    if !same(convert2string([]int{7,9,8}), keys) {
        t.Errorf("should be [7 8 9], but %v", keys)
    }

    if v, _ := cache.Get("1"); v != nil {
        t.Errorf("%v should not be in cache", v)
    }

    cache.Get("7")
    cache.Show("1", "1")

    if v, _ := cache.Get("8"); v != nil {
        keys, _ := cache.Keys()
        t.Errorf("%v should not be in cache; cache content: %v", v, keys)
    }
    fmt.Print("OK\n")
}

func BenchmarkKeyValueCache(b *testing.B) {
    strategy := NewLRUPeriodFlushStrategy(200, 100, 0)
    storage := NewInMemoryKeyValueStorage(200)
    flusher := &FakeFlusher{}

    cache := NewKeyValueCache(storage, strategy, flusher)
    for i:= 0; i < 100000; i++ {
        key_int := rand.Int() % 1000
        key := fmt.Sprintf("%d", key_int)
        v, _ := cache.Get(key)
        if v == nil {
            cache.Show(key, key)
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2017 Tom Thorogood. All rights reserved.
// Use of this source code is governed by a Modified
// BSD License that can be found in the LICENSE file.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/tmthrgd/id3v2"
)

var dryrun = flag.Bool("dry-run", false, "does not perform the file renaming")

func scan(work workUnit) error {
	f, err := os.Open(work.path)
	if err != nil {
		return err
	}
	defer f.Close()

	frames, err := id3v2.Scan(f)
	if err != nil {
		return err
	}

	tpe1 := frames.Lookup(id3v2.FrameTPE1)
	if tpe1 == nil {
		if filepath.Ext(work.path) == ".mp3" {
			fmt.Printf("<%s>: missing TPE1 frame\n",
				work.path)
		}

		return nil
	}

	artist, err := tpe1.Text()
	if err != nil {
		return err
	}

	ext := filepath.Ext(work.path)
	if strings.HasSuffix(strings.TrimSuffix(work.path, ext), artist) {
		return nil
	}

	newPath := strings.TrimSuffix(work.path, ext) + " - " + artist + ext

	newURL := (&url.URL{
		Scheme: "file",
		Path:   newPath,
	}).String()
	*work.out = newURL

	fmt.Printf("%s: %s\n", filepath.Base(work.path), artist)

	if *dryrun {
		return nil
	}

	return os.Rename(work.path, newPath)
}

func worker(ch chan workUnit, wg *sync.WaitGroup) {
	for work := range ch {
		if err := scan(work); err != nil {
			log.Println(work.path, err)
		}

		wg.Done()
	}
}

type workUnit struct {
	path string
	out  *string
}

func main() {
	flag.Parse()

	if flag.NArg() == 0 {
		fmt.Println("Usage: id3-music-renamer.go [-dry-run] <m3u8 playlist> [<m3u8 playlist out>]")
		os.Exit(1)
	}

	r, err := os.Open(flag.Arg(0))
	if err != nil {
		panic(err)
	}
	defer r.Close()

	var wg sync.WaitGroup

	work := make(chan workUnit, 32)
	defer close(work)

	for i := 0; i < cap(work); i++ {
		go worker(work, &wg)
	}

	s := bufio.NewScanner(r)

	var out []*string

	for s.Scan() {
		line := s.Text()

		path, err := url.Parse(line)
		if err != nil {
			panic(err)
		}

		out = append(out, &line)

		if path.Scheme != "file" {
			continue
		}

		wg.Add(1)
		work <- workUnit{path.Path, out[len(out)-1]}
	}

	if s.Err() != nil {
		panic(s.Err())
	}

	wg.Wait()

	var outPath string
	if flag.NArg() == 2 {
		outPath = flag.Arg(1)
	} else {
		outPath = flag.Arg(0) + "-new.m3u8"
	}

	w, err := os.Create(outPath)
	if err != nil {
		panic(err)
	}
	defer w.Close()

	bw := bufio.NewWriter(w)
	defer bw.Flush()

	for _, line := range out {
		if _, err := io.WriteString(bw, *line); err != nil {
			panic(err)
		}

		if _, err := io.WriteString(bw, "\n"); err != nil {
			panic(err)
		}
	}
}
<commit_msg>Print worker errors to stderr<commit_after>// Copyright 2017 Tom Thorogood. All rights reserved.
// Use of this source code is governed by a Modified
// BSD License that can be found in the LICENSE file.

package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"io"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/tmthrgd/id3v2"
)

var dryrun = flag.Bool("dry-run", false, "does not perform the file renaming")

func scan(work workUnit) error {
	f, err := os.Open(work.path)
	if err != nil {
		return err
	}
	defer f.Close()

	frames, err := id3v2.Scan(f)
	if err != nil {
		return err
	}

	tpe1 := frames.Lookup(id3v2.FrameTPE1)
	if tpe1 == nil {
		if filepath.Ext(work.path) != ".mp3" {
			return nil
		}

		return errors.New("missing TIT2 or TPE1 frame")
	}

	artist, err := tpe1.Text()
	if err != nil {
		return err
	}

	ext := filepath.Ext(work.path)
	if strings.HasSuffix(strings.TrimSuffix(work.path, ext), artist) {
		return nil
	}

	newPath := strings.TrimSuffix(work.path, ext) + " - " + artist + ext

	newURL := (&url.URL{
		Scheme: "file",
		Path:   newPath,
	}).String()
	*work.out = newURL

	fmt.Printf("%s: %s\n", filepath.Base(work.path), artist)

	if *dryrun {
		return nil
	}

	return os.Rename(work.path, newPath)
}

func worker(ch chan workUnit, wg *sync.WaitGroup) {
	for work := range ch {
		if err := scan(work); err != nil {
			fmt.Fprintf(os.Stderr, "<%s>: %v\n", work.path, err)
		}

		wg.Done()
	}
}

type workUnit struct {
	path string
	out  *string
}

func main() {
	flag.Parse()

	if flag.NArg() == 0 {
		fmt.Println("Usage: id3-music-renamer.go [-dry-run] <m3u8 playlist> [<m3u8 playlist out>]")
		os.Exit(1)
	}

	r, err := os.Open(flag.Arg(0))
	if err != nil {
		panic(err)
	}
	defer r.Close()

	var wg sync.WaitGroup

	work := make(chan workUnit, 32)
	defer close(work)

	for i := 0; i < cap(work); i++ {
		go worker(work, &wg)
	}

	s := bufio.NewScanner(r)

	var out []*string

	for s.Scan() {
		line := s.Text()

		path, err := url.Parse(line)
		if err != nil {
			panic(err)
		}

		out = append(out, &line)

		if path.Scheme != "file" {
			continue
		}

		wg.Add(1)
		work <- workUnit{path.Path, out[len(out)-1]}
	}

	if s.Err() != nil {
		panic(s.Err())
	}

	wg.Wait()

	var outPath string
	if flag.NArg() == 2 {
		outPath = flag.Arg(1)
	} else {
		outPath = flag.Arg(0) + "-new.m3u8"
	}

	w, err := os.Create(outPath)
	if err != nil {
		panic(err)
	}
	defer w.Close()

	bw := bufio.NewWriter(w)
	defer bw.Flush()

	for _, line := range out {
		if _, err := io.WriteString(bw, *line); err != nil {
			panic(err)
		}

		if _, err := io.WriteString(bw, "\n"); err != nil {
			panic(err)
		}
	}
}
<|endoftext|>


<commit_before>// This example demonstrates some feutures of Go language: gorutines, channels,
// empty interface and type switch. This program looks like Go program and works
// like Go program but can run on MCU that has only 4 KB RAM and 16 KB Flash.
package main

import (
	"delay"
	"math/rand"
	"rtos"

	"stm32/hal/gpio"
	"stm32/hal/system"
	"stm32/hal/system/timer/systick"
)

var led gpio.Pin

func init() {
	system.SetupPLL(8, 1, 48/8)
	systick.Setup(2e6)

	gpio.A.EnableClock(false)
	led = gpio.A.Pin(4)

	cfg := &gpio.Config{Mode: gpio.Out, Driver: gpio.OpenDrain, Speed: gpio.Low}
	led.Setup(cfg)
}

type On int

func task1(ch chan<- interface{}) {
	var r rand.XorShift64
	r.Seed(rtos.Nanosec())
	for {
		ch <- On(rnd.Int32()&511 + 89)
	}
}

type Off int

func task2(ch chan<- interface{}) {
	var rnd rand.XorShift64
	rnd.Seed(rtos.Nanosec())
	for {
		ch <- Off(rnd.Int32()&511 + 50)
	}
}

func main() {
	ch := make(chan interface{})
	go task1(ch)
	go task2(ch)
	for val := range ch {
		switch ms := val.(type) {
		case On:
			led.Clear() // Turn LED on.
			delay.Millisec(int(ms))
		case Off:
			led.Set() // Turn LED off.
			delay.Millisec(int(ms))
		}
	}
}
<commit_msg>examples/f030-demo-board/demo: Fix regresion.<commit_after>// This example demonstrates some feutures of Go language: gorutines, channels,
// empty interface and type switch. This program looks like Go program and works
// like Go program but can run on MCU that has only 4 KB RAM and 16 KB Flash.
package main

import (
	"delay"
	"math/rand"
	"rtos"

	"stm32/hal/gpio"
	"stm32/hal/system"
	"stm32/hal/system/timer/systick"
)

var led gpio.Pin

func init() {
	system.SetupPLL(8, 1, 48/8)
	systick.Setup(2e6)

	gpio.A.EnableClock(false)
	led = gpio.A.Pin(4)

	cfg := &gpio.Config{Mode: gpio.Out, Driver: gpio.OpenDrain, Speed: gpio.Low}
	led.Setup(cfg)
}

type On int

func task1(ch chan<- interface{}) {
	var rnd rand.XorShift64
	rnd.Seed(rtos.Nanosec())
	for {
		ch <- On(rnd.Int32()&511 + 89)
	}
}

type Off int

func task2(ch chan<- interface{}) {
	var rnd rand.XorShift64
	rnd.Seed(rtos.Nanosec())
	for {
		ch <- Off(rnd.Int32()&511 + 50)
	}
}

func main() {
	ch := make(chan interface{})
	go task1(ch)
	go task2(ch)
	for val := range ch {
		switch ms := val.(type) {
		case On:
			led.Clear() // Turn LED on.
			delay.Millisec(int(ms))
		case Off:
			led.Set() // Turn LED off.
			delay.Millisec(int(ms))
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>add made-up addresses for witty outputs on segnet3<commit_after><|endoftext|>


<commit_before>package handlers

import (
	"errors"
	"fmt"
	"net"

	"github.com/go-openapi/runtime/middleware"
	"github.com/go-openapi/validate"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/sapcc/kubernikus/pkg/api"
	"github.com/sapcc/kubernikus/pkg/api/models"
	"github.com/sapcc/kubernikus/pkg/api/rest/operations"
	"github.com/sapcc/kubernikus/pkg/apis/kubernikus"
	"github.com/sapcc/kubernikus/pkg/util/ip"
	k8sutil "github.com/sapcc/kubernikus/pkg/util/k8s"
)

func NewCreateCluster(rt *api.Runtime) operations.CreateClusterHandler {
	return &createCluster{Runtime: rt}
}

type createCluster struct {
	*api.Runtime
	cpServiceCIDR *net.IPNet
	cpClusterCIDR *net.IPNet
}

func (d *createCluster) Handle(params operations.CreateClusterParams, principal *models.Principal) middleware.Responder {
	logger := getTracingLogger(params.HTTPRequest)
	name := params.Body.Name
	spec := params.Body.Spec

	if err := validate.UniqueItems("name", "body", params.Body.Spec.NodePools); err != nil {
		return NewErrorResponse(&operations.CreateClusterDefault{}, int(err.Code()), err.Error())
	}

	var metadata *models.OpenstackMetadata
	var defaultAVZ string
	if len(spec.NodePools) > 0 {
		m, err := FetchOpenstackMetadataFunc(params.HTTPRequest, principal)
		if err != nil {
			return NewErrorResponse(&operations.CreateClusterDefault{}, 500, err.Error())
		}
		metadata = m

		avz, err := getDefaultAvailabilityZone(metadata)
		if err != nil {
			return NewErrorResponse(&operations.CreateClusterDefault{}, 500, err.Error())
		}
		defaultAVZ = avz
	}

	spec.Name = name
	for i, pool := range spec.NodePools {
		// Set default image
		if pool.Image == "" {
			spec.NodePools[i].Image = DEFAULT_IMAGE
		}

		// Set default AvailabilityZone
		if pool.AvailabilityZone == "" {
			spec.NodePools[i].AvailabilityZone = defaultAVZ
		}

		allowReboot := true
		allowReplace := true
		if pool.Config == nil {
			spec.NodePools[i].Config = &models.NodePoolConfig{
				AllowReboot:  &allowReboot,
				AllowReplace: &allowReplace,
			}
		}

		if spec.NodePools[i].Config.AllowReboot == nil {
			spec.NodePools[i].Config.AllowReboot = &allowReboot
		}

		if spec.NodePools[i].Config.AllowReplace == nil {
			spec.NodePools[i].Config.AllowReplace = &allowReplace
		}

		// Validate AVZ
		if err := validateAavailabilityZone(spec.NodePools[i].AvailabilityZone, metadata); err != nil {
			return NewErrorResponse(&operations.CreateClusterDefault{}, 409, "Availability Zone %s is invalid: %s", spec.NodePools[i].AvailabilityZone, err)
		}
	}

	kluster, err := kubernikus.NewKlusterFactory().KlusterFor(spec)
	if err != nil {
		logger.Log(
			"msg", "failed to create cluster",
			"kluster", name,
			"project", principal.Account,
			"err", err)
		return NewErrorResponse(&operations.CreateClusterDefault{}, 400, err.Error())
	}

	//Ensure that the service CIDR range does not overlap with any control plane CIDR
	//Otherwise the wormhole server will prevent the kluster apiserver from functioning properly
	if overlap, err := d.overlapWithControlPlane(kluster.Spec.ServiceCIDR); overlap {
		return NewErrorResponse(&operations.CreateClusterDefault{}, 409, "Service CIDR %s not allowed: %s", kluster.Spec.ServiceCIDR, err)
	}
	//Ensure that the cluster CIDR range does not overlap with any control plane CIDR
	//Otherwise the wormhole server will prevent the kluster apiserver from functioning properly
	if overlap, err := d.overlapWithControlPlane(kluster.Spec.ClusterCIDR); overlap {
		return NewErrorResponse(&operations.CreateClusterDefault{}, 409, "Cluster CIDR %s not allowed: %s", kluster.Spec.ClusterCIDR, err)
	}

	//Ensure that the clust CIDR range does not overlap with other clusters in the same project
	if !kluster.Spec.NoCloud {
		if overlap, err := d.overlapWithSiblingCluster(kluster.Spec.ClusterCIDR, kluster.Spec.Openstack.RouterID, principal); overlap || err != nil {
			if overlap {
				return NewErrorResponse(&operations.CreateClusterDefault{}, 409, err.Error())
			}
			return NewErrorResponse(&operations.CreateClusterDefault{}, 500, err.Error())
		}
	}

	kluster.ObjectMeta = metav1.ObjectMeta{
		Name:        qualifiedName(name, principal.Account),
		Labels:      map[string]string{"account": principal.Account},
		Annotations: map[string]string{"creator": principal.Name},
	}

	k8sutil.EnsureNamespace(d.Kubernetes, d.Namespace)
	kluster, err = d.Kubernikus.Kubernikus().Klusters(d.Namespace).Create(kluster)
	if err != nil {
		logger.Log(
			"msg", "failed to create cluster",
			"kluster", kluster.GetName(),
			"project", kluster.Account(),
			"err", err)

		if apierrors.IsAlreadyExists(err) {
			return NewErrorResponse(&operations.CreateClusterDefault{}, 409, "Cluster with name %s already exists", name)
		}
		return NewErrorResponse(&operations.CreateClusterDefault{}, 500, err.Error())
	}

	return operations.NewCreateClusterCreated().WithPayload(klusterFromCRD(kluster))
}

func (d *createCluster) overlapWithControlPlane(cidr string) (bool, error) {
	_, inputCIDR, err := net.ParseCIDR(cidr)
	if err != nil {
		return false, err
	}
	clusterCIDR := d.controlPlaneClusterCIDR()
	if clusterCIDR != nil && ip.CIDROverlap(inputCIDR, clusterCIDR) {
		return true, errors.New("overlap with control plane cluster CIDR")
	}
	svcCIDR := d.controlPlaneServiceCIDR()
	if svcCIDR != nil && ip.CIDROverlap(inputCIDR, svcCIDR) {
		return true, errors.New("overlap with control plane service CIDR")
	}
	return false, nil
}

func (d *createCluster) overlapWithSiblingCluster(cidr string, routerID string, principal *models.Principal) (bool, error) {
	listOpts := metav1.ListOptions{LabelSelector: accountSelector(principal).String()}
	klusterList, err := d.Kubernikus.Kubernikus().Klusters(d.Namespace).List(listOpts)
	if err != nil {
		return false, err
	}
	for _, other := range klusterList.Items {
		if routerID == "" || routerID == other.Spec.Openstack.RouterID {
			_, ourCIDR, err := net.ParseCIDR(cidr)
			if err != nil {
				return false, err
			}
			_, otherCIDR, err := net.ParseCIDR(other.Spec.ClusterCIDR)
			if err != nil {
				return false, err
			}
			if ip.CIDROverlap(ourCIDR, otherCIDR) {
				return true, fmt.Errorf("Cluster CIDR %s overlaps with cluster CIDR %s from cluster '%s'. Specify a different CIDR Range or use a dedicated router for this cluster", cidr, other.Spec.ClusterCIDR, other.Spec.Name)
			}
		}
	}
	return false, nil
}

//approximate the control plane service CIDR by getting one service IP and assuming a /17 prefix
func (d *createCluster) controlPlaneServiceCIDR() *net.IPNet {
	if d.cpServiceCIDR != nil {
		return d.cpServiceCIDR
	}
	svc, err := d.Kubernetes.Core().Services("default").Get("kubernetes", metav1.GetOptions{})
	if err != nil {
		return nil
	}
	_, ipnet, err := net.ParseCIDR(svc.Spec.ClusterIP + "/17")
	if err != nil {
		return nil
	}
	d.cpServiceCIDR = ipnet
	return d.cpServiceCIDR
}

//we infer the clusterCIDR by taking a Pod IP and assuming /16
func (d *createCluster) controlPlaneClusterCIDR() *net.IPNet {
	if d.cpClusterCIDR != nil {
		return d.cpClusterCIDR
	}
	podList, err := d.Kubernetes.Core().Pods(metav1.NamespaceAll).List(metav1.ListOptions{Limit: 1})
	if err != nil || len(podList.Items) == 0 {
		return nil
	}
	_, ipnet, err := net.ParseCIDR(podList.Items[0].Status.PodIP + "/16")

	if err != nil {
		return nil
	}
	d.cpClusterCIDR = ipnet
	return d.cpClusterCIDR
}
<commit_msg>Wait up to 2 seconds after cluster creation to have it show up in the local cache<commit_after>package handlers

import (
	"errors"
	"fmt"
	"net"
	"time"

	"github.com/go-openapi/runtime/middleware"
	"github.com/go-openapi/validate"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"

	"github.com/sapcc/kubernikus/pkg/api"
	"github.com/sapcc/kubernikus/pkg/api/models"
	"github.com/sapcc/kubernikus/pkg/api/rest/operations"
	"github.com/sapcc/kubernikus/pkg/apis/kubernikus"
	"github.com/sapcc/kubernikus/pkg/util/ip"
	k8sutil "github.com/sapcc/kubernikus/pkg/util/k8s"
)

func NewCreateCluster(rt *api.Runtime) operations.CreateClusterHandler {
	return &createCluster{Runtime: rt}
}

type createCluster struct {
	*api.Runtime
	cpServiceCIDR *net.IPNet
	cpClusterCIDR *net.IPNet
}

func (d *createCluster) Handle(params operations.CreateClusterParams, principal *models.Principal) middleware.Responder {
	logger := getTracingLogger(params.HTTPRequest)
	name := params.Body.Name
	spec := params.Body.Spec

	if err := validate.UniqueItems("name", "body", params.Body.Spec.NodePools); err != nil {
		return NewErrorResponse(&operations.CreateClusterDefault{}, int(err.Code()), err.Error())
	}

	var metadata *models.OpenstackMetadata
	var defaultAVZ string
	if len(spec.NodePools) > 0 {
		m, err := FetchOpenstackMetadataFunc(params.HTTPRequest, principal)
		if err != nil {
			return NewErrorResponse(&operations.CreateClusterDefault{}, 500, err.Error())
		}
		metadata = m

		avz, err := getDefaultAvailabilityZone(metadata)
		if err != nil {
			return NewErrorResponse(&operations.CreateClusterDefault{}, 500, err.Error())
		}
		defaultAVZ = avz
	}

	spec.Name = name
	for i, pool := range spec.NodePools {
		// Set default image
		if pool.Image == "" {
			spec.NodePools[i].Image = DEFAULT_IMAGE
		}

		// Set default AvailabilityZone
		if pool.AvailabilityZone == "" {
			spec.NodePools[i].AvailabilityZone = defaultAVZ
		}

		allowReboot := true
		allowReplace := true
		if pool.Config == nil {
			spec.NodePools[i].Config = &models.NodePoolConfig{
				AllowReboot:  &allowReboot,
				AllowReplace: &allowReplace,
			}
		}

		if spec.NodePools[i].Config.AllowReboot == nil {
			spec.NodePools[i].Config.AllowReboot = &allowReboot
		}

		if spec.NodePools[i].Config.AllowReplace == nil {
			spec.NodePools[i].Config.AllowReplace = &allowReplace
		}

		// Validate AVZ
		if err := validateAavailabilityZone(spec.NodePools[i].AvailabilityZone, metadata); err != nil {
			return NewErrorResponse(&operations.CreateClusterDefault{}, 409, "Availability Zone %s is invalid: %s", spec.NodePools[i].AvailabilityZone, err)
		}
	}

	kluster, err := kubernikus.NewKlusterFactory().KlusterFor(spec)
	if err != nil {
		logger.Log(
			"msg", "failed to create cluster",
			"kluster", name,
			"project", principal.Account,
			"err", err)
		return NewErrorResponse(&operations.CreateClusterDefault{}, 400, err.Error())
	}

	//Ensure that the service CIDR range does not overlap with any control plane CIDR
	//Otherwise the wormhole server will prevent the kluster apiserver from functioning properly
	if overlap, err := d.overlapWithControlPlane(kluster.Spec.ServiceCIDR); overlap {
		return NewErrorResponse(&operations.CreateClusterDefault{}, 409, "Service CIDR %s not allowed: %s", kluster.Spec.ServiceCIDR, err)
	}
	//Ensure that the cluster CIDR range does not overlap with any control plane CIDR
	//Otherwise the wormhole server will prevent the kluster apiserver from functioning properly
	if overlap, err := d.overlapWithControlPlane(kluster.Spec.ClusterCIDR); overlap {
		return NewErrorResponse(&operations.CreateClusterDefault{}, 409, "Cluster CIDR %s not allowed: %s", kluster.Spec.ClusterCIDR, err)
	}

	//Ensure that the clust CIDR range does not overlap with other clusters in the same project
	if !kluster.Spec.NoCloud {
		if overlap, err := d.overlapWithSiblingCluster(kluster.Spec.ClusterCIDR, kluster.Spec.Openstack.RouterID, principal); overlap || err != nil {
			if overlap {
				return NewErrorResponse(&operations.CreateClusterDefault{}, 409, err.Error())
			}
			return NewErrorResponse(&operations.CreateClusterDefault{}, 500, err.Error())
		}
	}

	kluster.ObjectMeta = metav1.ObjectMeta{
		Name:        qualifiedName(name, principal.Account),
		Labels:      map[string]string{"account": principal.Account},
		Annotations: map[string]string{"creator": principal.Name},
	}

	k8sutil.EnsureNamespace(d.Kubernetes, d.Namespace)
	kluster, err = d.Kubernikus.Kubernikus().Klusters(d.Namespace).Create(kluster)
	if err != nil {
		logger.Log(
			"msg", "failed to create cluster",
			"kluster", kluster.GetName(),
			"project", kluster.Account(),
			"err", err)

		if apierrors.IsAlreadyExists(err) {
			return NewErrorResponse(&operations.CreateClusterDefault{}, 409, "Cluster with name %s already exists", name)
		}
		return NewErrorResponse(&operations.CreateClusterDefault{}, 500, err.Error())
	}

	//Wait for a second so that the newly created cluster shows up in the cache
	//This is a hack so that a subsequent GET /api/v1/cluster/:name will not return 404
	wait.Poll(50*time.Millisecond, 2*time.Second, func() (bool, error) {
		if _, err := d.Klusters.Klusters(d.Namespace).Get(kluster.Name); err != nil {
			return false, nil
		}
		return true, nil
	})

	return operations.NewCreateClusterCreated().WithPayload(klusterFromCRD(kluster))
}

func (d *createCluster) overlapWithControlPlane(cidr string) (bool, error) {
	_, inputCIDR, err := net.ParseCIDR(cidr)
	if err != nil {
		return false, err
	}
	clusterCIDR := d.controlPlaneClusterCIDR()
	if clusterCIDR != nil && ip.CIDROverlap(inputCIDR, clusterCIDR) {
		return true, errors.New("overlap with control plane cluster CIDR")
	}
	svcCIDR := d.controlPlaneServiceCIDR()
	if svcCIDR != nil && ip.CIDROverlap(inputCIDR, svcCIDR) {
		return true, errors.New("overlap with control plane service CIDR")
	}
	return false, nil
}

func (d *createCluster) overlapWithSiblingCluster(cidr string, routerID string, principal *models.Principal) (bool, error) {
	listOpts := metav1.ListOptions{LabelSelector: accountSelector(principal).String()}
	klusterList, err := d.Kubernikus.Kubernikus().Klusters(d.Namespace).List(listOpts)
	if err != nil {
		return false, err
	}
	for _, other := range klusterList.Items {
		if routerID == "" || routerID == other.Spec.Openstack.RouterID {
			_, ourCIDR, err := net.ParseCIDR(cidr)
			if err != nil {
				return false, err
			}
			_, otherCIDR, err := net.ParseCIDR(other.Spec.ClusterCIDR)
			if err != nil {
				return false, err
			}
			if ip.CIDROverlap(ourCIDR, otherCIDR) {
				return true, fmt.Errorf("Cluster CIDR %s overlaps with cluster CIDR %s from cluster '%s'. Specify a different CIDR Range or use a dedicated router for this cluster", cidr, other.Spec.ClusterCIDR, other.Spec.Name)
			}
		}
	}
	return false, nil
}

//approximate the control plane service CIDR by getting one service IP and assuming a /17 prefix
func (d *createCluster) controlPlaneServiceCIDR() *net.IPNet {
	if d.cpServiceCIDR != nil {
		return d.cpServiceCIDR
	}
	svc, err := d.Kubernetes.Core().Services("default").Get("kubernetes", metav1.GetOptions{})
	if err != nil {
		return nil
	}
	_, ipnet, err := net.ParseCIDR(svc.Spec.ClusterIP + "/17")
	if err != nil {
		return nil
	}
	d.cpServiceCIDR = ipnet
	return d.cpServiceCIDR
}

//we infer the clusterCIDR by taking a Pod IP and assuming /16
func (d *createCluster) controlPlaneClusterCIDR() *net.IPNet {
	if d.cpClusterCIDR != nil {
		return d.cpClusterCIDR
	}
	podList, err := d.Kubernetes.Core().Pods(metav1.NamespaceAll).List(metav1.ListOptions{Limit: 1})
	if err != nil || len(podList.Items) == 0 {
		return nil
	}
	_, ipnet, err := net.ParseCIDR(podList.Items[0].Status.PodIP + "/16")

	if err != nil {
		return nil
	}
	d.cpClusterCIDR = ipnet
	return d.cpClusterCIDR
}
<|endoftext|>


<commit_before>package proto

import (
	"os"
	"io/ioutil"
	pb "goprotobuf.googlecode.com/hg/proto"
)

func NewSlug(name string, version string, stages int) (slug *Slug, err os.Error) {
	slug = &Slug{
		Name:         pb.String(name),
		Version:      pb.String(version),
		Transformers: make([]*Transform, stages),
	}

	return
}

func NewSlugFromFile(filename string) (slug *Slug, err os.Error) {
	var data []byte

	data, err = ioutil.ReadFile(filename)
	if err != nil {
		return
	}

	slug = &Slug{}
	err = pb.Unmarshal(data, slug)
	if err != nil {
		return
	}

	return
}

func (slug *Slug) WriteFile(filename string) (err os.Error) {
	var data []byte

	data, err = pb.Marshal(slug)
	if err != nil {
		return
	}

	err = ioutil.WriteFile(filename, data, 0644)
	if err != nil {
		return
	}

	return
}
<commit_msg>Add ability to get slug from remote url<commit_after>package proto

import (
	"os"
	"io/ioutil"
	"http"
	pb "goprotobuf.googlecode.com/hg/proto"
)

func NewSlug(name string, version string, stages int) (slug *Slug, err os.Error) {
	slug = &Slug{
		Name:         pb.String(name),
		Version:      pb.String(version),
		Transformers: make([]*Transform, stages),
	}

	return
}

func NewSlugFromFile(filename string) (slug *Slug, err os.Error) {
	var data []byte

	data, err = ioutil.ReadFile(filename)
	if err != nil {
		return
	}

	slug = &Slug{}
	err = pb.Unmarshal(data, slug)
	if err != nil {
		return
	}

	return
}

func NewSlugFromURL(url string) (slug *Slug, err os.Error) {
	var data []byte
	var resp *http.Response

	resp, err = http.Get(url)
	if resp.StatusCode != 200 {
		err = os.NewError("Couldn't load slug from " + url)
		return
	}
	if err != nil {
		return
	}

	data, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	if len(data) == 0 {
		err = os.NewError("Received blank slug")
		return
	}

	slug = &Slug{}
	err = pb.Unmarshal(data, slug)
	if err != nil {
		return
	}

	return
}

func (slug *Slug) WriteFile(filename string) (err os.Error) {
	var data []byte

	data, err = pb.Marshal(slug)
	if err != nil {
		return
	}

	err = ioutil.WriteFile(filename, data, 0644)
	if err != nil {
		return
	}

	return
}
<|endoftext|>


<commit_before>package pipelines_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("A job with a task using an image within the plan", func() {
	BeforeEach(func() {
		configurePipeline(
			"-c", "fixtures/image-artifact.yml",
		)
	})

	It("uses the specified image artifact to run the task", func() {
		watch := flyWatch("artifact-test")
		Expect(watch).To(gbytes.Say("/bin/bash"))
	})
})
<commit_msg>skip image artifact test if no tagged worker<commit_after>package pipelines_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("A job with a task using an image within the plan", func() {
	BeforeEach(func() {
		configurePipeline(
			"-c", "fixtures/image-artifact.yml",
		)

		var hasTaggedWorker bool
	dance:
		for _, worker := range workers {
			for _, tag := range worker.Tags {
				if tag == "tagged" {
					hasTaggedWorker = true
					break dance
				}
			}
		}

		if !hasTaggedWorker {
			Skip("this only runs when a worker with the 'tagged' tag is available")
		}
	})

	It("uses the specified image artifact to run the task", func() {
		watch := flyWatch("artifact-test")
		Expect(watch).To(gbytes.Say("/bin/bash"))
	})
})
<|endoftext|>


<commit_before>package main

import "oauth1"
import "fmt"
import "net/http"
import "net/url"
import "io/ioutil"

//import "strings"

const VERSION = "V3"

type FactualQuery struct {
	query string
	limit int
}

func EncodeURLParameters(urlstring string) string {
	encodedURLValue, err := url.Parse(urlstring)
	if err != nil {
		panic("Could not parse the url")
	}
	encodedURLValue.RawQuery = encodedURLValue.Query().Encode()

	return encodedURLValue.String()
}

func AddQuery(urlstring *string, query string) string {

	query = "q=" + query
	return *urlstring + query

}

func Read() {

}

func main() {
	token := oauth1.NewAccessToken("Pbu7jRdBErgLW07g9c25JtGcwwt1KmpoxRTfFL3x", "vC4AgocPBhxe0GFkTsetoiuEAJEgqz6MCbAnXEoO", "", "")
	urlstring := EncodeURLParameters("http://api.v3.factual.com/t/restaurants-us/schema")
	fmt.Println("TESTING QUERY STRING", AddQuery(&urlstring, "Los gatos"))
	request, err := http.NewRequest("GET", urlstring, nil)
	client := &http.Client{}

	if err != nil {
		fmt.Println("ERROR: ", err)
	}

	token.SignRequestHeader(request)

	result, err := client.Do(request)

	if err != nil {
		fmt.Println("ERROR: ", err)
	}

	content, err := ioutil.ReadAll(result.Body)

	defer result.Body.Close()

	if err != nil {
		fmt.Println("ERROR: ", err)
	}

	//fmt.Println(request)
	fmt.Println(result)

	fmt.Println(request)
	fmt.Printf("%s", content)

}
<commit_msg>Modularized Code.  Added factualRequest struct, NewRequest function, NewRead Method, GetReadURL() and GetRead() Method<commit_after>package main

import "oauth1"
import "fmt"
import "net/http"
import "net/url"
import "io/ioutil"
import "./read"
import "errors"

//import "strings"

const VERSION = "v3"
const FACTUAL_BASE_URL = "http://api.v3.factual.com"

func EncodeURLParameters(urlstring string) string {
	encodedURLValue, err := url.Parse(urlstring)
	if err != nil {
		panic("Could not parse the url")
	}
	encodedURLValue.RawQuery = encodedURLValue.Query().Encode()

	return encodedURLValue.String()
}

type factualRequest struct {
	token *oauth1.AccessToken
	read.Read
}

func NewRequest(consumerKey, consumerSecret string) *factualRequest {
	f := new(factualRequest)
	f.token = oauth1.NewAccessToken(consumerKey, consumerSecret, "", "")
	return f
}

func (F *factualRequest) NewRead(tableName string) (*factualRequest, error) {
	f, err := read.NewReader(tableName)
	if err != nil {
		return nil, err
	}
	F.Read = f
	return F, nil

}

func (F *factualRequest) GetReadURL() string {
	return FACTUAL_BASE_URL + F.String()
}

func (F *factualRequest) GetRead() (string, error) {
	if F.Read == nil {
		return "", errors.New("Factual Request has no read data")
	}
	encodedURL := EncodeURLParameters(F.GetReadURL())
	request, err := http.NewRequest("GET", encodedURL, nil)
	if err != nil {
		return "", err
	}
	client := &http.Client{}
	F.token.SignRequestHeader(request)
	result, err := client.Do(request)
	if err != nil {
		return "", err
	}
	defer result.Body.Close()

	content, err := ioutil.ReadAll(result.Body)

	if err != nil {
		return "", err
	}

	return string(content), nil

}

func main() {

	fR := NewRequest("Pbu7jRdBErgLW07g9c25JtGcwwt1KmpoxRTfFL3x", "vC4AgocPBhxe0GFkTsetoiuEAJEgqz6MCbAnXEoO")

	_, err := fR.NewRead("restaurants-us")

	if err != nil {
		fmt.Println(err)
		return
	}

	fR.AddQuery("pizza", "sushi")
	fR.AddLimit(5)
	fR.AddSelect("factual_id")

	fmt.Println("frData", fR.GetReadURL())

	s, err := fR.GetRead()

	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Println(s)

	//token := oauth1.NewAccessToken("Pbu7jRdBErgLW07g9c25JtGcwwt1KmpoxRTfFL3x", "vC4AgocPBhxe0GFkTsetoiuEAJEgqz6MCbAnXEoO", "", "")
	/*
		r, err := read.NewRead("restaurants-us")
		if err != nil {
			fmt.Println(err)
			return
		}

		r, err = r.AddQuery("Coffee", "\"Los Angeles\"")

		if err != nil {
			fmt.Println(err)
			return
		}

		r = r.AddLimit(5)
		//r, err = r.AddSelect("cuisine")

		if err != nil {
			fmt.Println(err)
			return
		}*/

	/*fmt.Println("TESTING RAW QUERY STRING", FACTUAL_BASE_URL+r.String())
	urlstring := EncodeURLParameters(FACTUAL_BASE_URL + r.String())
	durlstring := EncodeURLParameters("http://api.v3.factual.com/t/restaurants-us?q=Coffee,\"Los Angeles\"&limit=5&select=factual_id")
	fmt.Println("TESTING QUERY STRING", durlstring, urlstring)
	request, err := http.NewRequest("GET", durlstring, nil)
	client := &http.Client{}

	if err != nil {
		fmt.Println("ERROR: ", err)
	}

	token.SignRequestHeader(request)

	result, err := client.Do(request)

	if err != nil {
		fmt.Println("ERROR: ", err)
	}

	//_, err := ioutil.ReadAll(result.Body)
	content, err := ioutil.ReadAll(result.Body)

	defer result.Body.Close()

	if err != nil {
		fmt.Println("ERROR: ", err)
	}

	//fmt.Println(request)
	fmt.Println("result => ", result)

	fmt.Println("request => ", request, "REQUEST_URL => ", request.URL)
	fmt.Printf("%s", content)
	*/

}
<|endoftext|>


<commit_before>// Package mysql implements reform.Dialect for MySQL.
package mysql

import (
	"fmt"
	"github.com/xaionaro/reform"
)

type mysql struct{}

func (mysql) String() string {
	return "mysql"
}

func (mysql) Placeholder(index int) string {
	return "?"
}

func (mysql) Placeholders(start, count int) []string {
	res := make([]string, count)
	for i := 0; i < count; i++ {
		res[i] = "?"
	}
	return res
}

func (mysql) QuoteIdentifier(identifier string) string {
	return "`" + identifier + "`"
}

func (mysql) LastInsertIdMethod() reform.LastInsertIdMethod {
	return reform.LastInsertId
}

func (mysql) SelectLimitMethod() reform.SelectLimitMethod {
	return reform.Limit
}

func (mysql) DefaultValuesMethod() reform.DefaultValuesMethod {
	return reform.EmptyLists
}

func (mysql) ColumnTypeForField(field reform.FieldInfo) string {
	switch field.Type {
	case "time.Time", "extime.Time":
		return "timestamp"
	case "int":
		return "integer"
	case "int64":
		return "bigint"
	case "string":
		if field.SQLSize > 0 && field.SQLSize < 256 {
			return fmt.Sprintf("varchar(%d)", field.SQLSize)
		}
		return "text"
	default:
		return "text"
	}
}

func (mysql) ColumnDefinitionForField(field reform.FieldInfo) string {
	canBeNull := false
	fieldType := field.Type
	if fieldType[0:1] == "*" {
		canBeNull = true
		fieldType = fieldType[1:]
	}

	columnType := Dialect.ColumnTypeForField(field)

	definition := field.Column + " " + columnType // TODO: escape everything

	if field.IsPK {
		definition += " PRIMARY KEY"
		if fieldType == "int" {
			definition += " AUTO_INCREMENT"
		}
	}

	if field.IsUnique {
			definition += " UNIQUE"
	}

	if !canBeNull {
		definition += " NOT NULL"
	}
	if field.HasIndex {
			definition += ", INDEX (`"+field.Column+"`)"
	}

	return definition
}

func (mysql) ColumnDefinitionPostQueryForField(structInfo reform.StructInfo, field reform.FieldInfo) string {
	return ""
}

// Dialect implements reform.Dialect for MySQL.
var Dialect mysql

// check interface
var _ reform.Dialect = Dialect
<commit_msg>A bugfix for d610ac7fa9ec4293e8c81ed98aee48d61acbd9a8<commit_after>// Package mysql implements reform.Dialect for MySQL.
package mysql

import (
	"fmt"
	"github.com/xaionaro/reform"
)

type mysql struct{}

func (mysql) String() string {
	return "mysql"
}

func (mysql) Placeholder(index int) string {
	return "?"
}

func (mysql) Placeholders(start, count int) []string {
	res := make([]string, count)
	for i := 0; i < count; i++ {
		res[i] = "?"
	}
	return res
}

func (mysql) QuoteIdentifier(identifier string) string {
	return "`" + identifier + "`"
}

func (mysql) LastInsertIdMethod() reform.LastInsertIdMethod {
	return reform.LastInsertId
}

func (mysql) SelectLimitMethod() reform.SelectLimitMethod {
	return reform.Limit
}

func (mysql) DefaultValuesMethod() reform.DefaultValuesMethod {
	return reform.EmptyLists
}

func (mysql) ColumnTypeForField(field reform.FieldInfo) string {
	if len(field.Type) == 0 {
		return "text"
	}
	if field.Type[:1] == "*" {
		field.Type = field.Type[1:]
	}
	switch field.Type {
	case "time.Time", "extime.Time":
		return "timestamp"
	case "int":
		return "integer"
	case "int64":
		return "bigint"
	case "string":
		if field.SQLSize > 0 && field.SQLSize < 256 {
			return fmt.Sprintf("varchar(%d)", field.SQLSize)
		}
		return "text"
	default:
		return "text"
	}
}

func (mysql) ColumnDefinitionForField(field reform.FieldInfo) string {
	canBeNull := false
	fieldType := field.Type
	if fieldType[0:1] == "*" {
		canBeNull = true
		fieldType = fieldType[1:]
	}

	columnType := Dialect.ColumnTypeForField(field)

	definition := field.Column + " " + columnType // TODO: escape everything

	if field.IsPK {
		definition += " PRIMARY KEY"
		if fieldType == "int" {
			definition += " AUTO_INCREMENT"
		}
	}

	if field.IsUnique {
			definition += " UNIQUE"
	}

	if !canBeNull {
		definition += " NOT NULL"
	}
	if field.HasIndex {
			definition += ", INDEX (`"+field.Column+"`)"
	}

	return definition
}

func (mysql) ColumnDefinitionPostQueryForField(structInfo reform.StructInfo, field reform.FieldInfo) string {
	return ""
}

// Dialect implements reform.Dialect for MySQL.
var Dialect mysql

// check interface
var _ reform.Dialect = Dialect
<|endoftext|>