<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/datasource/configdrive"
	"github.com/coreos/coreos-cloudinit/datasource/file"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/cloudsigma"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/digitalocean"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
	"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
	"github.com/coreos/coreos-cloudinit/datasource/url"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/pkg"
	"github.com/coreos/coreos-cloudinit/system"
)

const (
	version               = "0.9.6+git"
	datasourceInterval    = 100 * time.Millisecond
	datasourceMaxInterval = 30 * time.Second
	datasourceTimeout     = 5 * time.Minute
)

var (
	printVersion  bool
	ignoreFailure bool
	sources       struct {
		file                        string
		configDrive                 string
		metadataService             bool
		ec2MetadataService          string
		cloudSigmaMetadataService   bool
		digitalOceanMetadataService string
		url                         string
		procCmdLine                 bool
	}
	convertNetconf string
	workspace      string
	sshKeyName     string
)

func init() {
	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")
	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
	flag.StringVar(&sources.file, "from-file", "", "Read user-data from provided file")
	flag.StringVar(&sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
	flag.BoolVar(&sources.metadataService, "from-metadata-service", false, "[DEPRECATED - Use -from-ec2-metadata] Download data from metadata service")
	flag.StringVar(&sources.ec2MetadataService, "from-ec2-metadata", "", "Download EC2 data from the provided url")
	flag.BoolVar(&sources.cloudSigmaMetadataService, "from-cloudsigma-metadata", false, "Download data from CloudSigma server context")
	flag.StringVar(&sources.digitalOceanMetadataService, "from-digitalocean-metadata", "", "Download DigitalOcean data from the provided url")
	flag.StringVar(&sources.url, "from-url", "", "Download user-data from provided url")
	flag.BoolVar(&sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag))
	flag.StringVar(&convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files")
	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")
}

func main() {
	failure := false

	flag.Parse()

	if printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	switch convertNetconf {
	case "":
	case "debian":
	case "digitalocean":
	default:
		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian, digitalocean'\n", convertNetconf)
		os.Exit(2)
	}

	dss := getDatasources()
	if len(dss) == 0 {
		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-cloudsigma-metadata, --from-url or --from-proc-cmdline")
		os.Exit(2)
	}

	ds := selectDatasource(dss)
	if ds == nil {
		fmt.Println("No datasources available in time")
		os.Exit(1)
	}

	fmt.Printf("Fetching user-data from datasource of type %q\n", ds.Type())
	userdataBytes, err := ds.FetchUserdata()
	if err != nil {
		fmt.Printf("Failed fetching user-data from datasource: %v\nContinuing...\n", err)
		failure = true
	}

	fmt.Printf("Fetching meta-data from datasource of type %q\n", ds.Type())
	metadataBytes, err := ds.FetchMetadata()
	if err != nil {
		fmt.Printf("Failed fetching meta-data from datasource: %v\n", err)
		os.Exit(1)
	}

	// Extract IPv4 addresses from metadata if possible
	var subs map[string]string
	if len(metadataBytes) > 0 {
		subs, err = initialize.ExtractIPsFromMetadata(metadataBytes)
		if err != nil {
			fmt.Printf("Failed extracting IPs from meta-data: %v\n", err)
			os.Exit(1)
		}
	}

	// Apply environment to user-data
	env := initialize.NewEnvironment("/", ds.ConfigRoot(), workspace, convertNetconf, sshKeyName, subs)
	userdata := env.Apply(string(userdataBytes))

	var ccm, ccu *initialize.CloudConfig
	var script *system.Script
	if ccm, err = initialize.ParseMetaData(string(metadataBytes)); err != nil {
		fmt.Printf("Failed to parse meta-data: %v\n", err)
		os.Exit(1)
	}

	if ccm != nil {
		fmt.Printf("Fetching network config from datasource of type %q\n", ds.Type())
		netconfBytes, err := ds.FetchNetworkConfig(ccm.NetworkConfigPath)
		if err != nil {
			fmt.Printf("Failed fetching network config from datasource: %v\n", err)
			os.Exit(1)
		}
		ccm.NetworkConfig = string(netconfBytes)
	}

	if ud, err := initialize.ParseUserData(userdata); err != nil {
		fmt.Printf("Failed to parse user-data: %v\nContinuing...\n", err)
		failure = true
	} else {
		switch t := ud.(type) {
		case *initialize.CloudConfig:
			ccu = t
		case system.Script:
			script = &t
		}
	}

	var cc *initialize.CloudConfig
	if ccm != nil && ccu != nil {
		fmt.Println("Merging cloud-config from meta-data and user-data")
		merged := mergeCloudConfig(*ccm, *ccu)
		cc = &merged
	} else if ccm != nil && ccu == nil {
		fmt.Println("Processing cloud-config from meta-data")
		cc = ccm
	} else if ccm == nil && ccu != nil {
		fmt.Println("Processing cloud-config from user-data")
		cc = ccu
	} else {
		fmt.Println("No cloud-config data to handle.")
	}

	if cc != nil {
		if err = initialize.Apply(*cc, env); err != nil {
			fmt.Printf("Failed to apply cloud-config: %v\n", err)
			os.Exit(1)
		}
	}

	if script != nil {
		if err = runScript(*script, env); err != nil {
			fmt.Printf("Failed to run script: %v\n", err)
			os.Exit(1)
		}
	}

	if failure && !ignoreFailure {
		os.Exit(1)
	}
}

// mergeCloudConfig merges certain options from mdcc (a CloudConfig derived from
// meta-data) onto udcc (a CloudConfig derived from user-data), if they are
// not already set on udcc (i.e. user-data always takes precedence)
// NB: This needs to be kept in sync with ParseMetadata so that it tracks all
// elements of a CloudConfig which that function can populate.
func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudConfig) {
	if mdcc.Hostname != "" {
		if udcc.Hostname != "" {
			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", udcc.Hostname, mdcc.Hostname)
		} else {
			udcc.Hostname = mdcc.Hostname
		}

	}
	for _, key := range mdcc.SSHAuthorizedKeys {
		udcc.SSHAuthorizedKeys = append(udcc.SSHAuthorizedKeys, key)
	}
	if mdcc.NetworkConfigPath != "" {
		if udcc.NetworkConfigPath != "" {
			fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s\n", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
		} else {
			udcc.NetworkConfigPath = mdcc.NetworkConfigPath
		}
	}
	if mdcc.NetworkConfig != "" {
		if udcc.NetworkConfig != "" {
			fmt.Printf("Warning: user-data NetworkConfig %s overrides metadata NetworkConfig %s\n", udcc.NetworkConfig, mdcc.NetworkConfig)
		} else {
			udcc.NetworkConfig = mdcc.NetworkConfig
		}
	}
	return udcc
}

// getDatasources creates a slice of possible Datasources for cloudinit based
// on the different source command-line flags.
func getDatasources() []datasource.Datasource {
	dss := make([]datasource.Datasource, 0, 5)
	if sources.file != "" {
		dss = append(dss, file.NewDatasource(sources.file))
	}
	if sources.url != "" {
		dss = append(dss, url.NewDatasource(sources.url))
	}
	if sources.configDrive != "" {
		dss = append(dss, configdrive.NewDatasource(sources.configDrive))
	}
	if sources.metadataService {
		dss = append(dss, ec2.NewDatasource(ec2.DefaultAddress))
	}
	if sources.ec2MetadataService != "" {
		dss = append(dss, ec2.NewDatasource(sources.ec2MetadataService))
	}
	if sources.cloudSigmaMetadataService {
		dss = append(dss, cloudsigma.NewServerContextService())
	}
	if sources.digitalOceanMetadataService != "" {
		dss = append(dss, digitalocean.NewDatasource(sources.digitalOceanMetadataService))
	}
	if sources.procCmdLine {
		dss = append(dss, proc_cmdline.NewDatasource())
	}
	return dss
}

// selectDatasource attempts to choose a valid Datasource to use based on its
// current availability. The first Datasource to report to be available is
// returned. Datasources will be retried if possible if they are not
// immediately available. If all Datasources are permanently unavailable or
// datasourceTimeout is reached before one becomes available, nil is returned.
func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
	ds := make(chan datasource.Datasource)
	stop := make(chan struct{})
	var wg sync.WaitGroup

	for _, s := range sources {
		wg.Add(1)
		go func(s datasource.Datasource) {
			defer wg.Done()

			duration := datasourceInterval
			for {
				fmt.Printf("Checking availability of %q\n", s.Type())
				if s.IsAvailable() {
					ds <- s
					return
				} else if !s.AvailabilityChanges() {
					return
				}
				select {
				case <-stop:
					return
				case <-time.After(duration):
					duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
				}
			}
		}(s)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	var s datasource.Datasource
	select {
	case s = <-ds:
	case <-done:
	case <-time.After(datasourceTimeout):
	}

	close(stop)
	return s
}

// TODO(jonboulle): this should probably be refactored and moved into a different module
func runScript(script system.Script, env *initialize.Environment) error {
	err := initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		fmt.Printf("Failed preparing workspace: %v\n", err)
		return err
	}
	path, err := initialize.PersistScriptInWorkspace(script, env.Workspace())
	if err == nil {
		var name string
		name, err = system.ExecuteScript(path)
		initialize.PersistUnitNameInWorkspace(name, env.Workspace())
	}
	return err
}
<commit_msg>flags: add oem flag<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/datasource/configdrive"
	"github.com/coreos/coreos-cloudinit/datasource/file"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/cloudsigma"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/digitalocean"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
	"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
	"github.com/coreos/coreos-cloudinit/datasource/url"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/pkg"
	"github.com/coreos/coreos-cloudinit/system"
)

const (
	version               = "0.9.6+git"
	datasourceInterval    = 100 * time.Millisecond
	datasourceMaxInterval = 30 * time.Second
	datasourceTimeout     = 5 * time.Minute
)

var (
	printVersion  bool
	ignoreFailure bool
	sources       struct {
		file                        string
		configDrive                 string
		metadataService             bool
		ec2MetadataService          string
		cloudSigmaMetadataService   bool
		digitalOceanMetadataService string
		url                         string
		procCmdLine                 bool
	}
	convertNetconf string
	workspace      string
	sshKeyName     string
	oem            string
)

func init() {
	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")
	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
	flag.StringVar(&sources.file, "from-file", "", "Read user-data from provided file")
	flag.StringVar(&sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
	flag.BoolVar(&sources.metadataService, "from-metadata-service", false, "[DEPRECATED - Use -from-ec2-metadata] Download data from metadata service")
	flag.StringVar(&sources.ec2MetadataService, "from-ec2-metadata", "", "Download EC2 data from the provided url")
	flag.BoolVar(&sources.cloudSigmaMetadataService, "from-cloudsigma-metadata", false, "Download data from CloudSigma server context")
	flag.StringVar(&sources.digitalOceanMetadataService, "from-digitalocean-metadata", "", "Download DigitalOcean data from the provided url")
	flag.StringVar(&sources.url, "from-url", "", "Download user-data from provided url")
	flag.BoolVar(&sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag))
	flag.StringVar(&oem, "oem", "", "Use the settings specific to the provided OEM")
	flag.StringVar(&convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files")
	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")
}

type oemConfig map[string]string

var (
	oemConfigs = map[string]oemConfig{
		"digitalocean": oemConfig{
			"from-digitalocean-metadata": "http://169.254.169.254/",
			"convert-netconf":            "digitalocean",
		},
		"ec2-compat": oemConfig{
			"from-ec2-metadata": "http://169.254.169.254/",
			"from-configdrive":  "/media/configdrive",
		},
		"rackspace-onmetal": oemConfig{
			"from-configdrive": "/media/configdrive",
			"convert-netconf":  "debian",
		},
	}
)

func main() {
	failure := false

	flag.Parse()

	if c, ok := oemConfigs[oem]; ok {
		for k, v := range c {
			flag.Set(k, v)
		}
	} else if flags.oem != "" {
		oems := make([]string, 0, len(oemConfigs))
		for k := range oemConfigs {
			oems = append(oems, k)
		}
		fmt.Printf("Invalid option to --oem: %q. Supported options: %q\n", flags.oem, oems)
		os.Exit(2)
	}

	if printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	switch convertNetconf {
	case "":
	case "debian":
	case "digitalocean":
	default:
		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian, digitalocean'\n", convertNetconf)
		os.Exit(2)
	}

	dss := getDatasources()
	if len(dss) == 0 {
		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-cloudsigma-metadata, --from-url or --from-proc-cmdline")
		os.Exit(2)
	}

	ds := selectDatasource(dss)
	if ds == nil {
		fmt.Println("No datasources available in time")
		os.Exit(1)
	}

	fmt.Printf("Fetching user-data from datasource of type %q\n", ds.Type())
	userdataBytes, err := ds.FetchUserdata()
	if err != nil {
		fmt.Printf("Failed fetching user-data from datasource: %v\nContinuing...\n", err)
		failure = true
	}

	fmt.Printf("Fetching meta-data from datasource of type %q\n", ds.Type())
	metadataBytes, err := ds.FetchMetadata()
	if err != nil {
		fmt.Printf("Failed fetching meta-data from datasource: %v\n", err)
		os.Exit(1)
	}

	// Extract IPv4 addresses from metadata if possible
	var subs map[string]string
	if len(metadataBytes) > 0 {
		subs, err = initialize.ExtractIPsFromMetadata(metadataBytes)
		if err != nil {
			fmt.Printf("Failed extracting IPs from meta-data: %v\n", err)
			os.Exit(1)
		}
	}

	// Apply environment to user-data
	env := initialize.NewEnvironment("/", ds.ConfigRoot(), workspace, convertNetconf, sshKeyName, subs)
	userdata := env.Apply(string(userdataBytes))

	var ccm, ccu *initialize.CloudConfig
	var script *system.Script
	if ccm, err = initialize.ParseMetaData(string(metadataBytes)); err != nil {
		fmt.Printf("Failed to parse meta-data: %v\n", err)
		os.Exit(1)
	}

	if ccm != nil {
		fmt.Printf("Fetching network config from datasource of type %q\n", ds.Type())
		netconfBytes, err := ds.FetchNetworkConfig(ccm.NetworkConfigPath)
		if err != nil {
			fmt.Printf("Failed fetching network config from datasource: %v\n", err)
			os.Exit(1)
		}
		ccm.NetworkConfig = string(netconfBytes)
	}

	if ud, err := initialize.ParseUserData(userdata); err != nil {
		fmt.Printf("Failed to parse user-data: %v\nContinuing...\n", err)
		failure = true
	} else {
		switch t := ud.(type) {
		case *initialize.CloudConfig:
			ccu = t
		case system.Script:
			script = &t
		}
	}

	var cc *initialize.CloudConfig
	if ccm != nil && ccu != nil {
		fmt.Println("Merging cloud-config from meta-data and user-data")
		merged := mergeCloudConfig(*ccm, *ccu)
		cc = &merged
	} else if ccm != nil && ccu == nil {
		fmt.Println("Processing cloud-config from meta-data")
		cc = ccm
	} else if ccm == nil && ccu != nil {
		fmt.Println("Processing cloud-config from user-data")
		cc = ccu
	} else {
		fmt.Println("No cloud-config data to handle.")
	}

	if cc != nil {
		if err = initialize.Apply(*cc, env); err != nil {
			fmt.Printf("Failed to apply cloud-config: %v\n", err)
			os.Exit(1)
		}
	}

	if script != nil {
		if err = runScript(*script, env); err != nil {
			fmt.Printf("Failed to run script: %v\n", err)
			os.Exit(1)
		}
	}

	if failure && !ignoreFailure {
		os.Exit(1)
	}
}

// mergeCloudConfig merges certain options from mdcc (a CloudConfig derived from
// meta-data) onto udcc (a CloudConfig derived from user-data), if they are
// not already set on udcc (i.e. user-data always takes precedence)
// NB: This needs to be kept in sync with ParseMetadata so that it tracks all
// elements of a CloudConfig which that function can populate.
func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudConfig) {
	if mdcc.Hostname != "" {
		if udcc.Hostname != "" {
			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", udcc.Hostname, mdcc.Hostname)
		} else {
			udcc.Hostname = mdcc.Hostname
		}

	}
	for _, key := range mdcc.SSHAuthorizedKeys {
		udcc.SSHAuthorizedKeys = append(udcc.SSHAuthorizedKeys, key)
	}
	if mdcc.NetworkConfigPath != "" {
		if udcc.NetworkConfigPath != "" {
			fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s\n", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
		} else {
			udcc.NetworkConfigPath = mdcc.NetworkConfigPath
		}
	}
	if mdcc.NetworkConfig != "" {
		if udcc.NetworkConfig != "" {
			fmt.Printf("Warning: user-data NetworkConfig %s overrides metadata NetworkConfig %s\n", udcc.NetworkConfig, mdcc.NetworkConfig)
		} else {
			udcc.NetworkConfig = mdcc.NetworkConfig
		}
	}
	return udcc
}

// getDatasources creates a slice of possible Datasources for cloudinit based
// on the different source command-line flags.
func getDatasources() []datasource.Datasource {
	dss := make([]datasource.Datasource, 0, 5)
	if sources.file != "" {
		dss = append(dss, file.NewDatasource(sources.file))
	}
	if sources.url != "" {
		dss = append(dss, url.NewDatasource(sources.url))
	}
	if sources.configDrive != "" {
		dss = append(dss, configdrive.NewDatasource(sources.configDrive))
	}
	if sources.metadataService {
		dss = append(dss, ec2.NewDatasource(ec2.DefaultAddress))
	}
	if sources.ec2MetadataService != "" {
		dss = append(dss, ec2.NewDatasource(sources.ec2MetadataService))
	}
	if sources.cloudSigmaMetadataService {
		dss = append(dss, cloudsigma.NewServerContextService())
	}
	if sources.digitalOceanMetadataService != "" {
		dss = append(dss, digitalocean.NewDatasource(sources.digitalOceanMetadataService))
	}
	if sources.procCmdLine {
		dss = append(dss, proc_cmdline.NewDatasource())
	}
	return dss
}

// selectDatasource attempts to choose a valid Datasource to use based on its
// current availability. The first Datasource to report to be available is
// returned. Datasources will be retried if possible if they are not
// immediately available. If all Datasources are permanently unavailable or
// datasourceTimeout is reached before one becomes available, nil is returned.
func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
	ds := make(chan datasource.Datasource)
	stop := make(chan struct{})
	var wg sync.WaitGroup

	for _, s := range sources {
		wg.Add(1)
		go func(s datasource.Datasource) {
			defer wg.Done()

			duration := datasourceInterval
			for {
				fmt.Printf("Checking availability of %q\n", s.Type())
				if s.IsAvailable() {
					ds <- s
					return
				} else if !s.AvailabilityChanges() {
					return
				}
				select {
				case <-stop:
					return
				case <-time.After(duration):
					duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
				}
			}
		}(s)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	var s datasource.Datasource
	select {
	case s = <-ds:
	case <-done:
	case <-time.After(datasourceTimeout):
	}

	close(stop)
	return s
}

// TODO(jonboulle): this should probably be refactored and moved into a different module
func runScript(script system.Script, env *initialize.Environment) error {
	err := initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		fmt.Printf("Failed preparing workspace: %v\n", err)
		return err
	}
	path, err := initialize.PersistScriptInWorkspace(script, env.Workspace())
	if err == nil {
		var name string
		name, err = system.ExecuteScript(path)
		initialize.PersistUnitNameInWorkspace(name, env.Workspace())
	}
	return err
}
<|endoftext|>


<commit_before>package cmd

import (
	"errors"
	"io"
	"os"

	"github.com/spf13/cobra"
	//"github.com/spf13/viper"
)

func NewCmdConfig(out io.Writer) *cobra.Command {
	cmd := &cobra.Command{
		Use:	"config name [value]",
		Short:	"Get or set global options",
		RunE: func(cmd *cobra.Command, args []string) error {
			return dispatchConfig(args, os.Stdout)
		},
	}
	return cmd
}

func dispatchConfig(args []string, out io.Writer) error {
	if len(args) == 0 {
		return errors.New("Not enough arguments")
	} else if len(args) == 1 {
		return errors.New("Any operation other than setting a value is not implemented yet")
	} else if len(args) == 2 {
		return setConfigValue(args[0], args[1], out)
	} else {
		return errors.New("Too many arguments")
	}
}

func setConfigValue(key, value string, out io.Writer) error {
	return nil
}
<commit_msg>Failing test.<commit_after>package cmd

import (
	"errors"
	"io"
	"os"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

func NewCmdConfig(out io.Writer) *cobra.Command {
	cmd := &cobra.Command{
		Use:	"config name [value]",
		Short:	"Get or set global options",
		RunE: func(cmd *cobra.Command, args []string) error {
			return dispatchConfig(args, os.Stdout)
		},
	}
	return cmd
}

func dispatchConfig(args []string, out io.Writer) error {
	if len(args) == 0 {
		return errors.New("Not enough arguments")
	} else if len(args) == 1 {
		return errors.New("Any operation other than setting a value is not implemented yet")
	} else if len(args) == 2 {
		return setConfigValue(args[0], args[1], out)
	} else {
		return errors.New("Too many arguments")
	}
}

func setConfigValue(key, value string, out io.Writer) error {
	return viper.SetString(key, value)
}
<|endoftext|>


<commit_before>// Copyright 2018 The nvim-go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package config

import (
	"sync"

	envpkg "github.com/lestrrat-go/config/env"
	"go.uber.org/zap/zapcore"
)

// e is the global env variable
var e env

// env represents a environment variabels with lestrrat-go/config/env struct tag.
type env struct {
	GCPProjectID string `envconfig:"GCP_PROJECT_ID"`
	Debug        bool   `envconfig:"DEBUG"`
	LogLevel     string `envconfig:"LOG_LEVEL"`
}

func (e env) MarshalLogObject(enc zapcore.ObjectEncoder) error {
	enc.AddString("gcp_project_id", e.GCPProjectID)
	enc.AddBool("debug", e.Debug)
	enc.AddString("log_level", e.LogLevel)

	return nil
}

// envOnce for run Process once.
var envOnce sync.Once

// Process populates the specified struct based on environment variables.
func Process() (e env, err error) {
	envOnce.Do(func() {
		err = envpkg.NewDecoder(envpkg.System).Prefix("NVIM_GO").Decode(&e)
	})
	return e, err
}

func GCPProjectID() string {
	_, _ = Process()
	return e.GCPProjectID
}

func IsDebug() bool {
	_, _ = Process()
	return e.Debug
}
<commit_msg>pkg/config: change lestrrat-go/config/env to kelseyhightower/envconfig<commit_after>// Copyright 2018 The nvim-go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package config

import (
	"sync"

	"github.com/kelseyhightower/envconfig"
	"go.uber.org/zap/zapcore"
)

// env represents a environment variabels for nvim-go.
type env struct {
	GCPProjectID string `envconfig:"gcp_project_id"`
	Debug        bool   `envconfig:"debug"`
	LogLevel     string `envconfig:"log_level"`
}

func (e env) MarshalLogObject(enc zapcore.ObjectEncoder) error {
	enc.AddString("gcp_project_id", e.GCPProjectID)
	enc.AddBool("debug", e.Debug)
	enc.AddString("log_level", e.LogLevel)

	return nil
}

// e is the global env variable
var e env

// envOnce for run Process once.
var envOnce sync.Once

// Process populates the specified struct based on environment variables.
func Process() env {
	envOnce.Do(func() {
		envconfig.MustProcess("nvim_go", &e)
	})

	return e
}

func GCPProjectID() string {
	_ = Process()
	return e.GCPProjectID
}

func HasGCPProjectID() bool {
	_ = Process()
	return e.GCPProjectID != ""
}

func IsDebug() bool {
	_ = Process()
	return e.Debug
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"time"

	"github.com/golang/glog"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	informerv1 "k8s.io/client-go/informers/core/v1"
	"k8s.io/client-go/tools/cache"
)

// IngressStability denotes the stabilization status of all Ingresses in a sandbox.
type IngressStability string

var (
	// Stable indicates an Ingress is stable (i.e consistently serving 200's)
	Stable IngressStability = "Stable"
	// Unstable indicates an Ingress is unstable (i.e serving 404/502's).
	Unstable IngressStability = "Unstable"
)

const (
	configMapName  = "status-cm"
	cmPollInterval = 30 * time.Second
	flushInterval  = 30 * time.Second
	// ExitKey is the key used to indicate to the status manager
	// whether to gracefully finish the e2e test execution.
	// Value associated with it is a timestamp string.
	exitKey = "exit"
	// masterUpgradingKey is the key used to indicate to the status manager that
	// the k8s master is in the process of upgrading.
	// Value associated with it is a timestamp string.
	masterUpgradingKey = "master-upgrading"
	// masterUpgradedKey is the key used to indicate to the status manager that
	// the k8s master has successfully finished upgrading.
	// Value associated with it is a timestamp string.
	masterUpgradedKey = "master-upgraded"
)

// StatusManager manages the status of sandboxed Ingresses via a ConfigMap.
// It interacts with the an external framework test portion as follows:
// 1. StatusManager initializes and creates the ConfigMap status-cm. It listens
// on updates via informers.
// 2. e2e test calls StatusManager.putStatus with the Ingress name as key,
// and Unstable as the status
// 3. e2e test watches for when Ingress stabilizes, then uses StatusManager to
// update the Ingress's status to Stable
// 4. The external framework test reads from ConfigMap status-cm. When it detects that all
// Ingresses are stable (i.e., no value in the map is Unstable), it starts
// the MasterUpgrade.
// 5. When the k8s master finishes upgrading, the framework test writes the
// timestamp to the master-upgraded key in the ConfigMap
// 6. The external framework test writes the exit key in the ConfigMap to indicate that the e2e
// test can exit.
// 7. The StatusManager loop reads the exit key, then starts shutdown().
type StatusManager struct {
	cm              *v1.ConfigMap
	f               *Framework
	informerCh      chan struct{}
	informerRunning bool
}

func NewStatusManager(f *Framework) *StatusManager {
	return &StatusManager{
		cm: &v1.ConfigMap{
			ObjectMeta: metav1.ObjectMeta{
				Name: configMapName,
			},
		},
		f: f,
	}
}

func (sm *StatusManager) init() error {
	var err error
	sm.cm, err = sm.f.Clientset.Core().ConfigMaps("default").Create(sm.cm)
	if err != nil {
		return fmt.Errorf("error creating ConfigMap: %v", err)
	}

	go func() {
		for _ = range time.NewTicker(flushInterval).C {
			sm.flush()
		}
	}()

	sm.startInformer()
	return nil
}

func (sm *StatusManager) startInformer() {
	newIndexer := func() cache.Indexers {
		return cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}
	}

	sm.informerCh = make(chan struct{})
	cmInformer := informerv1.NewConfigMapInformer(sm.f.Clientset, "default", cmPollInterval, newIndexer())
	cmInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		UpdateFunc: func(old, cur interface{}) {
			curCm := cur.(*v1.ConfigMap)
			if len(curCm.Data[exitKey]) > 0 {
				glog.V(2).Infof("ConfigMap was updated with exit switch at %s", curCm.Data[exitKey])
				close(sm.informerCh)
				sm.f.shutdown(0)
			}
		},
	})

	glog.V(4).Info("Started ConfigMap informer")
	sm.informerRunning = true
	go cmInformer.Run(sm.informerCh)
}

func (sm *StatusManager) stopInformer() {
	glog.V(4).Info("Stopped ConfigMap informer")
	sm.informerRunning = false
	close(sm.informerCh)
}

func (sm *StatusManager) shutdown() {
	glog.V(2).Infof("Shutting down status manager.")
	glog.V(3).Infof("ConfigMap: %+v", sm.cm.Data)
	if err := sm.f.Clientset.Core().ConfigMaps("default").Delete(configMapName, &metav1.DeleteOptions{}); err != nil {
		glog.Errorf("Error deleting ConfigMap: %v", err)
	}
}

func (sm *StatusManager) putStatus(key string, status IngressStability) {
	sm.f.lock.Lock()
	defer sm.f.lock.Unlock()
	if sm.cm.Data == nil {
		sm.cm.Data = make(map[string]string)
	}
	sm.cm.Data[key] = string(status)
}

func (sm *StatusManager) masterUpgrading() bool {
	return len(sm.cm.Data[masterUpgradingKey]) > 0
}

func (sm *StatusManager) masterUpgraded() bool {
	if len(sm.cm.Data[masterUpgradedKey]) > 0 {
		glog.V(4).Infof("Master has successfully upgraded at %s", sm.cm.Data[masterUpgradedKey])
		return true
	}
	return false
}

func (sm *StatusManager) flush() {
	sm.f.lock.Lock()
	defer sm.f.lock.Unlock()

	// If master is in the process of upgrading, we exit early and turn off the
	// ConfigMap informer.
	if sm.masterUpgrading() && sm.informerRunning {
		sm.stopInformer()
		return
	}

	// Restart ConfigMap informer if it was previously shut down
	if sm.masterUpgraded() && !sm.informerRunning {
		sm.startInformer()
	}

	// Loop until we successfully update the config map
	for {
		updatedCm, err := sm.f.Clientset.Core().ConfigMaps("default").Get(configMapName, metav1.GetOptions{})
		if err != nil {
			glog.Warningf("Error getting ConfigMap: %v", err)
		}

		if updatedCm.Data == nil {
			updatedCm.Data = make(map[string]string)
		}

		// K8s considers its version of the ConfigMap to be latest, so we must get
		// the configmap from k8s first.
		// We give precedence to the master-upgraded and master-upgrading flags
		// set by the external test framework, but otherwise we prioritize
		// Ingress statuses set by StatusManager.
		for key, value := range sm.cm.Data {
			if key != masterUpgradedKey && key != masterUpgradingKey {
				updatedCm.Data[key] = value
			}
		}
		sm.cm = updatedCm
		sm.cm.Name = configMapName

		_, err = sm.f.Clientset.Core().ConfigMaps("default").Update(sm.cm)
		if err != nil {
			glog.Warningf("Error updating ConfigMap: %v", err)
		} else {
			// ConfigMap successfully updated
			break
		}
	}
	glog.V(3).Infof("Flushed statuses to ConfigMap")
	glog.V(3).Infof("ConfigMap: %+v", sm.cm)
}
<commit_msg>remove loop from flush(). start/stop informers depending on k8s master<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"time"

	"github.com/golang/glog"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	informerv1 "k8s.io/client-go/informers/core/v1"
	"k8s.io/client-go/tools/cache"
)

// IngressStability denotes the stabilization status of all Ingresses in a sandbox.
type IngressStability string

var (
	// Stable indicates an Ingress is stable (i.e consistently serving 200's)
	Stable IngressStability = "Stable"
	// Unstable indicates an Ingress is unstable (i.e serving 404/502's).
	Unstable IngressStability = "Unstable"
)

const (
	configMapName  = "status-cm"
	cmPollInterval = 30 * time.Second
	flushInterval  = 30 * time.Second
	// ExitKey is the key used to indicate to the status manager
	// whether to gracefully finish the e2e test execution.
	// Value associated with it is a timestamp string.
	exitKey = "exit"
	// masterUpgradingKey is the key used to indicate to the status manager that
	// the k8s master is in the process of upgrading.
	// Value associated with it is a timestamp string.
	masterUpgradingKey = "master-upgrading"
	// masterUpgradedKey is the key used to indicate to the status manager that
	// the k8s master has successfully finished upgrading.
	// Value associated with it is a timestamp string.
	masterUpgradedKey = "master-upgraded"
)

// StatusManager manages the status of sandboxed Ingresses via a ConfigMap.
// It interacts with the an external framework test portion as follows:
// 1. StatusManager initializes and creates the ConfigMap status-cm. It listens
// on updates via informers.
// 2. e2e test calls StatusManager.putStatus with the Ingress name as key,
// and Unstable as the status
// 3. e2e test watches for when Ingress stabilizes, then uses StatusManager to
// update the Ingress's status to Stable
// 4. The external framework test reads from ConfigMap status-cm. When it detects that all
// Ingresses are stable (i.e., no value in the map is Unstable), it starts
// the MasterUpgrade.
// 5. When the k8s master finishes upgrading, the framework test writes the
// timestamp to the master-upgraded key in the ConfigMap
// 6. The external framework test writes the exit key in the ConfigMap to indicate that the e2e
// test can exit.
// 7. The StatusManager loop reads the exit key, then starts shutdown().
type StatusManager struct {
	cm              *v1.ConfigMap
	f               *Framework
	informerCh      chan struct{}
	informerRunning bool
}

func NewStatusManager(f *Framework) *StatusManager {
	return &StatusManager{
		cm: &v1.ConfigMap{
			ObjectMeta: metav1.ObjectMeta{
				Name: configMapName,
			},
		},
		f: f,
	}
}

func (sm *StatusManager) init() error {
	var err error
	sm.cm, err = sm.f.Clientset.Core().ConfigMaps("default").Create(sm.cm)
	if err != nil {
		return fmt.Errorf("error creating ConfigMap: %v", err)
	}

	go func() {
		for _ = range time.NewTicker(flushInterval).C {
			sm.flush()
		}
	}()

	sm.startInformer()
	return nil
}

func (sm *StatusManager) startInformer() {
	newIndexer := func() cache.Indexers {
		return cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}
	}

	sm.informerCh = make(chan struct{})
	cmInformer := informerv1.NewConfigMapInformer(sm.f.Clientset, "default", cmPollInterval, newIndexer())
	cmInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		UpdateFunc: func(old, cur interface{}) {
			curCm := cur.(*v1.ConfigMap)
			upgradeTs := curCm.Data[masterUpgradingKey]
			// We need to pick up the master-upgrading flag here, because otherwise
			// if the k8s master is in the process of upgrading we can't access
			// the ConfigMap.
			if len(upgradeTs) > 0 {
				glog.V(2).Infof("Master upgrade began at %v", upgradeTs)
				sm.setMasterUpgrading(upgradeTs)
			}

			if len(curCm.Data[exitKey]) > 0 {
				glog.V(2).Infof("ConfigMap was updated with exit switch at %s", curCm.Data[exitKey])
				close(sm.informerCh)
				sm.f.shutdown(0)
			}
		},
	})

	glog.V(2).Info("Started ConfigMap informer")
	sm.informerRunning = true
	go cmInformer.Run(sm.informerCh)
}

func (sm *StatusManager) stopInformer() {
	glog.V(2).Info("Stopped ConfigMap informer")
	sm.informerRunning = false
	close(sm.informerCh)
}

func (sm *StatusManager) shutdown() {
	glog.V(2).Infof("Shutting down status manager.")
	glog.V(3).Infof("ConfigMap: %+v", sm.cm.Data)
	if err := sm.f.Clientset.Core().ConfigMaps("default").Delete(configMapName, &metav1.DeleteOptions{}); err != nil {
		glog.Errorf("Error deleting ConfigMap: %v", err)
	}
}

func (sm *StatusManager) putStatus(key string, status IngressStability) {
	sm.f.lock.Lock()
	defer sm.f.lock.Unlock()
	if sm.cm.Data == nil {
		sm.cm.Data = make(map[string]string)
	}
	sm.cm.Data[key] = string(status)
}

func (sm *StatusManager) setMasterUpgrading(ts string) {
	sm.f.lock.Lock()
	defer sm.f.lock.Unlock()
	if sm.cm.Data == nil {
		sm.cm.Data = make(map[string]string)
	}

	sm.cm.Data[masterUpgradingKey] = ts
}

func (sm *StatusManager) masterUpgrading() bool {
	return len(sm.cm.Data[masterUpgradingKey]) > 0
}

func (sm *StatusManager) masterUpgraded() bool {
	return len(sm.cm.Data[masterUpgradedKey]) > 0
}

func (sm *StatusManager) flush() {
	sm.f.lock.Lock()
	defer sm.f.lock.Unlock()

	glog.V(3).Infof("Attempting to flush %v", sm.cm.Data)

	// If master is in the process of upgrading, we exit early and turn off the
	// ConfigMap informer.
	if sm.masterUpgrading() && sm.informerRunning {
		sm.stopInformer()
	}

	// Restart ConfigMap informer if it was previously shut down
	if !sm.informerRunning && sm.masterUpgraded() {
		glog.V(2).Infof("Master has successfully upgraded at %s", sm.cm.Data[masterUpgradedKey])
		sm.startInformer()
	}

	// K8s considers its version of the ConfigMap to be latest, so we must get
	// the configmap from k8s first.
	updatedCm, err := sm.f.Clientset.Core().ConfigMaps("default").Get(configMapName, metav1.GetOptions{})
	// The k8s API returns an empty ConfigMap upon error - we return early in
	// order to not overwrite our ConfigMap data.
	if err != nil {
		// if the k8s master is upgrading, we suppress the error message because
		// the error is expected.
		if !sm.masterUpgrading() {
			glog.Warningf("Error getting ConfigMap: %v", err)
		}

		glog.Warningf("Error getting ConfigMap: %v", err)
		return
	}

	if updatedCm.Data == nil {
		updatedCm.Data = make(map[string]string)
	}

	// We give precedence to the master-upgraded and master-upgrading flags
	// set by the external test framework, but otherwise we prioritize
	// Ingress statuses set by StatusManager.
	for key, value := range sm.cm.Data {
		if key != masterUpgradedKey && key != masterUpgradingKey {
			updatedCm.Data[key] = value
		}
	}
	sm.cm = updatedCm
	sm.cm.Name = configMapName

	_, err = sm.f.Clientset.Core().ConfigMaps("default").Update(sm.cm)
	if err != nil {
		glog.Warningf("Error updating ConfigMap: %v", err)
	} else {
		glog.V(3).Infof("Flushed statuses %v to ConfigMap", sm.cm.Data)
	}
}
<|endoftext|>


<commit_before>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package limes

import (
	"fmt"
	"net/http"
	"sync"

	policy "github.com/databus23/goslo.policy"
	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/identity/v3/tokens"
	"github.com/sapcc/limes/pkg/util"
)

//AuthParameters contains credentials for authenticating with Keystone (i.e.
//everything that's needed to set up a gophercloud.ProviderClient instance).
type AuthParameters struct {
	AuthURL           string      `yaml:"auth_url"`
	UserName          string      `yaml:"user_name"`
	UserDomainName    string      `yaml:"user_domain_name"`
	ProjectName       string      `yaml:"project_name"`
	ProjectDomainName string      `yaml:"project_domain_name"`
	Password          string      `yaml:"password"`
	RegionName        string      `yaml:"region_name"`
	tokenRenewalMutex *sync.Mutex `yaml:"-"`
	//ProviderClient is only valid after calling Connect().
	ProviderClient *gophercloud.ProviderClient `yaml:"-"`
}

//CanReauth implements the
//gophercloud/openstack/identity/v3/tokens.AuthOptionsBuilder interface.
func (auth AuthParameters) CanReauth() bool {
	return true
}

//ToTokenV3CreateMap implements the
//gophercloud/openstack/identity/v3/tokens.AuthOptionsBuilder interface.
func (auth AuthParameters) ToTokenV3CreateMap(scope map[string]interface{}) (map[string]interface{}, error) {
	gophercloudAuthOpts := gophercloud.AuthOptions{
		Username:    auth.UserName,
		Password:    auth.Password,
		DomainName:  auth.UserDomainName,
		AllowReauth: true,
	}
	return gophercloudAuthOpts.ToTokenV3CreateMap(scope)
}

//ToTokenV3ScopeMap implements the
//gophercloud/openstack/identity/v3/tokens.AuthOptionsBuilder interface.
func (auth AuthParameters) ToTokenV3ScopeMap() (map[string]interface{}, error) {
	return map[string]interface{}{
		"project": map[string]interface{}{
			"name":   auth.ProjectName,
			"domain": map[string]interface{}{"name": auth.ProjectDomainName},
		},
	}, nil
}

//Connect creates the gophercloud.ProviderClient instance for these credentials.
func (auth *AuthParameters) Connect() error {
	if auth.tokenRenewalMutex == nil {
		auth.tokenRenewalMutex = &sync.Mutex{}
	}

	if auth.ProviderClient != nil {
		//already done
		return nil
	}

	var err error
	auth.ProviderClient, err = openstack.NewClient(auth.AuthURL)
	if err != nil {
		return fmt.Errorf("cannot initialize OpenStack client: %v", err)
	}
	//use http.DefaultClient, esp. to pick up LIMES_INSECURE flag
	auth.ProviderClient.HTTPClient = *http.DefaultClient

	err = auth.refreshToken()
	if err != nil {
		return fmt.Errorf("cannot fetch initial Keystone token: %v", err)
	}

	return nil
}

//refreshToken fetches a new Keystone token for this cluster. It is also used
//to fetch the initial token on startup.
func (auth *AuthParameters) refreshToken() error {
	//NOTE: This function is very similar to v3auth() in
	//gophercloud/openstack/client.go, but with a few differences:
	//
	//1. thread-safe token renewal
	//2. proper support for cross-domain scoping

	auth.tokenRenewalMutex.Lock()
	defer auth.tokenRenewalMutex.Unlock()
	util.LogDebug("renewing Keystone token...")

	auth.ProviderClient.TokenID = ""

	keystone := &gophercloud.ServiceClient{
		ProviderClient: auth.ProviderClient,
		Endpoint:       auth.AuthURL,
	}
	result := tokens.Create(keystone, auth)
	token, err := result.ExtractToken()
	if err != nil {
		return fmt.Errorf("cannot read token: %v", err)
	}
	catalog, err := result.ExtractServiceCatalog()
	if err != nil {
		return fmt.Errorf("cannot read service catalog: %v", err)
	}

	auth.ProviderClient.TokenID = token.ID
	auth.ProviderClient.ReauthFunc = auth.refreshToken //TODO: exponential backoff necessary or already provided by gophercloud?
	auth.ProviderClient.EndpointLocator = func(opts gophercloud.EndpointOpts) (string, error) {
		return openstack.V3EndpointURL(catalog, opts)
	}

	return nil
}

//ValidateToken validates the given Keystone token and returns a policy context for
//checking authorization.
func (auth *AuthParameters) ValidateToken(token string) (policy.Context, error) {
	//special case for unit tests
	if auth.AuthURL == "" {
		return policy.Context{}, nil
	}

	client, err := openstack.NewIdentityV3(auth.ProviderClient,
		gophercloud.EndpointOpts{Availability: gophercloud.AvailabilityPublic},
	)
	if err != nil {
		return policy.Context{}, err
	}

	response := tokens.Get(client, token)
	if response.Err != nil {
		//this includes 4xx responses, so after this point, we can be sure that the token is valid
		return policy.Context{}, response.Err
	}

	//use a custom token struct instead of tokens.Token which is way incomplete
	var tokenData keystoneToken
	err = response.ExtractInto(&tokenData)
	if err != nil {
		return policy.Context{}, err
	}
	return tokenData.ToContext(), nil
}

type keystoneToken struct {
	DomainScope  keystoneTokenThing         `json:"domain"`
	ProjectScope keystoneTokenThingInDomain `json:"project"`
	Roles        []keystoneTokenThing       `json:"roles"`
	User         keystoneTokenThingInDomain `json:"user"`
}

type keystoneTokenThing struct {
	ID   string `json:"id"`
	Name string `json:"name"`
}

type keystoneTokenThingInDomain struct {
	keystoneTokenThing
	Domain keystoneTokenThing `json:"domain"`
}

func (t *keystoneToken) ToContext() policy.Context {
	c := policy.Context{
		Roles: make([]string, 0, len(t.Roles)),
		Auth: map[string]string{
			"user_id":             t.User.ID,
			"user_name":           t.User.Name,
			"user_domain_id":      t.User.Domain.ID,
			"user_domain_name":    t.User.Domain.Name,
			"domain_id":           t.DomainScope.ID,
			"domain_name":         t.DomainScope.Name,
			"project_id":          t.ProjectScope.ID,
			"project_name":        t.ProjectScope.Name,
			"project_domain_id":   t.ProjectScope.Domain.ID,
			"project_domain_name": t.ProjectScope.Domain.Name,
			"tenant_id":           t.ProjectScope.ID,
			"tenant_name":         t.ProjectScope.Name,
			"tenant_domain_id":    t.ProjectScope.Domain.ID,
			"tenant_domain_name":  t.ProjectScope.Domain.Name,
		},
		Request: nil,
		Logger:  util.LogDebug,
	}
	for key, value := range c.Auth {
		if value == "" {
			delete(c.Auth, key)
		}
	}
	for _, role := range t.Roles {
		c.Roles = append(c.Roles, role.Name)
	}
	if c.Request == nil {
		c.Request = map[string]string{}
	}

	return c
}
<commit_msg>fix last remaining bug with multi-region clusters<commit_after>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package limes

import (
	"fmt"
	"net/http"
	"sync"

	policy "github.com/databus23/goslo.policy"
	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/identity/v3/tokens"
	"github.com/sapcc/limes/pkg/util"
)

//AuthParameters contains credentials for authenticating with Keystone (i.e.
//everything that's needed to set up a gophercloud.ProviderClient instance).
type AuthParameters struct {
	AuthURL           string      `yaml:"auth_url"`
	UserName          string      `yaml:"user_name"`
	UserDomainName    string      `yaml:"user_domain_name"`
	ProjectName       string      `yaml:"project_name"`
	ProjectDomainName string      `yaml:"project_domain_name"`
	Password          string      `yaml:"password"`
	RegionName        string      `yaml:"region_name"`
	tokenRenewalMutex *sync.Mutex `yaml:"-"`
	//ProviderClient is only valid after calling Connect().
	ProviderClient *gophercloud.ProviderClient `yaml:"-"`
}

//CanReauth implements the
//gophercloud/openstack/identity/v3/tokens.AuthOptionsBuilder interface.
func (auth AuthParameters) CanReauth() bool {
	return true
}

//ToTokenV3CreateMap implements the
//gophercloud/openstack/identity/v3/tokens.AuthOptionsBuilder interface.
func (auth AuthParameters) ToTokenV3CreateMap(scope map[string]interface{}) (map[string]interface{}, error) {
	gophercloudAuthOpts := gophercloud.AuthOptions{
		Username:    auth.UserName,
		Password:    auth.Password,
		DomainName:  auth.UserDomainName,
		AllowReauth: true,
	}
	return gophercloudAuthOpts.ToTokenV3CreateMap(scope)
}

//ToTokenV3ScopeMap implements the
//gophercloud/openstack/identity/v3/tokens.AuthOptionsBuilder interface.
func (auth AuthParameters) ToTokenV3ScopeMap() (map[string]interface{}, error) {
	return map[string]interface{}{
		"project": map[string]interface{}{
			"name":   auth.ProjectName,
			"domain": map[string]interface{}{"name": auth.ProjectDomainName},
		},
	}, nil
}

//Connect creates the gophercloud.ProviderClient instance for these credentials.
func (auth *AuthParameters) Connect() error {
	if auth.tokenRenewalMutex == nil {
		auth.tokenRenewalMutex = &sync.Mutex{}
	}

	if auth.ProviderClient != nil {
		//already done
		return nil
	}

	var err error
	auth.ProviderClient, err = openstack.NewClient(auth.AuthURL)
	if err != nil {
		return fmt.Errorf("cannot initialize OpenStack client: %v", err)
	}
	//use http.DefaultClient, esp. to pick up LIMES_INSECURE flag
	auth.ProviderClient.HTTPClient = *http.DefaultClient

	err = auth.refreshToken()
	if err != nil {
		return fmt.Errorf("cannot fetch initial Keystone token: %v", err)
	}

	return nil
}

//refreshToken fetches a new Keystone token for this cluster. It is also used
//to fetch the initial token on startup.
func (auth *AuthParameters) refreshToken() error {
	//NOTE: This function is very similar to v3auth() in
	//gophercloud/openstack/client.go, but with a few differences:
	//
	//1. thread-safe token renewal
	//2. proper support for cross-domain scoping

	auth.tokenRenewalMutex.Lock()
	defer auth.tokenRenewalMutex.Unlock()
	util.LogDebug("renewing Keystone token...")

	auth.ProviderClient.TokenID = ""

	keystone := &gophercloud.ServiceClient{
		ProviderClient: auth.ProviderClient,
		Endpoint:       auth.AuthURL,
	}
	result := tokens.Create(keystone, auth)
	token, err := result.ExtractToken()
	if err != nil {
		return fmt.Errorf("cannot read token: %v", err)
	}
	catalog, err := result.ExtractServiceCatalog()
	if err != nil {
		return fmt.Errorf("cannot read service catalog: %v", err)
	}

	auth.ProviderClient.TokenID = token.ID
	auth.ProviderClient.ReauthFunc = auth.refreshToken //TODO: exponential backoff necessary or already provided by gophercloud?
	auth.ProviderClient.EndpointLocator = func(opts gophercloud.EndpointOpts) (string, error) {
		opts.Region = auth.RegionName
		return openstack.V3EndpointURL(catalog, opts)
	}

	return nil
}

//ValidateToken validates the given Keystone token and returns a policy context for
//checking authorization.
func (auth *AuthParameters) ValidateToken(token string) (policy.Context, error) {
	//special case for unit tests
	if auth.AuthURL == "" {
		return policy.Context{}, nil
	}

	client, err := openstack.NewIdentityV3(auth.ProviderClient,
		gophercloud.EndpointOpts{Availability: gophercloud.AvailabilityPublic},
	)
	if err != nil {
		return policy.Context{}, err
	}

	response := tokens.Get(client, token)
	if response.Err != nil {
		//this includes 4xx responses, so after this point, we can be sure that the token is valid
		return policy.Context{}, response.Err
	}

	//use a custom token struct instead of tokens.Token which is way incomplete
	var tokenData keystoneToken
	err = response.ExtractInto(&tokenData)
	if err != nil {
		return policy.Context{}, err
	}
	return tokenData.ToContext(), nil
}

type keystoneToken struct {
	DomainScope  keystoneTokenThing         `json:"domain"`
	ProjectScope keystoneTokenThingInDomain `json:"project"`
	Roles        []keystoneTokenThing       `json:"roles"`
	User         keystoneTokenThingInDomain `json:"user"`
}

type keystoneTokenThing struct {
	ID   string `json:"id"`
	Name string `json:"name"`
}

type keystoneTokenThingInDomain struct {
	keystoneTokenThing
	Domain keystoneTokenThing `json:"domain"`
}

func (t *keystoneToken) ToContext() policy.Context {
	c := policy.Context{
		Roles: make([]string, 0, len(t.Roles)),
		Auth: map[string]string{
			"user_id":             t.User.ID,
			"user_name":           t.User.Name,
			"user_domain_id":      t.User.Domain.ID,
			"user_domain_name":    t.User.Domain.Name,
			"domain_id":           t.DomainScope.ID,
			"domain_name":         t.DomainScope.Name,
			"project_id":          t.ProjectScope.ID,
			"project_name":        t.ProjectScope.Name,
			"project_domain_id":   t.ProjectScope.Domain.ID,
			"project_domain_name": t.ProjectScope.Domain.Name,
			"tenant_id":           t.ProjectScope.ID,
			"tenant_name":         t.ProjectScope.Name,
			"tenant_domain_id":    t.ProjectScope.Domain.ID,
			"tenant_domain_name":  t.ProjectScope.Domain.Name,
		},
		Request: nil,
		Logger:  util.LogDebug,
	}
	for key, value := range c.Auth {
		if value == "" {
			delete(c.Auth, key)
		}
	}
	for _, role := range t.Roles {
		c.Roles = append(c.Roles, role.Name)
	}
	if c.Request == nil {
		c.Request = map[string]string{}
	}

	return c
}
<|endoftext|>


<commit_before>// Copyright 2020 the Exposure Notifications Verification Server authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sms

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// TwilioMessagingServiceSidPrefix is the prefix for a 34 character messaging service identifier
const TwilioMessagingServiceSidPrefix = "MG"

var _ Provider = (*Twilio)(nil)

// Twilio sends messages via the Twilio API.
type Twilio struct {
	client *http.Client
	from   string
}

// NewTwilio creates a new Twilio SMS sender with the given auth.
func NewTwilio(ctx context.Context, accountSid, authToken, from string) (Provider, error) {
	client := &http.Client{
		Timeout:   5 * time.Second,
		Transport: &twilioAuthRoundTripper{accountSid, authToken},
	}

	return &Twilio{
		client: client,
		from:   from,
	}, nil
}

// SendSMS sends a message using the Twilio API.
func (p *Twilio) SendSMS(ctx context.Context, to, message string) error {
	params := url.Values{}
	params.Set("To", to)
	if strings.HasPrefix(p.from, TwilioMessagingServiceSidPrefix) {
		params.Set("MessagingServiceSid", p.from)
	} else {
		params.Set("From", p.from)
	}

	params.Set("Body", message)
	body := strings.NewReader(params.Encode())

	req, err := http.NewRequestWithContext(ctx, http.MethodPost, "/Messages.json", body)
	if err != nil {
		return fmt.Errorf("failed to build request: %w", err)
	}

	resp, err := p.client.Do(req)
	if err != nil {
		return fmt.Errorf("failed to make request: %w", err)
	}
	defer resp.Body.Close()

	respBody, err := io.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("failed to read response body: %w", err)
	}

	if code := resp.StatusCode; code < http.StatusOK || code >= http.StatusMultipleChoices {
		var terr TwilioError
		if err := json.Unmarshal(respBody, &terr); err != nil {
			return fmt.Errorf("twilio error %d: %s", code, respBody)
		}
		return &terr
	}

	return nil
}

// twilioAuthRoundTripper is an http.RoundTripper that updates the
// authentication and headers to match Twilio's API.
type twilioAuthRoundTripper struct {
	accountSid, authToken string
}

// RoundTrip implements http.RoundTripper.
func (rt *twilioAuthRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
	r.URL = &url.URL{
		Scheme: "https",
		Host:   "api.twilio.com",
		Path:   "/2010-04-01/Accounts/" + rt.accountSid + "/" + strings.Trim(r.URL.Path, "/"),
	}

	r.SetBasicAuth(rt.accountSid, rt.authToken)
	r.Header.Set("Accept", "application/json")
	r.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	return http.DefaultTransport.RoundTrip(r)
}

// TwilioError represents an error returned from the Twilio API.
type TwilioError struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
}

func (e *TwilioError) Error() string {
	return e.Message
}

// IsSMSQueueFull returns if the given error is Twilio's message queue full error.
func IsSMSQueueFull(err error) bool {
	return IsTwilioCode(err, 21611) // https://www.twilio.com/docs/api/errors/21611
}

// IsTwilioCode returns if the given error matches a Twilio error code.
func IsTwilioCode(err error, code int) bool {
	var tErr *TwilioError
	if errors.As(err, &tErr) {
		return tErr.Code == code
	}
	return false
}
<commit_msg>Retry Twilio communication on transient network errors (#2206)<commit_after>// Copyright 2020 the Exposure Notifications Verification Server authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sms

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/sethvargo/go-retry"
)

// TwilioMessagingServiceSidPrefix is the prefix for a 34 character messaging service identifier
const TwilioMessagingServiceSidPrefix = "MG"

var _ Provider = (*Twilio)(nil)

// Twilio sends messages via the Twilio API.
type Twilio struct {
	client *http.Client
	from   string
}

// NewTwilio creates a new Twilio SMS sender with the given auth.
func NewTwilio(ctx context.Context, accountSid, authToken, from string) (Provider, error) {
	client := &http.Client{
		Timeout:   5 * time.Second,
		Transport: &twilioAuthRoundTripper{accountSid, authToken},
	}

	return &Twilio{
		client: client,
		from:   from,
	}, nil
}

// SendSMS sends a message using the Twilio API.
func (p *Twilio) SendSMS(ctx context.Context, to, message string) error {
	b, err := retry.NewFibonacci(250 * time.Millisecond)
	if err != nil {
		return fmt.Errorf("failed to create backoff: %w", err)
	}
	b = retry.WithMaxRetries(4, b)

	return retry.Do(ctx, b, func(ctx context.Context) error {
		params := url.Values{}
		params.Set("To", to)
		if strings.HasPrefix(p.from, TwilioMessagingServiceSidPrefix) {
			params.Set("MessagingServiceSid", p.from)
		} else {
			params.Set("From", p.from)
		}

		params.Set("Body", message)
		body := strings.NewReader(params.Encode())

		req, err := http.NewRequestWithContext(ctx, http.MethodPost, "/Messages.json", body)
		if err != nil {
			return fmt.Errorf("failed to build request: %w", err)
		}
		req.Close = true

		resp, err := p.client.Do(req)
		if err != nil {
			return retry.RetryableError(fmt.Errorf("failed to make request: %w", err))
		}
		defer resp.Body.Close()

		respBody, err := io.ReadAll(resp.Body)
		if err != nil {
			return fmt.Errorf("failed to read response body: %w", err)
		}

		if code := resp.StatusCode; code < http.StatusOK || code >= http.StatusMultipleChoices {
			var terr TwilioError
			if err := json.Unmarshal(respBody, &terr); err != nil {
				return fmt.Errorf("twilio error %d: %s", code, respBody)
			}
			return &terr
		}

		return nil
	})
}

// twilioAuthRoundTripper is an http.RoundTripper that updates the
// authentication and headers to match Twilio's API.
type twilioAuthRoundTripper struct {
	accountSid, authToken string
}

// RoundTrip implements http.RoundTripper.
func (rt *twilioAuthRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
	r.URL = &url.URL{
		Scheme: "https",
		Host:   "api.twilio.com",
		Path:   "/2010-04-01/Accounts/" + rt.accountSid + "/" + strings.Trim(r.URL.Path, "/"),
	}

	r.SetBasicAuth(rt.accountSid, rt.authToken)
	r.Header.Set("Accept", "application/json")
	r.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	return http.DefaultTransport.RoundTrip(r)
}

// TwilioError represents an error returned from the Twilio API.
type TwilioError struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
}

func (e *TwilioError) Error() string {
	return e.Message
}

// IsSMSQueueFull returns if the given error is Twilio's message queue full error.
func IsSMSQueueFull(err error) bool {
	return IsTwilioCode(err, 21611) // https://www.twilio.com/docs/api/errors/21611
}

// IsTwilioCode returns if the given error matches a Twilio error code.
func IsTwilioCode(err error, code int) bool {
	var tErr *TwilioError
	if errors.As(err, &tErr) {
		return tErr.Code == code
	}
	return false
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package utils

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
)

// MapToJson converts the specified map object to indented JSON.
// It panics in case if the map connot be converted.
func MapToJson(m map[string]interface{}) string {
	bs, err := json.MarshalIndent(m, "", "  ")
	if err != nil {
		log.Panicf("error marshalling json: %v", err)
	}
	return string(bs)
}

// MapToJson converts the specified map object to unindented JSON.
// It panics in case if the map connot be converted.
func MapToJsonUnindented(m map[string]interface{}) string {
	bs, err := json.Marshal(m)
	if err != nil {
		log.Panicf("error marshalling json: %v", err)
	}
	return string(bs)
}

func ReadJson(filename string, v interface{}) error {
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return fmt.Errorf("error reading json file %q: %v", filename, err)
	}

	if err := json.Unmarshal(content, v); err != nil {
		return fmt.Errorf("failed to parse json file %q: %v", filename, err)
	}

	return nil
}

func WriteJson(filename string, v interface{}, perm os.FileMode) error {
	content, err := json.Marshal(v)
	if err != nil {
		return fmt.Errorf("couldn't marshal the data to JSON for %q: %v", filename, err)
	}
	if err := ioutil.WriteFile(filename, content, perm); err != nil {
		return fmt.Errorf("error writing JSON data file %q: %V", filename, err)
	}
	return nil
}
<commit_msg>Fix typo in json error format string<commit_after>/*
Copyright 2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package utils

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
)

// MapToJson converts the specified map object to indented JSON.
// It panics in case if the map connot be converted.
func MapToJson(m map[string]interface{}) string {
	bs, err := json.MarshalIndent(m, "", "  ")
	if err != nil {
		log.Panicf("error marshalling json: %v", err)
	}
	return string(bs)
}

// MapToJson converts the specified map object to unindented JSON.
// It panics in case if the map connot be converted.
func MapToJsonUnindented(m map[string]interface{}) string {
	bs, err := json.Marshal(m)
	if err != nil {
		log.Panicf("error marshalling json: %v", err)
	}
	return string(bs)
}

func ReadJson(filename string, v interface{}) error {
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return fmt.Errorf("error reading json file %q: %v", filename, err)
	}

	if err := json.Unmarshal(content, v); err != nil {
		return fmt.Errorf("failed to parse json file %q: %v", filename, err)
	}

	return nil
}

func WriteJson(filename string, v interface{}, perm os.FileMode) error {
	content, err := json.Marshal(v)
	if err != nil {
		return fmt.Errorf("couldn't marshal the data to JSON for %q: %v", filename, err)
	}
	if err := ioutil.WriteFile(filename, content, perm); err != nil {
		return fmt.Errorf("error writing JSON data file %q: %v", filename, err)
	}
	return nil
}
<|endoftext|>


<commit_before>package dory

import (
	"bytes"
	"encoding/binary"
	"errors"

	"github.com/dgryski/go-farm"
)

var (
	ErrNoSpace = errors.New("insufficent space left")
)

type PackedTable struct {
	buf  []byte
	off  int
	keys map[uint32]int32

	added   int
	deleted int
}

func NewPackedTable(buf []byte) *PackedTable {
	if len(buf) > 1<<30 {
		panic("len(buf) > 1GiB")
	}

	return &PackedTable{
		buf:  buf,
		keys: make(map[uint32]int32),
	}
}

func (t *PackedTable) hashKey(key []byte) uint32 {
	return farm.Hash32(key)
}

func (t *PackedTable) readSize(off int) (int, int) {
	// TODO: Use "unsafe"?
	keySize := binary.LittleEndian.Uint32(t.buf[off:])
	valSize := binary.LittleEndian.Uint32(t.buf[off+4:])
	return int(keySize), int(valSize)
}

func (t *PackedTable) writeSize(key, val int) int {
	off := t.off
	t.off += 8
	binary.LittleEndian.PutUint32(t.buf[off:], uint32(key))
	binary.LittleEndian.PutUint32(t.buf[off+4:], uint32(val))
	return off
}

func (t *PackedTable) hashEntry(key []byte) uint32 {
	hash := t.hashKey(key)
	for ; ; hash++ {
		off, ok := t.keys[hash]
		if !ok {
			break
		} else if off < 0 {
			continue
		}

		keySize, _ := t.readSize(int(off))
		if keySize == len(key) && bytes.Compare(key, t.buf[int(off)+8:int(off)+8+len(key)]) == 0 {
			break
		}
	}
	return hash
}

func (t *PackedTable) findKey(key []byte) int {
	hash := t.hashKey(key)
	for ; ; hash++ {
		off, ok := t.keys[hash]
		if !ok {
			break
		} else if off < 0 {
			continue
		}

		keySize, _ := t.readSize(int(off))
		if keySize == len(key) && bytes.Compare(key, t.buf[int(off)+8:int(off)+8+len(key)]) == 0 {
			return int(off)
		}
	}
	return -1
}

func (t *PackedTable) EntrySize(key, val []byte) int {
	return len(key) + len(val) + 8
}

func (t *PackedTable) FreeSpace() int {
	return len(t.buf) - t.off
}

func (t *PackedTable) NumEntries() int {
	return t.added - t.deleted
}

func (t *PackedTable) NumDeleted() int {
	return t.deleted
}

func (t *PackedTable) Has(key []byte) bool {
	if len(key) == 0 {
		panic("zero-sized key")
	}

	return t.findKey(key) >= 0
}

func (t *PackedTable) Get(key, buf []byte) []byte {
	if len(key) == 0 {
		panic("zero-sized key")
	}

	off := t.findKey(key)
	if off < 0 {
		return nil
	}

	keySize, valSize := t.readSize(off)
	buf = append(buf, t.buf[off+8+keySize:off+8+keySize+valSize]...)
	return buf
}

func (t *PackedTable) Put(key, val []byte) error {
	if len(key) == 0 {
		panic("zero-sized key")
	}

	size := t.EntrySize(key, val)
	if size > t.FreeSpace() {
		return ErrNoSpace
	}

	hash := t.hashEntry(key)
	if off, ok := t.keys[hash]; ok && off >= 0 {
		t.deleted++
	}

	off := t.writeSize(len(key), len(val))
	n := copy(t.buf[t.off:], key)
	t.off += n
	if n != len(key) {
		panic("n != len(key)")
	}
	n = copy(t.buf[t.off:], val)
	t.off += n
	if n != len(val) {
		panic("n != len(val)")
	}
	t.keys[hash] = int32(off)
	t.added++
	return nil
}

func (t *PackedTable) Delete(key []byte) bool {
	if len(key) == 0 {
		panic("zero-sized key")
	}

	hash := t.hashEntry(key)
	off, ok := t.keys[hash]
	if ok && off >= 0 {
		_, ok := t.keys[hash+1]
		if ok {
			t.keys[hash] = -1
		} else {
			delete(t.keys, hash)
		}
		t.deleted++
		return true
	}
	return false
}
<commit_msg>Make code a bit easier to read.<commit_after>package dory

import (
	"bytes"
	"encoding/binary"
	"errors"

	"github.com/dgryski/go-farm"
)

var (
	ErrNoSpace = errors.New("insufficent space left")
)

type PackedTable struct {
	buf  []byte
	off  int
	keys map[uint32]int32

	added   int
	deleted int
}

func NewPackedTable(buf []byte) *PackedTable {
	if len(buf) > 1<<30 {
		panic("len(buf) > 1GiB")
	}

	return &PackedTable{
		buf:  buf,
		keys: make(map[uint32]int32),
	}
}

func (t *PackedTable) hashKey(key []byte) uint32 {
	return farm.Hash32(key)
}

func (t *PackedTable) readSize(off int) (int, int) {
	// TODO: Use "unsafe"?
	keySize := binary.LittleEndian.Uint32(t.buf[off:])
	valSize := binary.LittleEndian.Uint32(t.buf[off+4:])
	return int(keySize), int(valSize)
}

func (t *PackedTable) writeSize(key, val int) int {
	off := t.off
	t.off += 8
	binary.LittleEndian.PutUint32(t.buf[off:], uint32(key))
	binary.LittleEndian.PutUint32(t.buf[off+4:], uint32(val))
	return off
}

func (t *PackedTable) hashEntry(key []byte) uint32 {
	hash := t.hashKey(key)
	for ; ; hash++ {
		off, ok := t.keys[hash]
		if !ok {
			break
		} else if off < 0 {
			continue
		}

		keySize, _ := t.readSize(int(off))
		keyOff := int(off) + 8
		if keySize == len(key) && bytes.Compare(key, t.buf[keyOff:keyOff+len(key)]) == 0 {
			break
		}
	}
	return hash
}

func (t *PackedTable) findKey(key []byte) int {
	hash := t.hashKey(key)
	for ; ; hash++ {
		off, ok := t.keys[hash]
		if !ok {
			break
		} else if off < 0 {
			continue
		}

		keySize, _ := t.readSize(int(off))
		keyOff := int(off) + 8
		if keySize == len(key) && bytes.Compare(key, t.buf[keyOff:keyOff+len(key)]) == 0 {
			return int(off)
		}
	}
	return -1
}

func (t *PackedTable) EntrySize(key, val []byte) int {
	return len(key) + len(val) + 8
}

func (t *PackedTable) FreeSpace() int {
	return len(t.buf) - t.off
}

func (t *PackedTable) NumEntries() int {
	return t.added - t.deleted
}

func (t *PackedTable) NumDeleted() int {
	return t.deleted
}

func (t *PackedTable) Has(key []byte) bool {
	if len(key) == 0 {
		panic("zero-sized key")
	}

	return t.findKey(key) >= 0
}

func (t *PackedTable) Get(key, buf []byte) []byte {
	if len(key) == 0 {
		panic("zero-sized key")
	}

	off := t.findKey(key)
	if off < 0 {
		return nil
	}

	keySize, valSize := t.readSize(off)
	buf = append(buf, t.buf[off+8+keySize:off+8+keySize+valSize]...)
	return buf
}

func (t *PackedTable) Put(key, val []byte) error {
	if len(key) == 0 {
		panic("zero-sized key")
	}

	size := t.EntrySize(key, val)
	if size > t.FreeSpace() {
		return ErrNoSpace
	}

	hash := t.hashEntry(key)
	if off, ok := t.keys[hash]; ok && off >= 0 {
		t.deleted++
	}

	off := t.writeSize(len(key), len(val))
	n := copy(t.buf[t.off:], key)
	t.off += n
	if n != len(key) {
		panic("n != len(key)")
	}
	n = copy(t.buf[t.off:], val)
	t.off += n
	if n != len(val) {
		panic("n != len(val)")
	}
	t.keys[hash] = int32(off)
	t.added++
	return nil
}

func (t *PackedTable) Delete(key []byte) bool {
	if len(key) == 0 {
		panic("zero-sized key")
	}

	hash := t.hashEntry(key)
	off, ok := t.keys[hash]
	if ok && off >= 0 {
		_, ok := t.keys[hash+1]
		if ok {
			t.keys[hash] = -1
		} else {
			delete(t.keys, hash)
		}
		t.deleted++
		return true
	}
	return false
}
<|endoftext|>


<commit_before>// +build go1.12

package gmf

/*
#cgo pkg-config: libavcodec

#include "libavcodec/avcodec.h"

void shift_data(AVPacket *pkt, int offset) {
    pkt->data += offset;
    pkt->size -= offset;

    return;
}

*/
import "C"

import (
	"fmt"
	"unsafe"
)

type Packet struct {
	avPacket C.struct_AVPacket
}

func NewPacket() *Packet {
	p := &Packet{}

	C.av_init_packet(&p.avPacket)

	p.avPacket.data = nil
	p.avPacket.size = 0

	return p
}

// Init same to NewPacket and av_init_packet
//   Initialize optional fields of a packet with default values.
//   Note, this does not touch the data and size members, which have to be
//   initialized separately.
func Init() *Packet {
	p := &Packet{}

	C.av_init_packet(&p.avPacket)

	p.avPacket.data = nil
	p.avPacket.size = 0

	return p
}

func (p *Packet) Pts() int64 {
	return int64(p.avPacket.pts)
}

func (p *Packet) SetPts(pts int64) *Packet {
	p.avPacket.pts = C.int64_t(pts)
	return p
}

func (p *Packet) Dts() int64 {
	return int64(p.avPacket.dts)
}

func (p *Packet) SetDts(val int64) *Packet {
	p.avPacket.dts = C.int64_t(val)
	return p
}

func (p *Packet) Flags() int {
	return int(p.avPacket.flags)
}

func (p *Packet) Duration() int64 {
	return int64(p.avPacket.duration)
}

func (p *Packet) SetDuration(duration int64) *Packet {
	p.avPacket.duration = C.int64_t(duration)
	return p
}

func (p *Packet) StreamIndex() int {
	return int(p.avPacket.stream_index)
}

func (p *Packet) Size() int {
	return int(p.avPacket.size)
}

func (p *Packet) Pos() int64 {
	return int64(p.avPacket.pos)
}

func (p *Packet) Data() []byte {
	return C.GoBytes(unsafe.Pointer(p.avPacket.data), C.int(p.avPacket.size))
}

// SetData [NOT SUGGESTED] should free data later
//   p := gmf.NewPacket()
//   defer p.Free()
//   p.SetData([]byte{0x00, 0x00, 0x00, 0x01, 0x67})
//   defer p.FreeData()
func (p *Packet) SetData(data []byte) *Packet {
	p.avPacket.size = C.int(len(data))
	p.avPacket.data = (*C.uint8_t)(C.CBytes(data))
	return p
}

// FreeData free data when use SetData
//   p := gmf.NewPacket()
//   defer p.Free()
//   p.SetData([]byte{0x00, 0x00, 0x00, 0x01, 0x67})
//   defer p.FreeData()
func (p *Packet) FreeData() *Packet {
	if p.avPacket.data != nil {
		C.free(p.avPacket.data)
		p.avPacket.data = nil
		p.avPacket.size = 0
	}
	return p
}

func (p *Packet) Clone() *Packet {
	np := NewPacket()

	C.av_packet_ref(&np.avPacket, &p.avPacket)

	return np
}

func (p *Packet) Dump() {
	fmt.Printf("idx: %d\npts: %d\ndts: %d\nsize: %d\nduration:%d\npos:%d\ndata: % x\n", p.StreamIndex(), p.avPacket.pts, p.avPacket.dts, p.avPacket.size, p.avPacket.duration, p.avPacket.pos, C.GoBytes(unsafe.Pointer(p.avPacket.data), 128))
	fmt.Println("------------------------------")

}

func (p *Packet) SetStreamIndex(val int) *Packet {
	p.avPacket.stream_index = C.int(val)
	return p
}

func (p *Packet) Free() {
	C.av_packet_unref(&p.avPacket)
}

func (p *Packet) Time(timebase AVRational) int {
	return int(float64(timebase.AVR().Num) / float64(timebase.AVR().Den) * float64(p.Pts()))
}
<commit_msg>fixed C.free type error<commit_after>// +build go1.12

package gmf

/*
#cgo pkg-config: libavcodec

#include "libavcodec/avcodec.h"

void shift_data(AVPacket *pkt, int offset) {
    pkt->data += offset;
    pkt->size -= offset;

    return;
}

*/
import "C"

import (
	"fmt"
	"unsafe"
)

type Packet struct {
	avPacket C.struct_AVPacket
}

func NewPacket() *Packet {
	p := &Packet{}

	C.av_init_packet(&p.avPacket)

	p.avPacket.data = nil
	p.avPacket.size = 0

	return p
}

// Init same to NewPacket and av_init_packet
//   Initialize optional fields of a packet with default values.
//   Note, this does not touch the data and size members, which have to be
//   initialized separately.
func Init() *Packet {
	p := &Packet{}

	C.av_init_packet(&p.avPacket)

	p.avPacket.data = nil
	p.avPacket.size = 0

	return p
}

func (p *Packet) Pts() int64 {
	return int64(p.avPacket.pts)
}

func (p *Packet) SetPts(pts int64) *Packet {
	p.avPacket.pts = C.int64_t(pts)
	return p
}

func (p *Packet) Dts() int64 {
	return int64(p.avPacket.dts)
}

func (p *Packet) SetDts(val int64) *Packet {
	p.avPacket.dts = C.int64_t(val)
	return p
}

func (p *Packet) Flags() int {
	return int(p.avPacket.flags)
}

func (p *Packet) Duration() int64 {
	return int64(p.avPacket.duration)
}

func (p *Packet) SetDuration(duration int64) *Packet {
	p.avPacket.duration = C.int64_t(duration)
	return p
}

func (p *Packet) StreamIndex() int {
	return int(p.avPacket.stream_index)
}

func (p *Packet) Size() int {
	return int(p.avPacket.size)
}

func (p *Packet) Pos() int64 {
	return int64(p.avPacket.pos)
}

func (p *Packet) Data() []byte {
	return C.GoBytes(unsafe.Pointer(p.avPacket.data), C.int(p.avPacket.size))
}

// SetData [NOT SUGGESTED] should free data later
//   p := gmf.NewPacket()
//   defer p.Free()
//   p.SetData([]byte{0x00, 0x00, 0x00, 0x01, 0x67})
//   defer p.FreeData()
func (p *Packet) SetData(data []byte) *Packet {
	p.avPacket.size = C.int(len(data))
	p.avPacket.data = (*C.uint8_t)(C.CBytes(data))
	return p
}

// FreeData free data when use SetData
//   p := gmf.NewPacket()
//   defer p.Free()
//   p.SetData([]byte{0x00, 0x00, 0x00, 0x01, 0x67})
//   defer p.FreeData()
func (p *Packet) FreeData() *Packet {
	if p.avPacket.data != nil {
		C.free(unsafe.Pointer(p.avPacket.data))
		p.avPacket.data = nil
		p.avPacket.size = 0
	}
	return p
}

func (p *Packet) Clone() *Packet {
	np := NewPacket()

	C.av_packet_ref(&np.avPacket, &p.avPacket)

	return np
}

func (p *Packet) Dump() {
	fmt.Printf("idx: %d\npts: %d\ndts: %d\nsize: %d\nduration:%d\npos:%d\ndata: % x\n", p.StreamIndex(), p.avPacket.pts, p.avPacket.dts, p.avPacket.size, p.avPacket.duration, p.avPacket.pos, C.GoBytes(unsafe.Pointer(p.avPacket.data), 128))
	fmt.Println("------------------------------")

}

func (p *Packet) SetStreamIndex(val int) *Packet {
	p.avPacket.stream_index = C.int(val)
	return p
}

func (p *Packet) Free() {
	C.av_packet_unref(&p.avPacket)
}

func (p *Packet) Time(timebase AVRational) int {
	return int(float64(timebase.AVR().Num) / float64(timebase.AVR().Den) * float64(p.Pts()))
}
<|endoftext|>


<commit_before>package crawl

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"path"

	"github.com/blang/semver"

	"github.com/google/go-github/github"

	"github.com/xogeny/impact/dirinfo"
	"github.com/xogeny/impact/parsing"
)

func getUses(client *github.Client, user string, reponame string,
	mopath string, opts *github.RepositoryContentGetOptions) (map[string]semver.Version, error) {
	blank := map[string]semver.Version{}

	// Get data for parent directory (we do this because the API sometimes chokes
	// on downloading large Modelica files, but if we do it this way, we can always
	// get the file).
	dir := path.Dir(mopath)
	fname := path.Base(mopath)

	contents := ""

	log.Printf("Getting contents of %s", dir)
	_, dcon, _, err := client.Repositories.GetContents(user, reponame, dir, opts)
	for _, con := range dcon {
		if *con.Name == fname {
			if con.DownloadURL == nil || *con.DownloadURL == "" {
				return blank, fmt.Errorf("No download link present for %s", mopath)
			}
			resp, err := http.Get(*con.DownloadURL)
			if err != nil {
				return blank, fmt.Errorf("Error downloading %s from %s: %v", mopath,
					*con.DownloadURL, err)
			}
			raw, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return blank, fmt.Errorf("Error reading response: %v", err)
			}
			contents = string(raw)
		}
	}

	uses, err := parsing.ParseUses(contents)
	if err != nil {
		return blank,
			fmt.Errorf("Error while parsing contents of %s in github repository %s: %v",
				mopath, reponame, err)
	}

	return uses, nil
}

func Exists(client *github.Client, user string, reponame string,
	path string, opts *github.RepositoryContentGetOptions) (file bool, dir bool) {
	f, d, _, err := client.Repositories.GetContents(user, reponame, path, opts)
	if err != nil {
		return false, false
	}
	if f != nil {
		return true, false
	}
	if d != nil {
		return false, true
	}
	return false, false
}

// The goal of this function is to construct a DirectoryInfo object.  It does this by first
// reading whatever directory information it can find in impact.json.  Then it tries to
// "infer" the rest using some heuristics (to lower the burden on library developers)
func ExtractInfo(client *github.Client, user string, altname string, repo github.Repository,
	sha string, versionString string, verbose bool, logger *log.Logger) dirinfo.DirectoryInfo {

	// Extract the name of the respository
	repostr := *repo.Name

	// Extract information about the owner of this repository (note: this is a URI)
	owner_uri := user
	if repo.Owner.HTMLURL != nil {
		owner_uri = *repo.Owner.HTMLURL
	}

	// Get the owner's email address, if provided
	email := ""
	if repo.Owner.Email != nil {
		email = *repo.Owner.Email
	}

	// Specify which version of the repository we are interested in
	opts := &github.RepositoryContentGetOptions{
		Ref: sha,
	}

	// Create a "blank" directory info as default
	di := dirinfo.MakeDirectoryInfo()

	// Parse any impact.json file the is present
	dcon, _, _, err := client.Repositories.GetContents(user, repostr, "impact.json", opts)

	// If impact.json exists, parse it and use that as our baseline
	if dcon != nil && err == nil {
		pdi, perr := dirinfo.Parse(*dcon.Content)
		if perr == nil {
			log.Printf("Parsed impact.json file in %s: %v", repostr, pdi)
			di = pdi
		} else {
			log.Printf("Unable to parse impact.json in %s: %v", repostr, perr)
		}
	}

	// NOW, use heuristics to infer missing information

	// Is owner information missing.  If so, use either the owner whose repositories
	// are being scanned or the owner of this particular repository (see logic above
	// regarding 'owner')
	if di.OwnerURI == "" {
		di.OwnerURI = owner_uri
	}

	// Ditto with email
	if di.Email == "" {
		di.Email = email
	}

	// This is local function where only the path is an argument, all the other
	// arguments come from the local environment.
	lookupPath := func(path string) (bool, bool) {
		return Exists(client, user, repostr, path, opts)
	}

	// Are any libraries mentioned?  If not, we need to figure out what the structure
	// is here.  There are two patterns.  Either a file named <RepoName>.mo or a
	// directory named <RepoName>.  If neither of these conventions is followed, the
	// library developers needs to add an explicit impact.json
	if len(di.Libraries) == 0 {
		// Provide default values for path and isfile
		path := ""
		isfile := true

		// This is a simplified version of the version string that strips build
		// information away.  This is necessary because some library developers
		// don't include build information in the names of files and directories
		// that include an explicit version number.  So we check for this as well.
		sver := parsing.SimpleVersion(versionString)

		// Flag to indicate whether we have found a library
		found := false

		// A list of Modelica files to look for that could be libraries
		// (stored as files)
		filenames := []string{
			repostr + ".mo",
			repostr + " " + versionString + ".mo",
			altname + ".mo",
			altname + " " + versionString + ".mo",
		}

		// A list of directories to look for that could be Modelica libraries
		// (stored as directories)
		dirnames := []string{
			repostr,
			repostr + " " + versionString,
			altname,
			altname + " " + versionString,
		}

		// If the simple version string isn't the same as the full version string,
		// add a few more variations.
		if sver != versionString {
			filenames = append(filenames, repostr+" "+sver+".mo")
			filenames = append(filenames, altname+" "+sver+".mo")
			dirnames = append(dirnames, repostr+" "+sver)
			dirnames = append(dirnames, altname+" "+sver)
		}

		// First, check for the library stored as a file...
		for _, fpath := range filenames {
			// Look for that file in the repository
			fexists, _ := lookupPath(fpath)
			if verbose {
				log.Printf("Looking for file %s in %s:%s: %v", fpath, repostr,
					versionString, fexists)
			}
			// If no library has been found yet and this file exists, record
			// its information and set the found flag
			if !found && fexists {
				path = fpath
				found = true
				if verbose {
					log.Printf("  Found")
				}
			}
		}

		// Next, check for the library stored as a directory...
		for _, dpath := range dirnames {
			// Look for a directory in the repository with the matching name
			// TODO: Look for package.mo inside!
			_, dexists := lookupPath(dpath)
			if verbose {
				log.Printf("Looking for directory %s in %s:%s: %v", dpath, repostr,
					versionString, dexists)
			}
			// If we haven't yet found a match and this directory exists, then
			// record the information and set found flag
			if !found && dexists {
				path = dpath
				isfile = false
				found = true
				if verbose {
					log.Printf("  Found")
				}
			}
		}

		// Finally, if we still haven't found a library, chec to see if the
		// repository itself is a library
		if !found {
			// Check if repository IS a package
			fexists, _ := lookupPath("package.mo")
			if verbose {
				log.Printf("Check if directory %s:%s is a package: %v", repostr, versionString,
					fexists)
			}
			// If so, record information about it
			if fexists {
				path = ""
				isfile = false
				found = true
				if verbose {
					log.Printf("  Found")
				}
			}
		}

		// If we found a library, create an instance of dirinfo.LocalLibrary
		// N.B. - These heuristics only look for one library in the repository.
		// If a library developer wants to store more than one, they need to
		// state this explicitly in the impact.json file and not rely on us
		// inferring it somehow.
		if found {
			di.Libraries = []*dirinfo.LocalLibrary{
				&dirinfo.LocalLibrary{
					Name:         repostr,
					Path:         path,
					IsFile:       isfile,
					Dependencies: []dirinfo.Dependency{},
				},
			}
		} else {
			if verbose {
				log.Printf("Nothing found in %s/%s for among %v or %v", repostr, versionString,
					filenames, dirnames)
			}
		}
	}

	// Now, let's loop over all the libraries we are aware of...
	for _, lib := range di.Libraries {
		// Determine path to top-level package in repository
		path := lib.Path
		if !lib.IsFile {
			path = fmt.Sprintf("%s/package.mo", lib.Path)
		}

		// Extract information about any libraries this library uses
		uses, err := getUses(client, user, repostr, path, opts)
		if err != nil {
			log.Printf("Error extracting uses annotation: %v", err)
			continue
		}

		for libname, ver := range uses {
			lib.Dependencies = append(lib.Dependencies, dirinfo.Dependency{
				Name:    libname,
				Version: ver,
			})
		}

		if lib.IssuesURL == "" {
			lib.IssuesURL = *repo.IssuesURL
		}
	}

	return di
}
<commit_msg>Added missing close<commit_after>package crawl

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"path"

	"github.com/blang/semver"

	"github.com/google/go-github/github"

	"github.com/xogeny/impact/dirinfo"
	"github.com/xogeny/impact/parsing"
)

func getUses(client *github.Client, user string, reponame string,
	mopath string, opts *github.RepositoryContentGetOptions) (map[string]semver.Version, error) {
	blank := map[string]semver.Version{}

	// Get data for parent directory (we do this because the API sometimes chokes
	// on downloading large Modelica files, but if we do it this way, we can always
	// get the file).
	dir := path.Dir(mopath)
	fname := path.Base(mopath)

	contents := ""

	log.Printf("Getting contents of %s", dir)
	_, dcon, _, err := client.Repositories.GetContents(user, reponame, dir, opts)
	for _, con := range dcon {
		if *con.Name == fname {
			if con.DownloadURL == nil || *con.DownloadURL == "" {
				return blank, fmt.Errorf("No download link present for %s", mopath)
			}
			resp, err := http.Get(*con.DownloadURL)
			if err != nil {
				return blank, fmt.Errorf("Error downloading %s from %s: %v", mopath,
					*con.DownloadURL, err)
			}
			defer resp.Body.Close()
			raw, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return blank, fmt.Errorf("Error reading response: %v", err)
			}
			contents = string(raw)
		}
	}

	uses, err := parsing.ParseUses(contents)
	if err != nil {
		return blank,
			fmt.Errorf("Error while parsing contents of %s in github repository %s: %v",
				mopath, reponame, err)
	}

	return uses, nil
}

func Exists(client *github.Client, user string, reponame string,
	path string, opts *github.RepositoryContentGetOptions) (file bool, dir bool) {
	f, d, _, err := client.Repositories.GetContents(user, reponame, path, opts)
	if err != nil {
		return false, false
	}
	if f != nil {
		return true, false
	}
	if d != nil {
		return false, true
	}
	return false, false
}

// The goal of this function is to construct a DirectoryInfo object.  It does this by first
// reading whatever directory information it can find in impact.json.  Then it tries to
// "infer" the rest using some heuristics (to lower the burden on library developers)
func ExtractInfo(client *github.Client, user string, altname string, repo github.Repository,
	sha string, versionString string, verbose bool, logger *log.Logger) dirinfo.DirectoryInfo {

	// Extract the name of the respository
	repostr := *repo.Name

	// Extract information about the owner of this repository (note: this is a URI)
	owner_uri := user
	if repo.Owner.HTMLURL != nil {
		owner_uri = *repo.Owner.HTMLURL
	}

	// Get the owner's email address, if provided
	email := ""
	if repo.Owner.Email != nil {
		email = *repo.Owner.Email
	}

	// Specify which version of the repository we are interested in
	opts := &github.RepositoryContentGetOptions{
		Ref: sha,
	}

	// Create a "blank" directory info as default
	di := dirinfo.MakeDirectoryInfo()

	// Parse any impact.json file the is present
	dcon, _, _, err := client.Repositories.GetContents(user, repostr, "impact.json", opts)

	// If impact.json exists, parse it and use that as our baseline
	if dcon != nil && err == nil {
		pdi, perr := dirinfo.Parse(*dcon.Content)
		if perr == nil {
			log.Printf("Parsed impact.json file in %s: %v", repostr, pdi)
			di = pdi
		} else {
			log.Printf("Unable to parse impact.json in %s: %v", repostr, perr)
		}
	}

	// NOW, use heuristics to infer missing information

	// Is owner information missing.  If so, use either the owner whose repositories
	// are being scanned or the owner of this particular repository (see logic above
	// regarding 'owner')
	if di.OwnerURI == "" {
		di.OwnerURI = owner_uri
	}

	// Ditto with email
	if di.Email == "" {
		di.Email = email
	}

	// This is local function where only the path is an argument, all the other
	// arguments come from the local environment.
	lookupPath := func(path string) (bool, bool) {
		return Exists(client, user, repostr, path, opts)
	}

	// Are any libraries mentioned?  If not, we need to figure out what the structure
	// is here.  There are two patterns.  Either a file named <RepoName>.mo or a
	// directory named <RepoName>.  If neither of these conventions is followed, the
	// library developers needs to add an explicit impact.json
	if len(di.Libraries) == 0 {
		// Provide default values for path and isfile
		path := ""
		isfile := true

		// This is a simplified version of the version string that strips build
		// information away.  This is necessary because some library developers
		// don't include build information in the names of files and directories
		// that include an explicit version number.  So we check for this as well.
		sver := parsing.SimpleVersion(versionString)

		// Flag to indicate whether we have found a library
		found := false

		// A list of Modelica files to look for that could be libraries
		// (stored as files)
		filenames := []string{
			repostr + ".mo",
			repostr + " " + versionString + ".mo",
			altname + ".mo",
			altname + " " + versionString + ".mo",
		}

		// A list of directories to look for that could be Modelica libraries
		// (stored as directories)
		dirnames := []string{
			repostr,
			repostr + " " + versionString,
			altname,
			altname + " " + versionString,
		}

		// If the simple version string isn't the same as the full version string,
		// add a few more variations.
		if sver != versionString {
			filenames = append(filenames, repostr+" "+sver+".mo")
			filenames = append(filenames, altname+" "+sver+".mo")
			dirnames = append(dirnames, repostr+" "+sver)
			dirnames = append(dirnames, altname+" "+sver)
		}

		// First, check for the library stored as a file...
		for _, fpath := range filenames {
			// Look for that file in the repository
			fexists, _ := lookupPath(fpath)
			if verbose {
				log.Printf("Looking for file %s in %s:%s: %v", fpath, repostr,
					versionString, fexists)
			}
			// If no library has been found yet and this file exists, record
			// its information and set the found flag
			if !found && fexists {
				path = fpath
				found = true
				if verbose {
					log.Printf("  Found")
				}
			}
		}

		// Next, check for the library stored as a directory...
		for _, dpath := range dirnames {
			// Look for a directory in the repository with the matching name
			// TODO: Look for package.mo inside!
			_, dexists := lookupPath(dpath)
			if verbose {
				log.Printf("Looking for directory %s in %s:%s: %v", dpath, repostr,
					versionString, dexists)
			}
			// If we haven't yet found a match and this directory exists, then
			// record the information and set found flag
			if !found && dexists {
				path = dpath
				isfile = false
				found = true
				if verbose {
					log.Printf("  Found")
				}
			}
		}

		// Finally, if we still haven't found a library, chec to see if the
		// repository itself is a library
		if !found {
			// Check if repository IS a package
			fexists, _ := lookupPath("package.mo")
			if verbose {
				log.Printf("Check if directory %s:%s is a package: %v", repostr, versionString,
					fexists)
			}
			// If so, record information about it
			if fexists {
				path = ""
				isfile = false
				found = true
				if verbose {
					log.Printf("  Found")
				}
			}
		}

		// If we found a library, create an instance of dirinfo.LocalLibrary
		// N.B. - These heuristics only look for one library in the repository.
		// If a library developer wants to store more than one, they need to
		// state this explicitly in the impact.json file and not rely on us
		// inferring it somehow.
		if found {
			di.Libraries = []*dirinfo.LocalLibrary{
				&dirinfo.LocalLibrary{
					Name:         repostr,
					Path:         path,
					IsFile:       isfile,
					Dependencies: []dirinfo.Dependency{},
				},
			}
		} else {
			if verbose {
				log.Printf("Nothing found in %s/%s for among %v or %v", repostr, versionString,
					filenames, dirnames)
			}
		}
	}

	// Now, let's loop over all the libraries we are aware of...
	for _, lib := range di.Libraries {
		// Determine path to top-level package in repository
		path := lib.Path
		if !lib.IsFile {
			path = fmt.Sprintf("%s/package.mo", lib.Path)
		}

		// Extract information about any libraries this library uses
		uses, err := getUses(client, user, repostr, path, opts)
		if err != nil {
			log.Printf("Error extracting uses annotation: %v", err)
			continue
		}

		for libname, ver := range uses {
			lib.Dependencies = append(lib.Dependencies, dirinfo.Dependency{
				Name:    libname,
				Version: ver,
			})
		}

		if lib.IssuesURL == "" {
			lib.IssuesURL = *repo.IssuesURL
		}
	}

	return di
}
<|endoftext|>