<commit_before>package filesystem

import (
	"context"
	"os"
	"path/filepath"
	"time"

	"github.com/pkg/errors"
)

const (
	// DefaultPollingInterval is the default watch polling interval, in seconds.
	DefaultPollingInterval = 10
	// defaultInitialContentMapCapacity is the default content map capacity if
	// the existing content map is empty.
	defaultInitialContentMapCapacity = 1024
)

// fileInfoEqual determines whether or not file metadata is equivalent (i.e.
// whether or not a change notification is warranted).
func fileInfoEqual(first, second os.FileInfo) bool {
	// Compare modes.
	if first.Mode() != second.Mode() {
		return false
	}

	// If we're dealing with directories, don't check size or time. Size doesn't
	// really make sense and modification time will be affected by our
	// executability preservation or Unicode decomposition probe file creation.
	if first.IsDir() {
		return true
	}

	// Compare size and time.
	return first.Size() == second.Size() &&
		first.ModTime().Equal(second.ModTime())
}

// poll creates a simple snapshot of a synchronization root, detecting (and
// optionally tracking) changes from an existing snapshot.
func poll(root string, existing map[string]os.FileInfo, trackChanges bool) (map[string]os.FileInfo, bool, map[string]bool, error) {
	// Create our result map.
	initialContentMapCapacity := len(existing)
	if initialContentMapCapacity == 0 {
		initialContentMapCapacity = defaultInitialContentMapCapacity
	}
	contents := make(map[string]os.FileInfo, initialContentMapCapacity)

	// Create our change tracking map (only allocating if we're actually
	// tracking changes).
	var changes map[string]bool
	if trackChanges {
		// TODO: Should we use an initial capacity here? It'll be small or empty
		// most of the time, but we'll have a large allocation penalty on the
		// first scan. Perhaps we can use a non-zero initial capacity if
		// len(existing) == 0?
		changes = make(map[string]bool)
	}

	// Create a walk visitor.
	changed := false
	rootDoesNotExist := false
	visitor := func(path string, info os.FileInfo, err error) error {
		// Handle walk error cases.
		if err != nil {
			// If we're at the root and this is a non-existence error, then we
			// can create a valid result (and empty map) as well as determine
			// whether or not there's been a change.
			if path == root && os.IsNotExist(err) {
				changed = len(existing) > 0
				rootDoesNotExist = true
				return err
			}

			// If this is a non-root non-existence error, then something was
			// seen during the directory listing and then failed the stat call.
			// This is a sign of concurrent deletion, so just ignore this file.
			// Our later checks will determine if this was concurent deletion of
			// a file we're meant to be watching or one of our probe files.
			if os.IsNotExist(err) {
				return nil
			}

			// Other errors are more problematic.
			return err
		}

		// If this is an executability preservation or Unicode decomposition
		// probe file, then ignore it. Also ignore intermediate files generated
		// by atomic operations.
		if name := filepath.Base(path); IsExecutabilityProbeFileName(name) {
			return nil
		} else if IsUnicodeProbeFileName(name) {
			return nil
		} else if IsAtomicOperationFileName(name) {
			return nil
		}

		// Insert the entry for this path.
		contents[path] = info

		// Compare the entry for this path.
		pathChanged := false
		if previous, ok := existing[path]; !ok || !fileInfoEqual(info, previous) {
			pathChanged = true
		}

		// Update change tracker.
		if pathChanged {
			changed = true
		}

		// If we're tracking changes and this path changed, register it. We
		// always track the parent of the path, and for directories we also
		// track the path itself, since these are where changes are going to be
		// visible.
		if trackChanges && pathChanged {
			if info.IsDir() {
				changes[path] = true
			}
			if path != root {
				changes[filepath.Dir(path)] = true
			}
		}

		// Success.
		return nil
	}

	// Perform the walk. If it fails, and it's not due to the root not existing,
	// then we can't return a valid result and need to abort.
	if err := Walk(root, visitor); err != nil && !rootDoesNotExist {
		return nil, false, nil, errors.Wrap(err, "unable to perform filesystem walk")
	}

	// If the length of the result map has changed, then there's been a change.
	// This could be due to files being deleted.
	if len(contents) != len(existing) {
		changed = true
	}

	// Done.
	return contents, changed, changes, nil
}

// watchPoll performs poll-based filesystem watching.
func watchPoll(context context.Context, root string, events chan struct{}, pollInterval uint32) error {
	// Compute the polling interval.
	if pollInterval == 0 {
		pollInterval = DefaultPollingInterval
	}
	pollIntervalDuration := time.Duration(pollInterval) * time.Second

	// Create a timer to regulate polling. Start it with a 0 duration so that
	// the first polling takes place immediately. Subsequent pollings will take
	// place at the normal interval.
	timer := time.NewTimer(0)

	// Loop and poll for changes, but watch for cancellation.
	var contents map[string]os.FileInfo
	for {
		select {
		case <-context.Done():
			// Abort the watch.
			return errors.New("watch cancelled")
		case <-timer.C:
			// Perform a scan. If there's an error or no change, then reset the
			// timer and try again. We have to assume that errors here are due
			// to concurrent modifications, so there's not much we can do to
			// handle them.
			newContents, changed, _, err := poll(root, contents, false)
			if err != nil || !changed {
				timer.Reset(pollIntervalDuration)
				continue
			}

			// Store the new contents.
			contents = newContents

			// Forward the event in a non-blocking fashion.
			select {
			case events <- struct{}{}:
			default:
			}

			// Reset the timer and continue polling.
			timer.Reset(pollIntervalDuration)
		}
	}
}
<commit_msg>Cleaned up poll-based watcher comment.<commit_after>package filesystem

import (
	"context"
	"os"
	"path/filepath"
	"time"

	"github.com/pkg/errors"
)

const (
	// DefaultPollingInterval is the default watch polling interval, in seconds.
	DefaultPollingInterval = 10
	// defaultInitialContentMapCapacity is the default content map capacity if
	// the existing content map is empty.
	defaultInitialContentMapCapacity = 1024
)

// fileInfoEqual determines whether or not file metadata is equivalent (i.e.
// whether or not a change notification is warranted).
func fileInfoEqual(first, second os.FileInfo) bool {
	// Compare modes.
	if first.Mode() != second.Mode() {
		return false
	}

	// If we're dealing with directories, don't check size or time. Size doesn't
	// really make sense and modification time will be affected by our
	// executability preservation or Unicode decomposition probe file creation.
	if first.IsDir() {
		return true
	}

	// Compare size and time.
	return first.Size() == second.Size() &&
		first.ModTime().Equal(second.ModTime())
}

// poll creates a simple snapshot of a synchronization root, detecting (and
// optionally tracking) changes from an existing snapshot.
func poll(root string, existing map[string]os.FileInfo, trackChanges bool) (map[string]os.FileInfo, bool, map[string]bool, error) {
	// Create our result map.
	initialContentMapCapacity := len(existing)
	if initialContentMapCapacity == 0 {
		initialContentMapCapacity = defaultInitialContentMapCapacity
	}
	contents := make(map[string]os.FileInfo, initialContentMapCapacity)

	// Create our change tracking map (only allocating if we're actually
	// tracking changes).
	var changes map[string]bool
	if trackChanges {
		// TODO: Should we use an initial capacity here? It'll be small or empty
		// most of the time, but we'll have a large allocation penalty on the
		// first scan. Perhaps we can use a non-zero initial capacity if
		// len(existing) == 0?
		changes = make(map[string]bool)
	}

	// Create a walk visitor.
	changed := false
	rootDoesNotExist := false
	visitor := func(path string, info os.FileInfo, err error) error {
		// Handle walk error cases.
		if err != nil {
			// If we're at the root and this is a non-existence error, then we
			// can create a valid result (and empty map) as well as determine
			// whether or not there's been a change.
			if path == root && os.IsNotExist(err) {
				changed = len(existing) > 0
				rootDoesNotExist = true
				return err
			}

			// If this is a non-root non-existence error, then something was
			// seen during the directory listing and then failed the stat call.
			// This is a sign of concurrent deletion, so just ignore this file.
			// Our later checks will determine if this was concurent deletion of
			// a file we're meant to be watching.
			if os.IsNotExist(err) {
				return nil
			}

			// Other errors are more problematic.
			return err
		}

		// If this is an executability preservation or Unicode decomposition
		// probe file, then ignore it. Also ignore intermediate files generated
		// by atomic operations.
		if name := filepath.Base(path); IsExecutabilityProbeFileName(name) {
			return nil
		} else if IsUnicodeProbeFileName(name) {
			return nil
		} else if IsAtomicOperationFileName(name) {
			return nil
		}

		// Insert the entry for this path.
		contents[path] = info

		// Compare the entry for this path.
		pathChanged := false
		if previous, ok := existing[path]; !ok || !fileInfoEqual(info, previous) {
			pathChanged = true
		}

		// Update change tracker.
		if pathChanged {
			changed = true
		}

		// If we're tracking changes and this path changed, register it. We
		// always track the parent of the path, and for directories we also
		// track the path itself, since these are where changes are going to be
		// visible.
		if trackChanges && pathChanged {
			if info.IsDir() {
				changes[path] = true
			}
			if path != root {
				changes[filepath.Dir(path)] = true
			}
		}

		// Success.
		return nil
	}

	// Perform the walk. If it fails, and it's not due to the root not existing,
	// then we can't return a valid result and need to abort.
	if err := Walk(root, visitor); err != nil && !rootDoesNotExist {
		return nil, false, nil, errors.Wrap(err, "unable to perform filesystem walk")
	}

	// If the length of the result map has changed, then there's been a change.
	// This could be due to files being deleted.
	if len(contents) != len(existing) {
		changed = true
	}

	// Done.
	return contents, changed, changes, nil
}

// watchPoll performs poll-based filesystem watching.
func watchPoll(context context.Context, root string, events chan struct{}, pollInterval uint32) error {
	// Compute the polling interval.
	if pollInterval == 0 {
		pollInterval = DefaultPollingInterval
	}
	pollIntervalDuration := time.Duration(pollInterval) * time.Second

	// Create a timer to regulate polling. Start it with a 0 duration so that
	// the first polling takes place immediately. Subsequent pollings will take
	// place at the normal interval.
	timer := time.NewTimer(0)

	// Loop and poll for changes, but watch for cancellation.
	var contents map[string]os.FileInfo
	for {
		select {
		case <-context.Done():
			// Abort the watch.
			return errors.New("watch cancelled")
		case <-timer.C:
			// Perform a scan. If there's an error or no change, then reset the
			// timer and try again. We have to assume that errors here are due
			// to concurrent modifications, so there's not much we can do to
			// handle them.
			newContents, changed, _, err := poll(root, contents, false)
			if err != nil || !changed {
				timer.Reset(pollIntervalDuration)
				continue
			}

			// Store the new contents.
			contents = newContents

			// Forward the event in a non-blocking fashion.
			select {
			case events <- struct{}{}:
			default:
			}

			// Reset the timer and continue polling.
			timer.Reset(pollIntervalDuration)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package markup_config

import (
	"github.com/gohugoio/hugo/config"
	"github.com/gohugoio/hugo/docshelper"
	"github.com/gohugoio/hugo/markup/asciidocext/asciidocext_config"
	"github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config"
	"github.com/gohugoio/hugo/markup/goldmark/goldmark_config"
	"github.com/gohugoio/hugo/markup/highlight"
	"github.com/gohugoio/hugo/markup/tableofcontents"
	"github.com/gohugoio/hugo/parser"
	"github.com/mitchellh/mapstructure"
)

type Config struct {
	// Default markdown handler for md/markdown extensions.
	// Default is "goldmark".
	// Before Hugo 0.60 this was "blackfriday".
	DefaultMarkdownHandler string

	Highlight       highlight.Config
	TableOfContents tableofcontents.Config

	// Content renderers
	Goldmark    goldmark_config.Config
	BlackFriday blackfriday_config.Config

	AsciidocExt asciidocext_config.Config
}

func Decode(cfg config.Provider) (conf Config, err error) {
	conf = Default

	normalizeConfig(cfg)

	m := cfg.GetStringMap("markup")
	if m == nil {
		return
	}

	err = mapstructure.WeakDecode(m, &conf)
	if err != nil {
		return
	}

	if err = applyLegacyConfig(cfg, &conf); err != nil {
		return
	}

	if err = highlight.ApplyLegacyConfig(cfg, &conf.Highlight); err != nil {
		return
	}

	return
}

func normalizeConfig(cfg config.Provider) {
	// Changed from a bool in 0.81.0
	const attrKey = "markup.goldmark.parser.attribute"
	av := cfg.Get(attrKey)
	if avb, ok := av.(bool); ok {
		cfg.Set(attrKey, goldmark_config.ParserAttribute{
			Title: avb,
		})
	}
}

func applyLegacyConfig(cfg config.Provider, conf *Config) error {
	if bm := cfg.GetStringMap("blackfriday"); bm != nil {
		// Legacy top level blackfriday config.
		err := mapstructure.WeakDecode(bm, &conf.BlackFriday)
		if err != nil {
			return err
		}
	}

	if conf.BlackFriday.FootnoteAnchorPrefix == "" {
		conf.BlackFriday.FootnoteAnchorPrefix = cfg.GetString("footnoteAnchorPrefix")
	}

	if conf.BlackFriday.FootnoteReturnLinkContents == "" {
		conf.BlackFriday.FootnoteReturnLinkContents = cfg.GetString("footnoteReturnLinkContents")
	}

	return nil
}

var Default = Config{
	DefaultMarkdownHandler: "goldmark",

	TableOfContents: tableofcontents.DefaultConfig,
	Highlight:       highlight.DefaultConfig,

	Goldmark:    goldmark_config.Default,
	BlackFriday: blackfriday_config.Default,

	AsciidocExt: asciidocext_config.Default,
}

func init() {
	docsProvider := func() docshelper.DocProvider {
		return docshelper.DocProvider{"config": map[string]interface{}{"markup": parser.LowerCaseCamelJSONMarshaller{Value: Default}}}
	}
	docshelper.AddDocProviderFunc(docsProvider)
}
<commit_msg>markup/goldmark: Fix handling of legacy attribute config<commit_after>// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package markup_config

import (
	"github.com/gohugoio/hugo/common/maps"
	"github.com/gohugoio/hugo/config"
	"github.com/gohugoio/hugo/docshelper"
	"github.com/gohugoio/hugo/markup/asciidocext/asciidocext_config"
	"github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config"
	"github.com/gohugoio/hugo/markup/goldmark/goldmark_config"
	"github.com/gohugoio/hugo/markup/highlight"
	"github.com/gohugoio/hugo/markup/tableofcontents"
	"github.com/gohugoio/hugo/parser"
	"github.com/mitchellh/mapstructure"
	"github.com/spf13/cast"
)

type Config struct {
	// Default markdown handler for md/markdown extensions.
	// Default is "goldmark".
	// Before Hugo 0.60 this was "blackfriday".
	DefaultMarkdownHandler string

	Highlight       highlight.Config
	TableOfContents tableofcontents.Config

	// Content renderers
	Goldmark    goldmark_config.Config
	BlackFriday blackfriday_config.Config

	AsciidocExt asciidocext_config.Config
}

func Decode(cfg config.Provider) (conf Config, err error) {
	conf = Default

	m := cfg.GetStringMap("markup")
	if m == nil {
		return
	}
	normalizeConfig(m)

	err = mapstructure.WeakDecode(m, &conf)
	if err != nil {
		return
	}

	if err = applyLegacyConfig(cfg, &conf); err != nil {
		return
	}

	if err = highlight.ApplyLegacyConfig(cfg, &conf.Highlight); err != nil {
		return
	}

	return
}

func normalizeConfig(m map[string]interface{}) {
	v, err := maps.GetNestedParam("goldmark.parser", ".", m)
	if err != nil {
		return
	}
	vm := cast.ToStringMap(v)
	// Changed from a bool in 0.81.0
	if vv, found := vm["attribute"]; found {
		if vvb, ok := vv.(bool); ok {
			vm["attribute"] = goldmark_config.ParserAttribute{
				Title: vvb,
			}
		}
	}
}

func applyLegacyConfig(cfg config.Provider, conf *Config) error {
	if bm := cfg.GetStringMap("blackfriday"); bm != nil {
		// Legacy top level blackfriday config.
		err := mapstructure.WeakDecode(bm, &conf.BlackFriday)
		if err != nil {
			return err
		}
	}

	if conf.BlackFriday.FootnoteAnchorPrefix == "" {
		conf.BlackFriday.FootnoteAnchorPrefix = cfg.GetString("footnoteAnchorPrefix")
	}

	if conf.BlackFriday.FootnoteReturnLinkContents == "" {
		conf.BlackFriday.FootnoteReturnLinkContents = cfg.GetString("footnoteReturnLinkContents")
	}

	return nil
}

var Default = Config{
	DefaultMarkdownHandler: "goldmark",

	TableOfContents: tableofcontents.DefaultConfig,
	Highlight:       highlight.DefaultConfig,

	Goldmark:    goldmark_config.Default,
	BlackFriday: blackfriday_config.Default,

	AsciidocExt: asciidocext_config.Default,
}

func init() {
	docsProvider := func() docshelper.DocProvider {
		return docshelper.DocProvider{"config": map[string]interface{}{"markup": parser.LowerCaseCamelJSONMarshaller{Value: Default}}}
	}
	docshelper.AddDocProviderFunc(docsProvider)
}
<|endoftext|>


<commit_before>package kmodule

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"unsafe"

	"golang.org/x/sys/unix"
)

// Flags to finit_module(2) / FileInit.
const (
	// Ignore symbol version hashes.
	MODULE_INIT_IGNORE_MODVERSIONS = 0x1

	// Ignore kernel version magic.
	MODULE_INIT_IGNORE_VERMAGIC = 0x2
)

// SyscallError contains an error message as well as the actual syscall Errno
type SyscallError struct {
	Msg   string
	Errno syscall.Errno
}

func (s *SyscallError) Error() string {
	if s.Errno != 0 {
		return fmt.Sprintf("%s: %v", s.Msg, s.Errno)
	}
	return s.Msg
}

// Init loads the kernel module given by image with the given options.
func Init(image []byte, opts string) *SyscallError {
	optsNull, err := unix.BytePtrFromString(opts)
	if err != nil {
		return &SyscallError{Msg: fmt.Sprintf("kmodule.Init: could not convert %q to C string: %v", opts, err)}
	}

	if _, _, e := unix.Syscall(unix.SYS_INIT_MODULE, uintptr(unsafe.Pointer(&image[0])), uintptr(len(image)), uintptr(unsafe.Pointer(optsNull))); e != 0 {
		return &SyscallError{
			Msg:   fmt.Sprintf("init_module(%v, %q) failed", image, opts),
			Errno: e,
		}
	}

	return nil
}

// FileInit loads the kernel module contained by `f` with the given opts and
// flags.
//
// FileInit falls back to Init when the finit_module(2) syscall is not available.
func FileInit(f *os.File, opts string, flags uintptr) *SyscallError {
	optsNull, err := unix.BytePtrFromString(opts)
	if err != nil {
		return &SyscallError{Msg: fmt.Sprintf("kmodule.Init: could not convert %q to C string: %v", opts, err)}
	}

	if _, _, e := unix.Syscall(unix.SYS_FINIT_MODULE, f.Fd(), uintptr(unsafe.Pointer(optsNull)), flags); e == unix.ENOSYS {
		if flags != 0 {
			return &SyscallError{Msg: fmt.Sprintf("finit_module unavailable"), Errno: e}
		}

		// Fall back to regular init_module(2).
		img, err := ioutil.ReadAll(f)
		if err != nil {
			return &SyscallError{Msg: fmt.Sprintf("kmodule.FileInit: %v", err)}
		}
		return Init(img, opts)
	} else if e != 0 {
		return &SyscallError{
			Msg:   fmt.Sprintf("finit_module(%v, %q, %#x) failed", f, opts, flags),
			Errno: e,
		}
	}

	return nil
}

// Delete removes a kernel module.
func Delete(name string, flags uintptr) *SyscallError {
	modnameptr, err := unix.BytePtrFromString(name)
	if err != nil {
		return &SyscallError{Msg: fmt.Sprintf("could not delete module %q: %v", name, err)}
	}

	if _, _, e := unix.Syscall(unix.SYS_DELETE_MODULE, uintptr(unsafe.Pointer(modnameptr)), flags, 0); e != 0 {
		return &SyscallError{Msg: fmt.Sprintf("could not delete module %q", name), Errno: e}
	}

	return nil
}

type modState uint8

const (
	unloaded modState = iota
	loading
	loaded
)

type dependency struct {
	state modState
	deps  []string
}

type depMap map[string]*dependency

// ProbeOpts contains optional parameters to Probe.
//
// An empty ProbeOpts{} should lead to the default behavior.
type ProbeOpts struct {
	DryRun bool
}

// Probe loads the given kernel module and its dependencies.
// It is calls ProbeOptions with the default ProbeOpts.
func Probe(name string, modParams string) *SyscallError {
	return ProbeOptions(name, modParams, ProbeOpts{})
}

// ProbeOptions loads the given kernel module and its dependencies.
// This functions takes ProbeOpts.
func ProbeOptions(name, modParams string, opts ProbeOpts) *SyscallError {
	deps, err := genDeps()
	if err != nil {
		return &SyscallError{Msg: fmt.Sprintf("could not generate dependency map %v", err)}
	}

	modPath, err := findModPath(name, deps)
	if err != nil {
		return &SyscallError{Msg: fmt.Sprintf("could not find module path %q: %v", name, err)}
	}

	if !opts.DryRun {
		// if the module is already loaded or does not have deps, or all of them are loaded
		// then this succeeds and we are done
		if err := loadModule(modPath, modParams, opts); err == nil {
			return nil
		}
		// okay, we have to try the hard way and load dependencies first.
	} else {
		fmt.Println("Unique dependencies in load order, already loaded ones get skipped:")
	}

	deps[modPath].state = loading
	for _, d := range deps[modPath].deps {
		if err := loadDeps(d, deps, opts); err != nil {
			return err
		}
	}
	if err := loadModule(modPath, modParams, opts); err != nil {
		return err
	}
	// we don't care to set the state to loaded
	// deps[modPath].state = loaded
	return nil
}

func genDeps() (depMap, error) {
	deps := make(depMap)

	var u unix.Utsname
	if err := unix.Uname(&u); err != nil {
		return nil, fmt.Errorf("could not get release (uname -r): %v", err)
	}
	rel := string(u.Release[:bytes.IndexByte(u.Release[:], 0)])

	moduleDir := filepath.Join("/lib/modules", strings.TrimSpace(rel))

	f, err := os.Open(filepath.Join(moduleDir, "modules.dep"))
	if err != nil {
		return nil, fmt.Errorf("could not open dependency file: %v", err)
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		txt := scanner.Text()
		nameDeps := strings.Split(txt, ":")
		modPath, modDeps := nameDeps[0], nameDeps[1]
		modPath = filepath.Join(moduleDir, strings.TrimSpace(modPath))

		var dependency dependency
		if len(modDeps) > 0 {
			for _, dep := range strings.Split(strings.TrimSpace(modDeps), " ") {
				dependency.deps = append(dependency.deps, filepath.Join(moduleDir, dep))
			}
		}
		deps[modPath] = &dependency
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return deps, nil
}

func findModPath(name string, m depMap) (string, error) {
	for mp := range m {
		if strings.HasSuffix(mp, name+".ko") {
			return mp, nil
		}
	}

	return "", fmt.Errorf("Could not find path for module %q", name)
}

func loadDeps(path string, m depMap, opts ProbeOpts) *SyscallError {
	dependency, ok := m[path]
	if !ok {
		return &SyscallError{Msg: fmt.Sprintf("could not find dependency %q", path)}
	}

	if dependency.state == loading {
		return &SyscallError{Msg: fmt.Sprintf("circular dependency! %q already LOADING", path)}
	} else if dependency.state == loaded {
		return nil
	}

	m[path].state = loading

	for _, dep := range dependency.deps {
		if err := loadDeps(dep, m, opts); err != nil {
			return err
		}
	}

	// done with dependencies, load module
	if err := loadModule(path, "", opts); err != nil {
		return err
	}
	m[path].state = loaded

	return nil
}

func loadModule(path, modParams string, opts ProbeOpts) *SyscallError {
	if opts.DryRun {
		fmt.Println(path)
		return nil
	}

	f, err := os.Open(path)
	if err != nil {
		return &SyscallError{Msg: fmt.Sprintf("could not open %q: %v", path, err)}
	}
	defer f.Close()

	if err := FileInit(f, modParams, 0); err != nil && err.Errno != unix.EEXIST {
		return err
	}

	return nil
}
<commit_msg>    Fixing errors in kmodule_linux.go<commit_after>package kmodule

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"unsafe"

	"golang.org/x/sys/unix"
)

// Flags to finit_module(2) / FileInit.
const (
	// Ignore symbol version hashes.
	MODULE_INIT_IGNORE_MODVERSIONS = 0x1

	// Ignore kernel version magic.
	MODULE_INIT_IGNORE_VERMAGIC = 0x2
)

// SyscallError contains an error message as well as the actual syscall Errno
type SyscallError struct {
	Msg   string
	Errno syscall.Errno
}

func (s *SyscallError) Error() string {
	if s.Errno != 0 {
		return fmt.Sprintf("%s: %v", s.Msg, s.Errno)
	}
	return s.Msg
}

// Init loads the kernel module given by image with the given options.
func Init(image []byte, opts string) error {
	optsNull, err := unix.BytePtrFromString(opts)
	if err != nil {
		return &SyscallError{Msg: fmt.Sprintf("kmodule.Init: could not convert %q to C string: %v", opts, err)}
	}

	if _, _, e := unix.Syscall(unix.SYS_INIT_MODULE, uintptr(unsafe.Pointer(&image[0])), uintptr(len(image)), uintptr(unsafe.Pointer(optsNull))); e != 0 {
		return &SyscallError{
			Msg:   fmt.Sprintf("init_module(%v, %q) failed", image, opts),
			Errno: e,
		}
	}

	return nil
}

// FileInit loads the kernel module contained by `f` with the given opts and
// flags.
//
// FileInit falls back to Init when the finit_module(2) syscall is not available.
func FileInit(f *os.File, opts string, flags uintptr) error {
	optsNull, err := unix.BytePtrFromString(opts)
	if err != nil {
		return &SyscallError{Msg: fmt.Sprintf("kmodule.Init: could not convert %q to C string: %v", opts, err)}
	}

	if _, _, e := unix.Syscall(unix.SYS_FINIT_MODULE, f.Fd(), uintptr(unsafe.Pointer(optsNull)), flags); e == unix.ENOSYS {
		if flags != 0 {
			return &SyscallError{Msg: fmt.Sprintf("finit_module unavailable"), Errno: e}
		}

		// Fall back to regular init_module(2).
		img, err := ioutil.ReadAll(f)
		if err != nil {
			return &SyscallError{Msg: fmt.Sprintf("kmodule.FileInit: %v", err)}
		}
		return Init(img, opts)
	} else if e != 0 {
		return &SyscallError{
			Msg:   fmt.Sprintf("finit_module(%v, %q, %#x) failed", f, opts, flags),
			Errno: e,
		}
	}

	return nil
}

// Delete removes a kernel module.
func Delete(name string, flags uintptr) error {
	modnameptr, err := unix.BytePtrFromString(name)
	if err != nil {
		return &SyscallError{Msg: fmt.Sprintf("could not delete module %q: %v", name, err)}
	}

	if _, _, e := unix.Syscall(unix.SYS_DELETE_MODULE, uintptr(unsafe.Pointer(modnameptr)), flags, 0); e != 0 {
		return &SyscallError{Msg: fmt.Sprintf("could not delete module %q", name), Errno: e}
	}

	return nil
}

type modState uint8

const (
	unloaded modState = iota
	loading
	loaded
)

type dependency struct {
	state modState
	deps  []string
}

type depMap map[string]*dependency

// ProbeOpts contains optional parameters to Probe.
//
// An empty ProbeOpts{} should lead to the default behavior.
type ProbeOpts struct {
	DryRun bool
}

// Probe loads the given kernel module and its dependencies.
// It is calls ProbeOptions with the default ProbeOpts.
func Probe(name string, modParams string) error {
	return ProbeOptions(name, modParams, ProbeOpts{})
}

// ProbeOptions loads the given kernel module and its dependencies.
// This functions takes ProbeOpts.
func ProbeOptions(name, modParams string, opts ProbeOpts) error {
	deps, err := genDeps()
	if err != nil {
		return &SyscallError{Msg: fmt.Sprintf("could not generate dependency map %v", err)}
	}

	modPath, err := findModPath(name, deps)
	if err != nil {
		return &SyscallError{Msg: fmt.Sprintf("could not find module path %q: %v", name, err)}
	}

	if !opts.DryRun {
		// if the module is already loaded or does not have deps, or all of them are loaded
		// then this succeeds and we are done
		if err := loadModule(modPath, modParams, opts); err == nil {
			return nil
		}
		// okay, we have to try the hard way and load dependencies first.
	} else {
		fmt.Println("Unique dependencies in load order, already loaded ones get skipped:")
	}

	deps[modPath].state = loading
	for _, d := range deps[modPath].deps {
		if err := loadDeps(d, deps, opts); err != nil {
			return err
		}
	}
	if err := loadModule(modPath, modParams, opts); err != nil {
		return err
	}
	// we don't care to set the state to loaded
	// deps[modPath].state = loaded
	return nil
}

func genDeps() (depMap, error) {
	deps := make(depMap)

	var u unix.Utsname
	if err := unix.Uname(&u); err != nil {
		return nil, fmt.Errorf("could not get release (uname -r): %v", err)
	}
	rel := string(u.Release[:bytes.IndexByte(u.Release[:], 0)])

	moduleDir := filepath.Join("/lib/modules", strings.TrimSpace(rel))

	f, err := os.Open(filepath.Join(moduleDir, "modules.dep"))
	if err != nil {
		return nil, fmt.Errorf("could not open dependency file: %v", err)
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		txt := scanner.Text()
		nameDeps := strings.Split(txt, ":")
		modPath, modDeps := nameDeps[0], nameDeps[1]
		modPath = filepath.Join(moduleDir, strings.TrimSpace(modPath))

		var dependency dependency
		if len(modDeps) > 0 {
			for _, dep := range strings.Split(strings.TrimSpace(modDeps), " ") {
				dependency.deps = append(dependency.deps, filepath.Join(moduleDir, dep))
			}
		}
		deps[modPath] = &dependency
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return deps, nil
}

func findModPath(name string, m depMap) (string, error) {
	for mp := range m {
		if strings.HasSuffix(mp, name+".ko") {
			return mp, nil
		}
	}

	return "", fmt.Errorf("Could not find path for module %q", name)
}

func loadDeps(path string, m depMap, opts ProbeOpts) error {
	dependency, ok := m[path]
	if !ok {
		return &SyscallError{Msg: fmt.Sprintf("could not find dependency %q", path)}
	}

	if dependency.state == loading {
		return &SyscallError{Msg: fmt.Sprintf("circular dependency! %q already LOADING", path)}
	} else if dependency.state == loaded {
		return nil
	}

	m[path].state = loading

	for _, dep := range dependency.deps {
		if err := loadDeps(dep, m, opts); err != nil {
			return err
		}
	}

	// done with dependencies, load module
	if err := loadModule(path, "", opts); err != nil {
		return err
	}
	m[path].state = loaded

	return nil
}

func loadModule(path, modParams string, opts ProbeOpts) error {
	if opts.DryRun {
		fmt.Println(path)
		return nil
	}

	f, err := os.Open(path)
	if err != nil {
		return &SyscallError{Msg: fmt.Sprintf("could not open %q: %v", path, err)}
	}
	defer f.Close()

	if err := FileInit(f, modParams, 0); err != nil {
		if serr, ok := err.(*SyscallError); !ok || (ok && serr.Errno != unix.EEXIST) {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"net"
	"reflect"
	"strings"
	"time"

	"github.com/golang/glog"

	configapi "github.com/openshift/origin/pkg/cmd/server/api"
	networkapi "github.com/openshift/origin/pkg/network/apis/network"
	networkclient "github.com/openshift/origin/pkg/network/generated/internalclientset"
	"github.com/openshift/origin/pkg/util/netutils"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	kerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apimachinery/pkg/watch"
	kcache "k8s.io/client-go/tools/cache"
	kapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/extensions"
	kinternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
)

func HostSubnetToString(subnet *networkapi.HostSubnet) string {
	return fmt.Sprintf("%s (host: %q, ip: %q, subnet: %q)", subnet.Name, subnet.Host, subnet.HostIP, subnet.Subnet)
}

func ClusterNetworkToString(n *networkapi.ClusterNetwork) string {
	return fmt.Sprintf("%s (network: %q, hostSubnetBits: %d, serviceNetwork: %q, pluginName: %q)", n.Name, n.Network, n.HostSubnetLength, n.ServiceNetwork, n.PluginName)
}

func ClusterNetworkListContains(clusterNetworks []ClusterNetwork, ipaddr net.IP) (*net.IPNet, bool) {
	for _, cn := range clusterNetworks {
		if cn.ClusterCIDR.Contains(ipaddr) {
			return cn.ClusterCIDR, true
		}
	}
	return nil, false
}

type NetworkInfo struct {
	ClusterNetworks []ClusterNetwork
	ServiceNetwork  *net.IPNet
}

type ClusterNetwork struct {
	ClusterCIDR      *net.IPNet
	HostSubnetLength uint32
}

func ParseNetworkInfo(clusterNetwork []networkapi.ClusterNetworkEntry, serviceNetwork string) (*NetworkInfo, error) {
	var cns []ClusterNetwork

	for _, entry := range clusterNetwork {
		cidr, err := netutils.ParseCIDRMask(entry.CIDR)
		if err != nil {
			_, cidr, err := net.ParseCIDR(entry.CIDR)
			if err != nil {
				return nil, fmt.Errorf("failed to parse ClusterNetwork CIDR %s: %v", cidr, err)
			}
			glog.Errorf("Configured clusterNetworks value %q is invalid; treating it as %q", entry.CIDR, cidr.String())
		}
		cns = append(cns, ClusterNetwork{ClusterCIDR: cidr, HostSubnetLength: entry.HostSubnetLength})
	}

	sn, err := netutils.ParseCIDRMask(serviceNetwork)
	if err != nil {
		_, sn, err := net.ParseCIDR(serviceNetwork)
		if err != nil {
			return nil, fmt.Errorf("failed to parse ServiceNetwork CIDR %s: %v", serviceNetwork, err)
		}
		glog.Errorf("Configured serviceNetworkCIDR value %q is invalid; treating it as %q", serviceNetwork, sn.String())
	}

	return &NetworkInfo{
		ClusterNetworks: cns,
		ServiceNetwork:  sn,
	}, nil
}

func (ni *NetworkInfo) ValidateNodeIP(nodeIP string) error {
	if nodeIP == "" || nodeIP == "127.0.0.1" {
		return fmt.Errorf("invalid node IP %q", nodeIP)
	}

	// Ensure each node's NodeIP is not contained by the cluster network,
	// which could cause a routing loop. (rhbz#1295486)
	ipaddr := net.ParseIP(nodeIP)
	if ipaddr == nil {
		return fmt.Errorf("failed to parse node IP %s", nodeIP)
	}

	if conflictingCIDR, found := ClusterNetworkListContains(ni.ClusterNetworks, ipaddr); found {
		return fmt.Errorf("node IP %s conflicts with cluster network %s", nodeIP, conflictingCIDR.String())
	}
	if ni.ServiceNetwork.Contains(ipaddr) {
		return fmt.Errorf("node IP %s conflicts with service network %s", nodeIP, ni.ServiceNetwork.String())
	}

	return nil
}

func (ni *NetworkInfo) CheckHostNetworks(hostIPNets []*net.IPNet) error {
	errList := []error{}
	for _, ipNet := range hostIPNets {
		for _, clusterNetwork := range ni.ClusterNetworks {
			if configapi.CIDRsOverlap(ipNet.String(), clusterNetwork.ClusterCIDR.String()) {
				errList = append(errList, fmt.Errorf("cluster IP: %s conflicts with host network: %s", clusterNetwork.ClusterCIDR.IP.String(), ipNet.String()))
			}
		}
		if configapi.CIDRsOverlap(ipNet.String(), ni.ServiceNetwork.String()) {
			errList = append(errList, fmt.Errorf("service IP: %s conflicts with host network: %s", ni.ServiceNetwork.String(), ipNet.String()))
		}
	}
	return kerrors.NewAggregate(errList)
}

func (ni *NetworkInfo) CheckClusterObjects(subnets []networkapi.HostSubnet, pods []kapi.Pod, services []kapi.Service) error {
	var errList []error

	for _, subnet := range subnets {
		subnetIP, _, _ := net.ParseCIDR(subnet.Subnet)
		if subnetIP == nil {
			errList = append(errList, fmt.Errorf("failed to parse network address: %s", subnet.Subnet))
		} else if _, contains := ClusterNetworkListContains(ni.ClusterNetworks, subnetIP); !contains {
			errList = append(errList, fmt.Errorf("existing node subnet: %s is not part of any cluster network CIDR", subnet.Subnet))
		}
		if len(errList) >= 10 {
			break
		}
	}
	for _, pod := range pods {
		if pod.Spec.SecurityContext != nil && pod.Spec.SecurityContext.HostNetwork {
			continue
		}
		if _, contains := ClusterNetworkListContains(ni.ClusterNetworks, net.ParseIP(pod.Status.PodIP)); !contains && pod.Status.PodIP != "" {
			errList = append(errList, fmt.Errorf("existing pod %s:%s with IP %s is not part of cluster network", pod.Namespace, pod.Name, pod.Status.PodIP))
			if len(errList) >= 10 {
				break
			}
		}
	}
	for _, svc := range services {
		svcIP := net.ParseIP(svc.Spec.ClusterIP)
		if svcIP != nil && !ni.ServiceNetwork.Contains(svcIP) {
			errList = append(errList, fmt.Errorf("existing service %s:%s with IP %s is not part of service network %s", svc.Namespace, svc.Name, svc.Spec.ClusterIP, ni.ServiceNetwork.String()))
			if len(errList) >= 10 {
				break
			}
		}
	}

	if len(errList) >= 10 {
		errList = append(errList, fmt.Errorf("too many errors... truncating"))
	}
	return kerrors.NewAggregate(errList)
}

func GetNetworkInfo(networkClient networkclient.Interface) (*NetworkInfo, error) {
	cn, err := networkClient.Network().ClusterNetworks().Get(networkapi.ClusterNetworkDefault, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}

	return ParseNetworkInfo(cn.ClusterNetworks, cn.ServiceNetwork)
}

type ResourceName string

const (
	Nodes                 ResourceName = "Nodes"
	Namespaces            ResourceName = "Namespaces"
	NetNamespaces         ResourceName = "NetNamespaces"
	Services              ResourceName = "Services"
	HostSubnets           ResourceName = "HostSubnets"
	Pods                  ResourceName = "Pods"
	EgressNetworkPolicies ResourceName = "EgressNetworkPolicies"
	NetworkPolicies       ResourceName = "NetworkPolicies"
)

func newEventQueue(client kcache.Getter, resourceName ResourceName, expectedType interface{}, namespace string) *EventQueue {
	rn := strings.ToLower(string(resourceName))
	lw := kcache.NewListWatchFromClient(client, rn, namespace, fields.Everything())
	eventQueue := NewEventQueue(DeletionHandlingMetaNamespaceKeyFunc)
	// Repopulate event queue every 30 mins
	// Existing items in the event queue will have watch.Modified event type
	kcache.NewReflector(lw, expectedType, eventQueue, 30*time.Minute).Run()
	return eventQueue
}

// Run event queue for the given resource. The 'process' function is called
// repeatedly with each available cache.Delta that describes state changes
// to an object. If the process function returns an error queued changes
// for that object are dropped but processing continues with the next available
// object's cache.Deltas.  The error is logged with call stack information.
//
// NOTE: this function will handle DeletedFinalStateUnknown delta objects
// automatically, which may not always be what you want since the now-deleted
// object may be stale.
func RunEventQueue(client kcache.Getter, resourceName ResourceName, process ProcessEventFunc) {
	var expectedType interface{}

	switch resourceName {
	case HostSubnets:
		expectedType = &networkapi.HostSubnet{}
	case NetNamespaces:
		expectedType = &networkapi.NetNamespace{}
	case EgressNetworkPolicies:
		expectedType = &networkapi.EgressNetworkPolicy{}
	case NetworkPolicies:
		expectedType = &extensions.NetworkPolicy{}
	default:
		glog.Fatalf("Unknown resource %s during initialization of event queue", resourceName)
	}

	eventQueue := newEventQueue(client, resourceName, expectedType, metav1.NamespaceAll)
	for {
		eventQueue.Pop(process, expectedType)
	}
}

// RegisterSharedInformerEventHandlers registers addOrUpdateFunc and delFunc event handlers with
// kubernetes shared informers for the given resource name.
func RegisterSharedInformerEventHandlers(kubeInformers kinternalinformers.SharedInformerFactory,
	addOrUpdateFunc func(interface{}, interface{}, watch.EventType),
	delFunc func(interface{}), resourceName ResourceName) {

	var expectedObjType interface{}
	var informer kcache.SharedIndexInformer

	internalVersion := kubeInformers.Core().InternalVersion()

	switch resourceName {
	case Nodes:
		informer = internalVersion.Nodes().Informer()
		expectedObjType = &kapi.Node{}
	case Namespaces:
		informer = internalVersion.Namespaces().Informer()
		expectedObjType = &kapi.Namespace{}
	case Services:
		informer = internalVersion.Services().Informer()
		expectedObjType = &kapi.Service{}
	case Pods:
		informer = internalVersion.Pods().Informer()
		expectedObjType = &kapi.Pod{}
	default:
		glog.Errorf("Unknown resource name: %s", resourceName)
		return
	}

	informer.AddEventHandler(kcache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			addOrUpdateFunc(obj, nil, watch.Added)
		},
		UpdateFunc: func(old, cur interface{}) {
			addOrUpdateFunc(cur, old, watch.Modified)
		},
		DeleteFunc: func(obj interface{}) {
			if reflect.TypeOf(expectedObjType) != reflect.TypeOf(obj) {
				tombstone, ok := obj.(kcache.DeletedFinalStateUnknown)
				if !ok {
					glog.Errorf("Couldn't get object from tombstone: %+v", obj)
					return
				}

				obj = tombstone.Obj
				if reflect.TypeOf(expectedObjType) != reflect.TypeOf(obj) {
					glog.Errorf("Tombstone contained object that is not a %s: %+v", resourceName, obj)
					return
				}
			}
			delFunc(obj)
		},
	})
}
<commit_msg>Fix an error message<commit_after>package common

import (
	"fmt"
	"net"
	"reflect"
	"strings"
	"time"

	"github.com/golang/glog"

	configapi "github.com/openshift/origin/pkg/cmd/server/api"
	networkapi "github.com/openshift/origin/pkg/network/apis/network"
	networkclient "github.com/openshift/origin/pkg/network/generated/internalclientset"
	"github.com/openshift/origin/pkg/util/netutils"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	kerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apimachinery/pkg/watch"
	kcache "k8s.io/client-go/tools/cache"
	kapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/extensions"
	kinternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
)

func HostSubnetToString(subnet *networkapi.HostSubnet) string {
	return fmt.Sprintf("%s (host: %q, ip: %q, subnet: %q)", subnet.Name, subnet.Host, subnet.HostIP, subnet.Subnet)
}

func ClusterNetworkToString(n *networkapi.ClusterNetwork) string {
	return fmt.Sprintf("%s (network: %q, hostSubnetBits: %d, serviceNetwork: %q, pluginName: %q)", n.Name, n.Network, n.HostSubnetLength, n.ServiceNetwork, n.PluginName)
}

func ClusterNetworkListContains(clusterNetworks []ClusterNetwork, ipaddr net.IP) (*net.IPNet, bool) {
	for _, cn := range clusterNetworks {
		if cn.ClusterCIDR.Contains(ipaddr) {
			return cn.ClusterCIDR, true
		}
	}
	return nil, false
}

type NetworkInfo struct {
	ClusterNetworks []ClusterNetwork
	ServiceNetwork  *net.IPNet
}

type ClusterNetwork struct {
	ClusterCIDR      *net.IPNet
	HostSubnetLength uint32
}

func ParseNetworkInfo(clusterNetwork []networkapi.ClusterNetworkEntry, serviceNetwork string) (*NetworkInfo, error) {
	var cns []ClusterNetwork

	for _, entry := range clusterNetwork {
		cidr, err := netutils.ParseCIDRMask(entry.CIDR)
		if err != nil {
			_, cidr, err := net.ParseCIDR(entry.CIDR)
			if err != nil {
				return nil, fmt.Errorf("failed to parse ClusterNetwork CIDR %s: %v", entry.CIDR, err)
			}
			glog.Errorf("Configured clusterNetworks value %q is invalid; treating it as %q", entry.CIDR, cidr.String())
		}
		cns = append(cns, ClusterNetwork{ClusterCIDR: cidr, HostSubnetLength: entry.HostSubnetLength})
	}

	sn, err := netutils.ParseCIDRMask(serviceNetwork)
	if err != nil {
		_, sn, err := net.ParseCIDR(serviceNetwork)
		if err != nil {
			return nil, fmt.Errorf("failed to parse ServiceNetwork CIDR %s: %v", serviceNetwork, err)
		}
		glog.Errorf("Configured serviceNetworkCIDR value %q is invalid; treating it as %q", serviceNetwork, sn.String())
	}

	return &NetworkInfo{
		ClusterNetworks: cns,
		ServiceNetwork:  sn,
	}, nil
}

func (ni *NetworkInfo) ValidateNodeIP(nodeIP string) error {
	if nodeIP == "" || nodeIP == "127.0.0.1" {
		return fmt.Errorf("invalid node IP %q", nodeIP)
	}

	// Ensure each node's NodeIP is not contained by the cluster network,
	// which could cause a routing loop. (rhbz#1295486)
	ipaddr := net.ParseIP(nodeIP)
	if ipaddr == nil {
		return fmt.Errorf("failed to parse node IP %s", nodeIP)
	}

	if conflictingCIDR, found := ClusterNetworkListContains(ni.ClusterNetworks, ipaddr); found {
		return fmt.Errorf("node IP %s conflicts with cluster network %s", nodeIP, conflictingCIDR.String())
	}
	if ni.ServiceNetwork.Contains(ipaddr) {
		return fmt.Errorf("node IP %s conflicts with service network %s", nodeIP, ni.ServiceNetwork.String())
	}

	return nil
}

func (ni *NetworkInfo) CheckHostNetworks(hostIPNets []*net.IPNet) error {
	errList := []error{}
	for _, ipNet := range hostIPNets {
		for _, clusterNetwork := range ni.ClusterNetworks {
			if configapi.CIDRsOverlap(ipNet.String(), clusterNetwork.ClusterCIDR.String()) {
				errList = append(errList, fmt.Errorf("cluster IP: %s conflicts with host network: %s", clusterNetwork.ClusterCIDR.IP.String(), ipNet.String()))
			}
		}
		if configapi.CIDRsOverlap(ipNet.String(), ni.ServiceNetwork.String()) {
			errList = append(errList, fmt.Errorf("service IP: %s conflicts with host network: %s", ni.ServiceNetwork.String(), ipNet.String()))
		}
	}
	return kerrors.NewAggregate(errList)
}

func (ni *NetworkInfo) CheckClusterObjects(subnets []networkapi.HostSubnet, pods []kapi.Pod, services []kapi.Service) error {
	var errList []error

	for _, subnet := range subnets {
		subnetIP, _, _ := net.ParseCIDR(subnet.Subnet)
		if subnetIP == nil {
			errList = append(errList, fmt.Errorf("failed to parse network address: %s", subnet.Subnet))
		} else if _, contains := ClusterNetworkListContains(ni.ClusterNetworks, subnetIP); !contains {
			errList = append(errList, fmt.Errorf("existing node subnet: %s is not part of any cluster network CIDR", subnet.Subnet))
		}
		if len(errList) >= 10 {
			break
		}
	}
	for _, pod := range pods {
		if pod.Spec.SecurityContext != nil && pod.Spec.SecurityContext.HostNetwork {
			continue
		}
		if _, contains := ClusterNetworkListContains(ni.ClusterNetworks, net.ParseIP(pod.Status.PodIP)); !contains && pod.Status.PodIP != "" {
			errList = append(errList, fmt.Errorf("existing pod %s:%s with IP %s is not part of cluster network", pod.Namespace, pod.Name, pod.Status.PodIP))
			if len(errList) >= 10 {
				break
			}
		}
	}
	for _, svc := range services {
		svcIP := net.ParseIP(svc.Spec.ClusterIP)
		if svcIP != nil && !ni.ServiceNetwork.Contains(svcIP) {
			errList = append(errList, fmt.Errorf("existing service %s:%s with IP %s is not part of service network %s", svc.Namespace, svc.Name, svc.Spec.ClusterIP, ni.ServiceNetwork.String()))
			if len(errList) >= 10 {
				break
			}
		}
	}

	if len(errList) >= 10 {
		errList = append(errList, fmt.Errorf("too many errors... truncating"))
	}
	return kerrors.NewAggregate(errList)
}

func GetNetworkInfo(networkClient networkclient.Interface) (*NetworkInfo, error) {
	cn, err := networkClient.Network().ClusterNetworks().Get(networkapi.ClusterNetworkDefault, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}

	return ParseNetworkInfo(cn.ClusterNetworks, cn.ServiceNetwork)
}

type ResourceName string

const (
	Nodes                 ResourceName = "Nodes"
	Namespaces            ResourceName = "Namespaces"
	NetNamespaces         ResourceName = "NetNamespaces"
	Services              ResourceName = "Services"
	HostSubnets           ResourceName = "HostSubnets"
	Pods                  ResourceName = "Pods"
	EgressNetworkPolicies ResourceName = "EgressNetworkPolicies"
	NetworkPolicies       ResourceName = "NetworkPolicies"
)

func newEventQueue(client kcache.Getter, resourceName ResourceName, expectedType interface{}, namespace string) *EventQueue {
	rn := strings.ToLower(string(resourceName))
	lw := kcache.NewListWatchFromClient(client, rn, namespace, fields.Everything())
	eventQueue := NewEventQueue(DeletionHandlingMetaNamespaceKeyFunc)
	// Repopulate event queue every 30 mins
	// Existing items in the event queue will have watch.Modified event type
	kcache.NewReflector(lw, expectedType, eventQueue, 30*time.Minute).Run()
	return eventQueue
}

// Run event queue for the given resource. The 'process' function is called
// repeatedly with each available cache.Delta that describes state changes
// to an object. If the process function returns an error queued changes
// for that object are dropped but processing continues with the next available
// object's cache.Deltas.  The error is logged with call stack information.
//
// NOTE: this function will handle DeletedFinalStateUnknown delta objects
// automatically, which may not always be what you want since the now-deleted
// object may be stale.
func RunEventQueue(client kcache.Getter, resourceName ResourceName, process ProcessEventFunc) {
	var expectedType interface{}

	switch resourceName {
	case HostSubnets:
		expectedType = &networkapi.HostSubnet{}
	case NetNamespaces:
		expectedType = &networkapi.NetNamespace{}
	case EgressNetworkPolicies:
		expectedType = &networkapi.EgressNetworkPolicy{}
	case NetworkPolicies:
		expectedType = &extensions.NetworkPolicy{}
	default:
		glog.Fatalf("Unknown resource %s during initialization of event queue", resourceName)
	}

	eventQueue := newEventQueue(client, resourceName, expectedType, metav1.NamespaceAll)
	for {
		eventQueue.Pop(process, expectedType)
	}
}

// RegisterSharedInformerEventHandlers registers addOrUpdateFunc and delFunc event handlers with
// kubernetes shared informers for the given resource name.
func RegisterSharedInformerEventHandlers(kubeInformers kinternalinformers.SharedInformerFactory,
	addOrUpdateFunc func(interface{}, interface{}, watch.EventType),
	delFunc func(interface{}), resourceName ResourceName) {

	var expectedObjType interface{}
	var informer kcache.SharedIndexInformer

	internalVersion := kubeInformers.Core().InternalVersion()

	switch resourceName {
	case Nodes:
		informer = internalVersion.Nodes().Informer()
		expectedObjType = &kapi.Node{}
	case Namespaces:
		informer = internalVersion.Namespaces().Informer()
		expectedObjType = &kapi.Namespace{}
	case Services:
		informer = internalVersion.Services().Informer()
		expectedObjType = &kapi.Service{}
	case Pods:
		informer = internalVersion.Pods().Informer()
		expectedObjType = &kapi.Pod{}
	default:
		glog.Errorf("Unknown resource name: %s", resourceName)
		return
	}

	informer.AddEventHandler(kcache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			addOrUpdateFunc(obj, nil, watch.Added)
		},
		UpdateFunc: func(old, cur interface{}) {
			addOrUpdateFunc(cur, old, watch.Modified)
		},
		DeleteFunc: func(obj interface{}) {
			if reflect.TypeOf(expectedObjType) != reflect.TypeOf(obj) {
				tombstone, ok := obj.(kcache.DeletedFinalStateUnknown)
				if !ok {
					glog.Errorf("Couldn't get object from tombstone: %+v", obj)
					return
				}

				obj = tombstone.Obj
				if reflect.TypeOf(expectedObjType) != reflect.TypeOf(obj) {
					glog.Errorf("Tombstone contained object that is not a %s: %+v", resourceName, obj)
					return
				}
			}
			delFunc(obj)
		},
	})
}
<|endoftext|>


<commit_before>package audio

import (
	"errors"
	"time"

	"github.com/CyCoreSystems/ari"
	uuid "github.com/satori/go.uuid"
	"golang.org/x/net/context"
)

// AllDTMF is a string which contains all possible
// DTMF digits.
const AllDTMF = "0123456789ABCD*#"

// PlaybackStartTimeout is the time to allow for Asterisk to
// send the PlaybackStarted before giving up.
var PlaybackStartTimeout = 1 * time.Second

// MaxPlaybackTime is the maximum amount of time to allow for
// a playback to complete.
var MaxPlaybackTime = 10 * time.Minute

// Play plays the given media URI
func Play(ctx context.Context, p Player, mediaURI string) (st Status, err error) {
	pb := PlayAsync(ctx, p, mediaURI)

	<-pb.Stopped()

	st, err = pb.Status(), pb.Err()
	return
}

// PlayAsync plays the audio asynchronously and returns a playback object
func PlayAsync(ctx context.Context, p Player, mediaURI string) *Playback {

	var pb Playback

	pb.startCh = make(chan struct{})
	pb.stopCh = make(chan struct{})
	pb.status = InProgress
	pb.err = nil
	pb.ctx, pb.cancel = context.WithCancel(ctx)

	//TODO: confirm whether we need to listen on bridge events if p Player is a bridge
	hangup := p.Subscribe(ari.Events.ChannelHangupRequest, ari.Events.ChannelDestroyed)

	id := uuid.NewV1().String()
	pb.handle, pb.err = p.Play(id, mediaURI)

	// register for events on the ~~playback~~ player handle. This means
	// we have to filter the events using the evnentual playback handle.
	playbackStarted := pb.handle.Subscribe(ari.Events.PlaybackStarted)
	playbackFinished := pb.handle.Subscribe(ari.Events.PlaybackFinished)

	go func() {
		defer func() {
			playbackStarted.Cancel()
			playbackFinished.Cancel()
			hangup.Cancel()
			close(pb.stopCh)
		}()

		// wait to check error here so
		// subscriptions are cleaned up
		if pb.err != nil {
			close(pb.startCh)
			pb.status = Failed
			return
		}

		go func() {
			defer close(pb.startCh)

			for {
				select {
				case <-time.After(PlaybackStartTimeout):
					pb.status = Timeout
					pb.err = errors.New("Timeout waiting for start of playback")
					return
				case <-hangup.Events():
					pb.status = Hangup
					return
				case <-pb.ctx.Done():
					pb.status = Canceled
					pb.err = pb.ctx.Err()
					return
				case evt := <-playbackStarted.Events():
					return
				}
			}
		}()

		<-pb.startCh

		if pb.status != InProgress {
			return
		}

		for {
			select {
			case <-time.After(MaxPlaybackTime):
				pb.status = Timeout
				pb.err = errors.New("Timeout waiting for stop of playback")
				return
			case <-hangup.Events():
				pb.status = Hangup
				return
			case <-pb.ctx.Done():
				pb.status = Canceled
				pb.err = pb.ctx.Err()
				return
			case evt := <-playbackFinished.Events():
				pb.status = Finished
				return
			}
		}
	}()

	return &pb
}
<commit_msg>ext/audio - fix compile errors<commit_after>package audio

import (
	"errors"
	"time"

	"github.com/CyCoreSystems/ari"
	uuid "github.com/satori/go.uuid"
	"golang.org/x/net/context"
)

// AllDTMF is a string which contains all possible
// DTMF digits.
const AllDTMF = "0123456789ABCD*#"

// PlaybackStartTimeout is the time to allow for Asterisk to
// send the PlaybackStarted before giving up.
var PlaybackStartTimeout = 1 * time.Second

// MaxPlaybackTime is the maximum amount of time to allow for
// a playback to complete.
var MaxPlaybackTime = 10 * time.Minute

// Play plays the given media URI
func Play(ctx context.Context, p Player, mediaURI string) (st Status, err error) {
	pb := PlayAsync(ctx, p, mediaURI)

	<-pb.Stopped()

	st, err = pb.Status(), pb.Err()
	return
}

// PlayAsync plays the audio asynchronously and returns a playback object
func PlayAsync(ctx context.Context, p Player, mediaURI string) *Playback {

	var pb Playback

	pb.startCh = make(chan struct{})
	pb.stopCh = make(chan struct{})
	pb.status = InProgress
	pb.err = nil
	pb.ctx, pb.cancel = context.WithCancel(ctx)

	//TODO: confirm whether we need to listen on bridge events if p Player is a bridge
	hangup := p.Subscribe(ari.Events.ChannelHangupRequest, ari.Events.ChannelDestroyed)

	id := uuid.NewV1().String()
	pb.handle, pb.err = p.Play(id, mediaURI)

	// register for events on the ~~playback~~ player handle. This means
	// we have to filter the events using the evnentual playback handle.
	playbackStarted := pb.handle.Subscribe(ari.Events.PlaybackStarted)
	playbackFinished := pb.handle.Subscribe(ari.Events.PlaybackFinished)

	go func() {
		defer func() {
			playbackStarted.Cancel()
			playbackFinished.Cancel()
			hangup.Cancel()
			close(pb.stopCh)
		}()

		// wait to check error here so
		// subscriptions are cleaned up
		if pb.err != nil {
			close(pb.startCh)
			pb.status = Failed
			return
		}

		go func() {
			defer close(pb.startCh)

			for {
				select {
				case <-time.After(PlaybackStartTimeout):
					pb.status = Timeout
					pb.err = errors.New("Timeout waiting for start of playback")
					return
				case <-hangup.Events():
					pb.status = Hangup
					return
				case <-pb.ctx.Done():
					pb.status = Canceled
					pb.err = pb.ctx.Err()
					return
				case <-playbackStarted.Events():
					return
				}
			}
		}()

		<-pb.startCh

		if pb.status != InProgress {
			return
		}

		for {
			select {
			case <-time.After(MaxPlaybackTime):
				pb.status = Timeout
				pb.err = errors.New("Timeout waiting for stop of playback")
				return
			case <-hangup.Events():
				pb.status = Hangup
				return
			case <-pb.ctx.Done():
				pb.status = Canceled
				pb.err = pb.ctx.Err()
				return
			case <-playbackFinished.Events():
				pb.status = Finished
				return
			}
		}
	}()

	return &pb
}
<|endoftext|>


<commit_before>package proxy

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestNewDefinitions(t *testing.T) {
	definition := NewDefinition()
	assert.NotNil(t, definition)
}

func TestSuccessfulValidation(t *testing.T) {
	definition := Definition{
		ListenPath:  "/*",
		UpstreamURL: "http://test.com",
	}
	isValid, err := definition.Validate()

	assert.NoError(t, err)
	assert.True(t, isValid)
}

func TestEmptyListenPathValidation(t *testing.T) {
	definition := Definition{}
	isValid, err := definition.Validate()

	assert.Error(t, err)
	assert.False(t, isValid)
}

func TestInvalidTargetURLValidation(t *testing.T) {
	definition := Definition{
		ListenPath:  " ",
		UpstreamURL: "wrong",
	}
	isValid, err := definition.Validate()

	assert.Error(t, err)
	assert.False(t, isValid)
}

func TestRouteToJSON(t *testing.T) {
	definition := NewDefinition()
	route := NewRoute(definition)
	json, err := route.JSONMarshal()
	assert.NoError(t, err)
	assert.JSONEq(
		t,
		`{"proxy": {"insecure_skip_verify": false, "append_path":false, "enable_load_balancing":false, "methods":[], "hosts":[], "preserve_host":false, "listen_path":"", "upstream_url":"", "strip_path":false, "upstreams": {"balancing": "", "targets": [] }}}`,
		string(json),
	)
}

func TestJSONToRoute(t *testing.T) {
	route, err := JSONUnmarshalRoute([]byte(`{"proxy": {"insecure_skip_verify": false, "append_path":false, "enable_load_balancing":false, "methods":[], "hosts":[], "preserve_host":false, "listen_path":"", "upstream_url":"/*", "strip_path":false}}`))

	assert.NoError(t, err)
	assert.IsType(t, &Route{}, route)
}

func TestJSONToRouteError(t *testing.T) {
	_, err := JSONUnmarshalRoute([]byte{})

	assert.Error(t, err)
}
<commit_msg>Add tests for proxy definition<commit_after>package proxy

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestDefinition(t *testing.T) {
	t.Parallel()

	tests := []struct {
		scenario string
		function func(*testing.T)
	}{
		{
			scenario: "new definitions",
			function: testNewDefinitions,
		},
		{
			scenario: "successful validation",
			function: testSuccessfulValidation,
		},
		{
			scenario: "empty listen path validation",
			function: testEmptyListenPathValidation,
		},
		{
			scenario: "invalid target url validation",
			function: testInvalidTargetURLValidation,
		},
		{
			scenario: "route to json",
			function: testRouteToJSON,
		},
		{
			scenario: "json to route",
			function: testJSONToRoute,
		},
		{
			scenario: "is balancer defined",
			function: testIsBalancerDefined,
		},
	}

	for _, test := range tests {
		t.Run(test.scenario, func(t *testing.T) {
			test.function(t)
		})
	}
}

func testNewDefinitions(t *testing.T) {
	definition := NewDefinition()
	assert.NotNil(t, definition)
}

func testSuccessfulValidation(t *testing.T) {
	definition := Definition{
		ListenPath:  "/*",
		UpstreamURL: "http://test.com",
	}
	isValid, err := definition.Validate()

	assert.NoError(t, err)
	assert.True(t, isValid)
}

func testEmptyListenPathValidation(t *testing.T) {
	definition := Definition{}
	isValid, err := definition.Validate()

	assert.Error(t, err)
	assert.False(t, isValid)
}

func testInvalidTargetURLValidation(t *testing.T) {
	definition := Definition{
		ListenPath:  " ",
		UpstreamURL: "wrong",
	}
	isValid, err := definition.Validate()

	assert.Error(t, err)
	assert.False(t, isValid)
}

func testRouteToJSON(t *testing.T) {
	expectedJSON := `
	{
		"proxy":{
			"insecure_skip_verify":false,
			"append_path":false,
			"enable_load_balancing":false,
			"methods":[

			],
			"hosts":[

			],
			"preserve_host":false,
			"listen_path":"",
			"upstream_url":"",
			"strip_path":false,
			"upstreams":{
				"balancing":"",
				"targets":[

				]
			}
		}
	}
	`
	definition := NewDefinition()
	route := NewRoute(definition)
	json, err := route.JSONMarshal()

	assert.NoError(t, err)
	assert.JSONEq(t, expectedJSON, string(json))
}

func testJSONToRoute(t *testing.T) {
	route, err := JSONUnmarshalRoute([]byte(`
	{
		"proxy":{
			"insecure_skip_verify":false,
			"append_path":false,
			"enable_load_balancing":false,
			"methods":[],
			"hosts":[],
			"preserve_host":false,
			"listen_path":"",
			"upstream_url":"/*",
			"strip_path":false
		}
	}`))

	assert.NoError(t, err)
	assert.IsType(t, &Route{}, route)
}

func testJSONToRouteError(t *testing.T) {
	_, err := JSONUnmarshalRoute([]byte{})
	assert.Error(t, err)
}

func testIsBalancerDefined(t *testing.T) {
	definition := NewDefinition()
	assert.False(t, definition.IsBalancerDefined())

	target := &Target{Target: "http://localhost:8080/api-name"}
	definition.Upstreams.Targets = append(definition.Upstreams.Targets, target)
	assert.True(t, definition.IsBalancerDefined())
}
<|endoftext|>


<commit_before>// +build linux

package ipvs

import (
	"net"
	"syscall"

	"github.com/vishvananda/netlink/nl"
	"github.com/vishvananda/netns"
)

// Service defines an IPVS service in its entirety.
type Service struct {
	// Virtual service address.
	Address  net.IP
	Protocol uint16
	Port     uint16
	FWMark   uint32 // Firewall mark of the service.

	// Virtual service options.
	SchedName     string
	Flags         uint32
	Timeout       uint32
	Netmask       uint32
	AddressFamily uint16
	PEName        string
}

// Destination defines an IPVS destination (real server) in its
// entirety.
type Destination struct {
	Address         net.IP
	Port            uint16
	Weight          int
	ConnectionFlags uint32
	AddressFamily   uint16
	UpperThreshold  uint32
	LowerThreshold  uint32
}

// Handle provides a namespace specific ipvs handle to program ipvs
// rules.
type Handle struct {
	sock *nl.NetlinkSocket
}

// New provides a new ipvs handle in the namespace pointed to by the
// passed path. It will return a valid handle or an error in case an
// error occured while creating the handle.
func New(path string) (*Handle, error) {
	setup()

	n := netns.None()
	if path != "" {
		var err error
		n, err = netns.GetFromPath(path)
		if err != nil {
			return nil, err
		}
	}

	sock, err := nl.GetNetlinkSocketAt(n, netns.None(), syscall.NETLINK_GENERIC)
	if err != nil {
		n.Close()
		return nil, err
	}

	return &Handle{sock: sock}, nil
}

// Close closes the ipvs handle. The handle is invalid after Close
// returns.
func (i *Handle) Close() {
	if i.sock != nil {
		i.sock.Close()
	}
}

// NewService creates a new ipvs service in the passed handle.
func (i *Handle) NewService(s *Service) error {
	return i.doCmd(s, nil, ipvsCmdNewService)
}

// UpdateService updates an already existing service in the passed
// handle.
func (i *Handle) UpdateService(s *Service) error {
	return i.doCmd(s, nil, ipvsCmdSetService)
}

// DelService deletes an already existing service in the passed
// handle.
func (i *Handle) DelService(s *Service) error {
	return i.doCmd(s, nil, ipvsCmdDelService)
}

// NewDestination creates an new real server in the passed ipvs
// service which should already be existing in the passed handle.
func (i *Handle) NewDestination(s *Service, d *Destination) error {
	return i.doCmd(s, d, ipvsCmdNewDest)
}

// UpdateDestination updates an already existing real server in the
// passed ipvs service in the passed handle.
func (i *Handle) UpdateDestination(s *Service, d *Destination) error {
	return i.doCmd(s, d, ipvsCmdSetDest)
}

// DelDestination deletes an already existing real server in the
// passed ipvs service in the passed handle.
func (i *Handle) DelDestination(s *Service, d *Destination) error {
	return i.doCmd(s, d, ipvsCmdDelDest)
}
<commit_msg>Make sure to close nshandle in ipvs.New<commit_after>// +build linux

package ipvs

import (
	"net"
	"syscall"

	"github.com/vishvananda/netlink/nl"
	"github.com/vishvananda/netns"
)

// Service defines an IPVS service in its entirety.
type Service struct {
	// Virtual service address.
	Address  net.IP
	Protocol uint16
	Port     uint16
	FWMark   uint32 // Firewall mark of the service.

	// Virtual service options.
	SchedName     string
	Flags         uint32
	Timeout       uint32
	Netmask       uint32
	AddressFamily uint16
	PEName        string
}

// Destination defines an IPVS destination (real server) in its
// entirety.
type Destination struct {
	Address         net.IP
	Port            uint16
	Weight          int
	ConnectionFlags uint32
	AddressFamily   uint16
	UpperThreshold  uint32
	LowerThreshold  uint32
}

// Handle provides a namespace specific ipvs handle to program ipvs
// rules.
type Handle struct {
	sock *nl.NetlinkSocket
}

// New provides a new ipvs handle in the namespace pointed to by the
// passed path. It will return a valid handle or an error in case an
// error occured while creating the handle.
func New(path string) (*Handle, error) {
	setup()

	n := netns.None()
	if path != "" {
		var err error
		n, err = netns.GetFromPath(path)
		if err != nil {
			return nil, err
		}
	}
	defer n.Close()

	sock, err := nl.GetNetlinkSocketAt(n, netns.None(), syscall.NETLINK_GENERIC)
	if err != nil {
		return nil, err
	}

	return &Handle{sock: sock}, nil
}

// Close closes the ipvs handle. The handle is invalid after Close
// returns.
func (i *Handle) Close() {
	if i.sock != nil {
		i.sock.Close()
	}
}

// NewService creates a new ipvs service in the passed handle.
func (i *Handle) NewService(s *Service) error {
	return i.doCmd(s, nil, ipvsCmdNewService)
}

// UpdateService updates an already existing service in the passed
// handle.
func (i *Handle) UpdateService(s *Service) error {
	return i.doCmd(s, nil, ipvsCmdSetService)
}

// DelService deletes an already existing service in the passed
// handle.
func (i *Handle) DelService(s *Service) error {
	return i.doCmd(s, nil, ipvsCmdDelService)
}

// NewDestination creates an new real server in the passed ipvs
// service which should already be existing in the passed handle.
func (i *Handle) NewDestination(s *Service, d *Destination) error {
	return i.doCmd(s, d, ipvsCmdNewDest)
}

// UpdateDestination updates an already existing real server in the
// passed ipvs service in the passed handle.
func (i *Handle) UpdateDestination(s *Service, d *Destination) error {
	return i.doCmd(s, d, ipvsCmdSetDest)
}

// DelDestination deletes an already existing real server in the
// passed ipvs service in the passed handle.
func (i *Handle) DelDestination(s *Service, d *Destination) error {
	return i.doCmd(s, d, ipvsCmdDelDest)
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"strings"

	"runtime"

	"github.com/fatih/color"
	"github.com/kataras/cli"
	"github.com/kataras/iris"
	"github.com/kataras/iris/utils"
)

const (
	// PackagesURL the url to download all the packages
	PackagesURL = "https://github.com/iris-contrib/iris-command-assets/archive/master.zip"
	// PackagesExportedName the folder created after unzip
	PackagesExportedName = "iris-command-assets-master"
)

var (
	app *cli.App
	// SuccessPrint prints with a green color
	SuccessPrint = color.New(color.FgGreen).Add(color.Bold).PrintfFunc()
	// InfoPrint prints with the cyan color
	InfoPrint          = color.New(color.FgHiCyan).Add(color.Bold).PrintfFunc()
	packagesInstallDir = os.Getenv("GOPATH") + utils.PathSeparator + "src" + utils.PathSeparator + "github.com" + utils.PathSeparator + "kataras" + utils.PathSeparator + "iris" + utils.PathSeparator + "iris" + utils.PathSeparator
	packagesDir        = packagesInstallDir + PackagesExportedName + utils.PathSeparator
)

func init() {
	app = cli.NewApp("iris", "Command line tool for Iris web framework", "0.0.2")
	app.Command(cli.Command("version", "\t      prints your iris version").Action(func(cli.Flags) error { app.Printf("%s", iris.Version); return nil }))

	createCmd := cli.Command("create", "create a project to a given directory").
		Flag("dir", "./", "-d ./ creates an iris starter kit to the current directory").
		Flag("type", "basic", "creates the project based on the -t package. Currently, available types are 'basic' & 'static'").
		Action(create)

	app.Command(createCmd)
}

func main() {
	app.Run(func(cli.Flags) error { return nil })
}

func create(flags cli.Flags) (err error) {

	/*	///TODO: add a version.json and check if the repository's version is bigger than local and then do the downloadPackages.

		if !utils.DirectoryExists(packagesDir) {
			downloadPackages()
		}
	*/

	// currently: always download packages, because I'm adding new things to the packages every day.
	downloadPackages()

	targetDir := flags.String("dir")

	if strings.HasPrefix(targetDir, "./") || strings.HasPrefix(targetDir, "."+utils.PathSeparator) {
		currentWdir, err := os.Getwd()
		if err != nil {
			return err
		}
		targetDir = currentWdir + utils.PathSeparator + targetDir[2:]
	}

	createPackage(flags.String("type"), targetDir)
	return
}

func downloadPackages() {
	_, err := utils.Install(PackagesURL, packagesInstallDir)
	if err != nil {
		app.Printf("\nProblem while downloading the assets from the internet for the first time. Trace: %s", err.Error())
	}
}

func createPackage(packageName string, targetDir string) error {
	packageDir := packagesDir + packageName
	err := utils.CopyDir(packageDir, targetDir)
	if err != nil {
		app.Printf("\nProblem while copying the %s package to the %s. Trace: %s", packageName, targetDir, err.Error())
		return err
	}

	InfoPrint("\n%s package was installed successfully", packageName)

	// build & run the server

	// go build
	buildCmd := utils.CommandBuilder("go", "build")
	if targetDir[len(targetDir)-1] != os.PathSeparator || targetDir[len(targetDir)-1] != '/' {
		targetDir += utils.PathSeparator
	}
	buildCmd.Dir = targetDir + "backend"
	buildCmd.Stderr = os.Stderr
	err = buildCmd.Start()
	if err != nil {
		app.Printf("\n Failed to build the %s package. Trace: %s", packageName, err.Error())
	}
	buildCmd.Wait()
	print("\n\n")

	// run backend/backend(.exe)
	executable := "backend"
	if runtime.GOOS == "windows" {
		executable += ".exe"
	}

	runCmd := utils.CommandBuilder("." + utils.PathSeparator + executable)
	runCmd.Dir = buildCmd.Dir
	runCmd.Stdout = os.Stdout
	runCmd.Stderr = os.Stderr

	err = runCmd.Start()
	if err != nil {
		app.Printf("\n Failed to run the %s package. Trace: %s", packageName, err.Error())
	}
	runCmd.Wait()

	return err
}
<commit_msg>iris cmd: some untested changes in order the samples' import paths works better<commit_after>package main

import (
	"os"

	"strings"

	"runtime"

	"github.com/fatih/color"
	"github.com/kataras/cli"
	"github.com/kataras/iris"
	"github.com/kataras/iris/utils"
)

const (
	// PackagesURL the url to download all the packages
	PackagesURL = "https://github.com/iris-contrib/iris-command-assets/archive/master.zip"
	// PackagesExportedName the folder created after unzip
	PackagesExportedName = "iris-command-assets-master"
)

var (
	app *cli.App
	// SuccessPrint prints with a green color
	SuccessPrint = color.New(color.FgGreen).Add(color.Bold).PrintfFunc()
	// InfoPrint prints with the cyan color
	InfoPrint          = color.New(color.FgHiCyan).Add(color.Bold).PrintfFunc()
	packagesInstallDir = os.Getenv("GOPATH") + utils.PathSeparator + "src" + utils.PathSeparator + "github.com" + utils.PathSeparator + "iris-contrib" + utils.PathSeparator + "iris-command-assets" + utils.PathSeparator
)

func init() {
	app = cli.NewApp("iris", "Command line tool for Iris web framework", "0.0.2")
	app.Command(cli.Command("version", "\t      prints your iris version").Action(func(cli.Flags) error { app.Printf("%s", iris.Version); return nil }))

	createCmd := cli.Command("create", "create a project to a given directory").
		Flag("dev", true, "set to false to disable the packages download on each create command").
		Flag("dir", "./", "creates an iris starter kit to the current directory").
		Flag("type", "basic", "creates the project based on the -t package. Currently, available types are 'basic' & 'static'").
		Action(create)

	app.Command(createCmd)
}

func main() {
	app.Run(func(cli.Flags) error { return nil })
}

func create(flags cli.Flags) (err error) {

	if !utils.DirectoryExists(packagesInstallDir) || flags.Bool("dev") {
		downloadPackages()
	}

	targetDir := flags.String("dir")

	if strings.HasPrefix(targetDir, "./") || strings.HasPrefix(targetDir, "."+utils.PathSeparator) {
		currentWdir, err := os.Getwd()
		if err != nil {
			return err
		}
		targetDir = currentWdir + utils.PathSeparator + targetDir[2:]
	}

	createPackage(flags.String("type"), targetDir)
	return
}

func downloadPackages() {
	installedDir, err := utils.Install(PackagesURL, packagesInstallDir)
	if err != nil {
		app.Printf("\nProblem while downloading the assets from the internet for the first time. Trace: %s", err.Error())
		return
	}
	// installedDir is the packagesInstallDir+PackagesExportedName, we will copy these contents to the parent, to the packagesInstallDir, because of import paths.

	err = utils.CopyDir(installedDir, packagesInstallDir)
	if err != nil {
		app.Printf("\nProblem while downloading the assets from the internet for the first time. Trace: %s", err.Error())
		return
	}

	// try to remove the unzipped folder
	utils.RemoveFile(installedDir)
}

func createPackage(packageName string, targetDir string) error {
	packageDir := packagesInstallDir + utils.PathSeparator + packageName
	err := utils.CopyDir(packageDir, targetDir)
	if err != nil {
		app.Printf("\nProblem while copying the %s package to the %s. Trace: %s", packageName, targetDir, err.Error())
		return err
	}

	InfoPrint("\n%s package was installed successfully", packageName)

	// build & run the server

	// go build
	buildCmd := utils.CommandBuilder("go", "build")
	if targetDir[len(targetDir)-1] != os.PathSeparator || targetDir[len(targetDir)-1] != '/' {
		targetDir += utils.PathSeparator
	}
	buildCmd.Dir = targetDir + "backend"
	buildCmd.Stderr = os.Stderr
	err = buildCmd.Start()
	if err != nil {
		app.Printf("\n Failed to build the %s package. Trace: %s", packageName, err.Error())
	}
	buildCmd.Wait()
	print("\n\n")

	// run backend/backend(.exe)
	executable := "backend"
	if runtime.GOOS == "windows" {
		executable += ".exe"
	}

	runCmd := utils.CommandBuilder("." + utils.PathSeparator + executable)
	runCmd.Dir = buildCmd.Dir
	runCmd.Stdout = os.Stdout
	runCmd.Stderr = os.Stderr

	err = runCmd.Start()
	if err != nil {
		app.Printf("\n Failed to run the %s package. Trace: %s", packageName, err.Error())
	}
	runCmd.Wait()

	return err
}
<|endoftext|>


<commit_before>package sync

import (
	"context"
	"encoding/json"
	"strings"
	"sync"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/pkg/errors"

	"github.com/micromdm/micromdm/dep"
	conf "github.com/micromdm/micromdm/platform/config"
	"github.com/micromdm/micromdm/platform/pubsub"
)

const (
	SyncTopic = "mdm.DepSync"

	syncDuration        = 30 * time.Minute
	cursorValidDuration = 7 * 24 * time.Hour
)

type Syncer interface{ SyncNow() }

type WatcherDB interface {
	LoadCursor() (*Cursor, error)
	SaveCursor(c Cursor) error
	LoadAutoAssigners() ([]AutoAssigner, error)
}

type Watcher struct {
	mtx    sync.RWMutex
	logger log.Logger
	client Client

	publisher pubsub.Publisher
	db        WatcherDB
	startSync chan bool
	syncNow   chan bool

	cursor Cursor
}

func NewWatcher(db WatcherDB, pub pubsub.PublishSubscriber, opts ...Option) (*Watcher, error) {
	w := Watcher{
		logger:    log.NewNopLogger(),
		db:        db,
		publisher: pub,
		startSync: make(chan bool),
		syncNow:   make(chan bool),
	}
	for _, optFn := range opts {
		optFn(&w)
	}

	cursor, err := w.db.LoadCursor()
	if err != nil {
		return nil, err
	}
	if cursor.Valid() {
		level.Debug(w.logger).Log("msg", "loaded DEP config", "cursor", cursor.Value)
		w.cursor = *cursor
	}

	if err := w.updateClient(pub); err != nil {
		return nil, err
	}

	saveCursor := func() {
		if err := db.SaveCursor(w.cursor); err != nil {
			level.Info(w.logger).Log("err", err, "msg", "saving cursor")
			return
		}
		level.Info(w.logger).Log("msg", "saved DEP config", "cursor", w.cursor.Value)
	}

	go func() {
		defer saveCursor()
		if w.client == nil {
			// block until we have a DEP client to start sync process
			level.Info(w.logger).Log("msg", "waiting for DEP token to be added before starting sync")
			<-w.startSync
		}
		err := w.Run()
		// the DEP sync should never end without an error, but log
		// unconditionally anyway so we never silently stop watching
		level.Info(w.logger).Log("err", err, "msg", "DEP watcher stopped")
	}()

	return &w, nil
}

type Client interface {
	FetchDevices(...dep.DeviceRequestOption) (*dep.DeviceResponse, error)
	SyncDevices(string, ...dep.DeviceRequestOption) (*dep.DeviceResponse, error)
	AssignProfile(string, ...string) (*dep.ProfileResponse, error)
}

type Option func(*Watcher)

func WithClient(client Client) Option {
	return func(w *Watcher) {
		w.client = client
	}
}

func WithLogger(logger log.Logger) Option {
	return func(w *Watcher) {
		w.logger = logger
	}
}

func (w *Watcher) updateClient(pubsub pubsub.Subscriber) error {
	tokenAdded, err := pubsub.Subscribe(context.TODO(), "token-events", conf.DEPTokenTopic)
	if err != nil {
		return err
	}

	go func() {
		for {
			select {
			case event := <-tokenAdded:
				var token conf.DEPToken
				if err := json.Unmarshal(event.Message, &token); err != nil {
					level.Info(w.logger).Log("err", err, "msg", "unmarshalling tokenAdd to token")
					continue
				}

				client, err := token.Client()
				if err != nil {
					level.Info(w.logger).Log("err", err, "msg", "creating new DEP client")
					continue
				}

				w.mtx.Lock()
				w.client = client
				w.mtx.Unlock()
				go func() { w.startSync <- true }() // unblock Run
			}
		}
	}()
	return nil
}

func (w *Watcher) SyncNow() {
	if w.client == nil {
		level.Info(w.logger).Log("msg", "waiting for DEP token to be added before starting sync")
		return
	}
	w.syncNow <- true
}

// TODO this needs to be a proper error in the micromdm/dep package.
func isCursorExhausted(err error) bool {
	return strings.Contains(err.Error(), "EXHAUSTED_CURSOR")
}

func isCursorExpired(err error) bool {
	return strings.Contains(err.Error(), "EXPIRED_CURSOR")
}

func isCursorInvalid(err error) bool {
	return strings.Contains(err.Error(), "INVALID_CURSOR")
}

// Process DEP messages and pull out filter-matching serial numbers
// associated to profile UUIDs for auto-assignment.
func (w *Watcher) filteredAutoAssignments(devices []dep.Device) (map[string][]string, error) {
	// load auto-assigners every run to make sure we get the latest set of
	// auto-assigner profile UUIDs/filters. Note this makes every *watcher
	// (i.e. every DEP sync instance) share the current DB set of auto-
	// assigners. perhaps to refactor to be more separated.
	assigners, err := w.db.LoadAutoAssigners()
	if err != nil {
		return nil, err
	}
	assigned := make(map[string][]string)
	// skip looping over serials if we have no autoassigners
	if len(assigners) < 1 {
		return assigned, nil
	}
	for _, d := range devices {
		// only process DEP "added" OpType messages
		if d.OpType != "added" {
			continue
		}
		// filter our devices by our assigner filters and get list of
		// which devices are to be assigned to which profiles
		for _, assigner := range assigners {
			if assigner.Filter == "*" { // only supported filter type right now
				if serials, ok := assigned[assigner.ProfileUUID]; ok {
					assigned[assigner.ProfileUUID] = append(serials, d.SerialNumber)
				} else {
					assigned[assigner.ProfileUUID] = []string{d.SerialNumber}
				}
			}
		}

	}
	return assigned, nil
}

func (w *Watcher) processAutoAssign(devices []dep.Device) error {
	assignments, err := w.filteredAutoAssignments(devices)
	if err != nil {
		return err
	}

	for profileUUID, serials := range assignments {
		resp, err := w.client.AssignProfile(profileUUID, serials...)
		if err != nil {
			level.Info(w.logger).Log(
				"err", err,
				"msg", "auto-assign error assigning serials to profile",
				"profile", profileUUID,
			)
			continue
		}
		// count our results for logging
		resultCounts := map[string]int{
			"SUCCESS":        0,
			"NOT_ACCESSIBLE": 0,
			"FAILED":         0,
		}
		for _, result := range resp.Devices {
			if ct, ok := resultCounts[result]; ok {
				// NOTE: we're logging _only_ the above pre-defined result types
				resultCounts[result] = ct + 1
			}
		}
		// TODO: alternate strategy is to log all failed devices
		// TODO: handle/requeue failed devices?
		level.Info(w.logger).Log(
			"msg", "DEP auto-assigned",
			"profile", profileUUID,
			"success", resultCounts["SUCCESS"],
			"not_accessible", resultCounts["NOT_ACCESSIBLE"],
			"failed", resultCounts["FAILED"],
		)
	}

	return nil
}

func (w *Watcher) publishAndProcessDevices(devices []dep.Device) error {
	e := NewEvent(devices)
	data, err := MarshalEvent(e)
	if err != nil {
		return err
	}
	err = w.publisher.Publish(context.TODO(), SyncTopic, data)
	if err != nil {
		return err
	}

	// TODO: instead of directly kicking off the auto-assigner process
	// consider placing a subscriber on the DEP pubsub topic. The same
	// information gets marshalled but it allows us the future
	// flexibility to separate out that component if we desired.
	go func() {
		err := w.processAutoAssign(devices)
		if err != nil {
			level.Info(w.logger).Log("err", err, "msg", "auto-assign error")
		}
	}()
	return nil
}

func (w *Watcher) Run() error {
	var (
		err       error
		resp      *dep.DeviceResponse
		fetchNext = true
		// for logging
		fetchNextLabel = map[bool]string{
			true:  "fetch",
			false: "sync",
		}
	)

	ticker := time.NewTicker(syncDuration).C
	for {
		if fetchNext {
			resp, err = w.client.FetchDevices(dep.Limit(100), dep.Cursor(w.cursor.Value))
			if err != nil && isCursorExhausted(err) {
				level.Info(w.logger).Log(
					"msg", "DEP cursor returned all devices previously",
					"phase", fetchNextLabel[fetchNext],
					"cursor", w.cursor.Value,
				)
				fetchNext = false
				continue
			}
		} else {
			resp, err = w.client.SyncDevices(w.cursor.Value)
		}

		if err != nil && (isCursorExpired(err) || isCursorInvalid(err)) {
			level.Info(w.logger).Log(
				"msg", "DEP cursor error, retrying with empty cursor",
				"phase", fetchNextLabel[fetchNext],
				"cursor", w.cursor.Value,
				"err", err,
			)
			w.cursor.Value = ""
			fetchNext = true
			continue
		} else if err != nil {
			// log any other error, but do not return from the run loop.
			// probably just a transient network issue.
			level.Info(w.logger).Log(
				"msg", "error syncing DEP devices",
				"phase", fetchNextLabel[fetchNext],
				"cursor", w.cursor.Value,
				"err", err,
			)
		} else {
			level.Info(w.logger).Log(
				"msg", "DEP sync",
				"phase", fetchNextLabel[fetchNext],
				"cursor", resp.Cursor,
				"fetched", resp.FetchedUntil,
				"devices", len(resp.Devices),
				"more", resp.MoreToFollow,
			)

			w.cursor = Cursor{Value: resp.Cursor, CreatedAt: time.Now()}
			if err := w.db.SaveCursor(w.cursor); err != nil {
				return errors.Wrap(err, "saving cursor from fetch")
			}
			if err := w.publishAndProcessDevices(resp.Devices); err != nil {
				return err
			}

			if resp.MoreToFollow {
				continue
			} else if fetchNext {
				fetchNext = false
				continue
			}
		}

		select {
		case <-ticker:
		case <-w.syncNow:
			level.Info(w.logger).Log("msg", "explicit DEP sync requested")
		}
	}
}
<commit_msg>Process devices before we sync (#779)<commit_after>package sync

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/pkg/errors"

	"github.com/micromdm/micromdm/dep"
	conf "github.com/micromdm/micromdm/platform/config"
	"github.com/micromdm/micromdm/platform/pubsub"
)

const (
	SyncTopic = "mdm.DepSync"

	syncDuration        = 30 * time.Minute
	cursorValidDuration = 7 * 24 * time.Hour
)

type Syncer interface{ SyncNow() }

type WatcherDB interface {
	LoadCursor() (*Cursor, error)
	SaveCursor(c Cursor) error
	LoadAutoAssigners() ([]AutoAssigner, error)
}

type Watcher struct {
	mtx    sync.RWMutex
	logger log.Logger
	client Client

	publisher pubsub.Publisher
	db        WatcherDB
	startSync chan bool
	syncNow   chan bool

	cursor Cursor
}

func NewWatcher(db WatcherDB, pub pubsub.PublishSubscriber, opts ...Option) (*Watcher, error) {
	w := Watcher{
		logger:    log.NewNopLogger(),
		db:        db,
		publisher: pub,
		startSync: make(chan bool),
		syncNow:   make(chan bool),
	}
	for _, optFn := range opts {
		optFn(&w)
	}

	cursor, err := w.db.LoadCursor()
	if err != nil {
		return nil, err
	}
	if cursor.Valid() {
		level.Debug(w.logger).Log("msg", "loaded DEP config", "cursor", cursor.Value)
		w.cursor = *cursor
	}

	if err := w.updateClient(pub); err != nil {
		return nil, err
	}

	saveCursor := func() {
		if err := db.SaveCursor(w.cursor); err != nil {
			level.Info(w.logger).Log("err", err, "msg", "saving cursor")
			return
		}
		level.Info(w.logger).Log("msg", "saved DEP config", "cursor", w.cursor.Value)
	}

	go func() {
		defer saveCursor()
		if w.client == nil {
			// block until we have a DEP client to start sync process
			level.Info(w.logger).Log("msg", "waiting for DEP token to be added before starting sync")
			<-w.startSync
		}
		err := w.Run()
		// the DEP sync should never end without an error, but log
		// unconditionally anyway so we never silently stop watching
		level.Info(w.logger).Log("err", err, "msg", "DEP watcher stopped")
	}()

	return &w, nil
}

type Client interface {
	FetchDevices(...dep.DeviceRequestOption) (*dep.DeviceResponse, error)
	SyncDevices(string, ...dep.DeviceRequestOption) (*dep.DeviceResponse, error)
	AssignProfile(string, ...string) (*dep.ProfileResponse, error)
}

type Option func(*Watcher)

func WithClient(client Client) Option {
	return func(w *Watcher) {
		w.client = client
	}
}

func WithLogger(logger log.Logger) Option {
	return func(w *Watcher) {
		w.logger = logger
	}
}

func (w *Watcher) updateClient(pubsub pubsub.Subscriber) error {
	tokenAdded, err := pubsub.Subscribe(context.TODO(), "token-events", conf.DEPTokenTopic)
	if err != nil {
		return err
	}

	go func() {
		for {
			select {
			case event := <-tokenAdded:
				var token conf.DEPToken
				if err := json.Unmarshal(event.Message, &token); err != nil {
					level.Info(w.logger).Log("err", err, "msg", "unmarshalling tokenAdd to token")
					continue
				}

				client, err := token.Client()
				if err != nil {
					level.Info(w.logger).Log("err", err, "msg", "creating new DEP client")
					continue
				}

				w.mtx.Lock()
				w.client = client
				w.mtx.Unlock()
				go func() { w.startSync <- true }() // unblock Run
			}
		}
	}()
	return nil
}

func (w *Watcher) SyncNow() {
	if w.client == nil {
		level.Info(w.logger).Log("msg", "waiting for DEP token to be added before starting sync")
		return
	}
	w.syncNow <- true
}

// TODO this needs to be a proper error in the micromdm/dep package.
func isCursorExhausted(err error) bool {
	return strings.Contains(err.Error(), "EXHAUSTED_CURSOR")
}

func isCursorExpired(err error) bool {
	return strings.Contains(err.Error(), "EXPIRED_CURSOR")
}

func isCursorInvalid(err error) bool {
	return strings.Contains(err.Error(), "INVALID_CURSOR")
}

// Process DEP messages and pull out filter-matching serial numbers
// associated to profile UUIDs for auto-assignment.
func (w *Watcher) filteredAutoAssignments(devices []dep.Device) (map[string][]string, error) {
	// load auto-assigners every run to make sure we get the latest set of
	// auto-assigner profile UUIDs/filters. Note this makes every *watcher
	// (i.e. every DEP sync instance) share the current DB set of auto-
	// assigners. perhaps to refactor to be more separated.
	assigners, err := w.db.LoadAutoAssigners()
	if err != nil {
		return nil, err
	}
	assigned := make(map[string][]string)
	// skip looping over serials if we have no autoassigners
	if len(assigners) < 1 {
		return assigned, nil
	}
	for _, d := range devices {
		// only process DEP "added" OpType messages
		if d.OpType != "added" {
			continue
		}
		// filter our devices by our assigner filters and get list of
		// which devices are to be assigned to which profiles
		for _, assigner := range assigners {
			if assigner.Filter == "*" { // only supported filter type right now
				if serials, ok := assigned[assigner.ProfileUUID]; ok {
					assigned[assigner.ProfileUUID] = append(serials, d.SerialNumber)
				} else {
					assigned[assigner.ProfileUUID] = []string{d.SerialNumber}
				}
			}
		}

	}
	return assigned, nil
}

func (w *Watcher) processAutoAssign(devices []dep.Device) error {
	assignments, err := w.filteredAutoAssignments(devices)
	if err != nil {
		return err
	}

	for profileUUID, serials := range assignments {
		resp, err := w.client.AssignProfile(profileUUID, serials...)
		if err != nil {
			level.Info(w.logger).Log(
				"err", err,
				"msg", "auto-assign error assigning serials to profile",
				"profile", profileUUID,
			)
			continue
		}
		// count our results for logging
		resultCounts := map[string]int{
			"SUCCESS":        0,
			"NOT_ACCESSIBLE": 0,
			"FAILED":         0,
		}
		for _, result := range resp.Devices {
			if ct, ok := resultCounts[result]; ok {
				// NOTE: we're logging _only_ the above pre-defined result types
				resultCounts[result] = ct + 1
			}
		}
		// TODO: alternate strategy is to log all failed devices
		// TODO: handle/requeue failed devices?
		level.Info(w.logger).Log(
			"msg", "DEP auto-assigned",
			"profile", profileUUID,
			"success", resultCounts["SUCCESS"],
			"not_accessible", resultCounts["NOT_ACCESSIBLE"],
			"failed", resultCounts["FAILED"],
		)
	}

	return nil
}

func (w *Watcher) publishAndProcessDevices(devices []dep.Device) error {
	e := NewEvent(devices)
	data, err := MarshalEvent(e)
	if err != nil {
		return err
	}
	err = w.publisher.Publish(context.TODO(), SyncTopic, data)
	if err != nil {
		return err
	}

	// TODO: instead of directly kicking off the auto-assigner process
	// consider placing a subscriber on the DEP pubsub topic. The same
	// information gets marshalled but it allows us the future
	// flexibility to separate out that component if we desired.
	go func() {
		err := w.processAutoAssign(devices)
		if err != nil {
			level.Info(w.logger).Log("err", err, "msg", "auto-assign error")
		}
	}()
	return nil
}

func (w *Watcher) Run() error {
	var (
		err       error
		resp      *dep.DeviceResponse
		fetchNext = true
		// for logging
		fetchNextLabel = map[bool]string{
			true:  "fetch",
			false: "sync",
		}
	)

	ticker := time.NewTicker(syncDuration).C
	for {
		if fetchNext {
			resp, err = w.client.FetchDevices(dep.Limit(100), dep.Cursor(w.cursor.Value))
			if err != nil && isCursorExhausted(err) {
				level.Info(w.logger).Log(
					"msg", "DEP cursor returned all devices previously",
					"phase", fetchNextLabel[fetchNext],
					"cursor", w.cursor.Value,
				)
				fetchNext = false
				continue
			}
		} else {
			resp, err = w.client.SyncDevices(w.cursor.Value)
		}

		if err != nil && (isCursorExpired(err) || isCursorInvalid(err)) {
			level.Info(w.logger).Log(
				"msg", "DEP cursor error, retrying with empty cursor",
				"phase", fetchNextLabel[fetchNext],
				"cursor", w.cursor.Value,
				"err", err,
			)
			w.cursor.Value = ""
			fetchNext = true
			continue
		} else if err != nil {
			// log any other error, but do not return from the run loop.
			// probably just a transient network issue.
			level.Info(w.logger).Log(
				"msg", "error syncing DEP devices",
				"phase", fetchNextLabel[fetchNext],
				"cursor", w.cursor.Value,
				"err", err,
			)
		} else {
			level.Info(w.logger).Log(
				"msg", "DEP sync",
				"phase", fetchNextLabel[fetchNext],
				"cursor", resp.Cursor,
				"fetched", resp.FetchedUntil,
				"devices", len(resp.Devices),
				"more", resp.MoreToFollow,
			)

			if err := w.publishAndProcessDevices(resp.Devices); err != nil {
				return fmt.Errorf("publish and process devices: %w", err)
			}
			w.cursor = Cursor{Value: resp.Cursor, CreatedAt: time.Now()}
			if err := w.db.SaveCursor(w.cursor); err != nil {
				return errors.Wrap(err, "saving cursor from fetch")
			}

			if resp.MoreToFollow {
				continue
			} else if fetchNext {
				fetchNext = false
				continue
			}
		}

		select {
		case <-ticker:
		case <-w.syncNow:
			level.Info(w.logger).Log("msg", "explicit DEP sync requested")
		}
	}
}
<|endoftext|>