<commit_before>package main

import (
	"fmt"
	"reflect"
	"sort"
	"sync"

	"github.com/litl/galaxy/log"
	"github.com/litl/galaxy/shuttle/client"
)

var (
	ErrNoService        = fmt.Errorf("service does not exist")
	ErrNoBackend        = fmt.Errorf("backend does not exist")
	ErrDuplicateService = fmt.Errorf("service already exists")
	ErrDuplicateBackend = fmt.Errorf("backend already exists")
)

type VirtualHost struct {
	sync.Mutex
	Name string
	// All services registered under this vhost name.
	services []*Service
	// The last one we returned so we can RoundRobin them.
	last int
}

func (v *VirtualHost) Len() int {
	v.Lock()
	defer v.Unlock()
	return len(v.services)
}

// Insert a service
// do nothing if the service already is registered
func (v *VirtualHost) Add(svc *Service) {
	v.Lock()
	defer v.Unlock()
	for _, s := range v.services {
		if s.Name == svc.Name {
			log.Debugf("Service %s already registered in VirtualHost %s", svc.Name, v.Name)
			return
		}
	}

	// TODO: is this the best place to log these?
	svcCfg := svc.Config()
	for _, backend := range svcCfg.Backends {
		log.Printf("Adding HTTP endpoint http://%s to %s", backend.Addr, v.Name)
	}
	v.services = append(v.services, svc)
}

func (v *VirtualHost) Remove(svc *Service) {
	v.Lock()
	defer v.Unlock()

	found := -1
	for i, s := range v.services {
		if s.Name == svc.Name {
			found = i
			break
		}
	}

	if found < 0 {
		log.Debugf("Service %s not found under VirtualHost %s", svc.Name, v.Name)
		return
	}

	// safe way to get the backends info for logging
	svcCfg := svc.Config()

	// Now removing this Service
	for _, backend := range svcCfg.Backends {
		log.Printf("Removing HTTP endpoint http://%s from %s", backend.Addr, v.Name)
	}

	v.services = append(v.services[:found], v.services[found+1:]...)
}

// Return a *Service for this VirtualHost
func (v *VirtualHost) Service() *Service {
	v.Lock()
	defer v.Unlock()

	if len(v.services) == 0 {
		log.Warnf("No Services registered for VirtualHost %s", v.Name)
		return nil
	}

	// start cycling through the services in case one has no backends available
	for i := 1; i <= len(v.services); i++ {
		idx := (v.last + i) % len(v.services)
		if v.services[idx].Available() > 0 {
			v.last = idx
			return v.services[idx]
		}
	}
	log.Warnf("No Available backends for VirtualHost %s", v.Name)
	return nil
}

//TODO: notify or prevent vhost name conflicts between services.
// ServiceRegistry is a global container for all configured services.
type ServiceRegistry struct {
	sync.Mutex
	svcs map[string]*Service
	// Multiple services may respond from a single vhost
	vhosts map[string]*VirtualHost
}

// Return a service by name.
func (s *ServiceRegistry) GetService(name string) *Service {
	s.Lock()
	defer s.Unlock()
	return s.svcs[name]
}

// Return a service that handles a particular vhost by name.
func (s *ServiceRegistry) GetVHostService(name string) *Service {
	s.Lock()
	defer s.Unlock()

	if vhost := s.vhosts[name]; vhost != nil {
		return vhost.Service()
	}
	return nil
}

func (s *ServiceRegistry) VHostsLen() int {
	s.Lock()
	defer s.Unlock()
	return len(s.vhosts)
}

// Add a new service to the Registry.
// Do not replace an existing service.
func (s *ServiceRegistry) AddService(cfg client.ServiceConfig) error {
	s.Lock()
	defer s.Unlock()

	log.Debug("Adding service:", cfg.Name)
	if _, ok := s.svcs[cfg.Name]; ok {
		log.Debug("Service already exists:", cfg.Name)
		return ErrDuplicateService
	}

	service := NewService(cfg)
	s.svcs[service.Name] = service

	for _, name := range cfg.VirtualHosts {
		vhost := s.vhosts[name]
		if vhost == nil {
			vhost = &VirtualHost{Name: name}
			s.vhosts[name] = vhost
		}
		vhost.Add(service)
	}

	return service.start()
}

// Replace the service's configuration, or update its list of backends.
// Replacing a configuration will shutdown the existing service, and start a
// new one, which will cause the listening socket to be temporarily
// unavailable.
func (s *ServiceRegistry) UpdateService(newCfg client.ServiceConfig) error {
	s.Lock()
	defer s.Unlock()

	log.Debug("Updating Service:", newCfg.Name)
	service, ok := s.svcs[newCfg.Name]
	if !ok {
		log.Debug("Service not found:", newCfg.Name)
		return ErrNoService
	}

	currentCfg := service.Config()

	// Lots of looping here (including fetching the Config, but the cardinality
	// of Backends shouldn't be very large, and the default RoundRobin balancing
	// is much simpler with a slice.

	// we're going to update just the backends for this config
	// get a map of what's already running
	currentBackends := make(map[string]client.BackendConfig)
	for _, backendCfg := range currentCfg.Backends {
		currentBackends[backendCfg.Name] = backendCfg
	}

	// Keep existing backends when they have equivalent config.
	// Update changed backends, and add new ones.
	for _, newBackend := range newCfg.Backends {
		current, ok := currentBackends[newBackend.Name]
		if ok && current.Equal(newBackend) {
			log.Debugf("Backend %s/%s unchanged", service.Name, current.Name)
			// no change for this one
			delete(currentBackends, current.Name)
			continue
		}

		// we need to remove and re-add this backend
		log.Debugf("Updating Backend %s/%s", service.Name, newBackend.Name)
		service.remove(newBackend.Name)
		service.add(NewBackend(newBackend))

		delete(currentBackends, newBackend.Name)
	}

	// remove any left over backends
	for name := range currentBackends {
		log.Debugf("Removing Backend %s/%s", service.Name, name)
		service.remove(name)
	}

	if currentCfg.Equal(newCfg) {
		log.Debugf("Service Unchanged %s", service.Name)
		return nil
	}

	// replace error pages if there's any change
	if !reflect.DeepEqual(service.errPagesCfg, newCfg.ErrorPages) {
		log.Debugf("Updating ErrorPages")
		service.errPagesCfg = newCfg.ErrorPages
		service.errorPages.Update(newCfg.ErrorPages)
	}

	s.updateVHosts(service, newCfg.VirtualHosts)

	return nil
}

// update the VirtualHost entries for this service
// only to be called from UpdateService.
func (s *ServiceRegistry) updateVHosts(service *Service, newHosts []string) {
	// We could just clear the vhosts and the new list since we're doing
	// this all while the registry is locked, but because we want sane log
	// messages about adding remove endpoints, we have to diff the slices
	// anyway.

	oldHosts := service.VirtualHosts
	sort.Strings(oldHosts)
	sort.Strings(newHosts)

	// find the relative compliments of each set of hostnames
	var remove, add []string
	i, j := 0, 0
	for i < len(oldHosts) && j < len(newHosts) {
		if oldHosts[i] != newHosts[j] {
			if oldHosts[i] < newHosts[j] {
				// oldHosts[i] can't be in newHosts
				remove = append(remove, oldHosts[i])
				i++
				continue
			} else {
				// newHosts[j] can't be in oldHosts
				add = append(add, newHosts[j])
				j++
				continue
			}
		}
		i++
		j++
	}
	if i < len(oldHosts) {
		// there's more!
		remove = append(remove, oldHosts[i:]...)
	}
	if j < len(newHosts) {
		add = append(add, newHosts[j:]...)
	}

	// remove existing vhost entries for this service, and add new ones
	for _, name := range remove {
		vhost := s.vhosts[name]
		if vhost != nil {
			vhost.Remove(service)
		}
		if vhost.Len() == 0 {
			log.Println("Removing empty VirtualHost", name)
			delete(s.vhosts, name)
		}
	}

	for _, name := range add {
		vhost := s.vhosts[name]
		if vhost == nil {
			vhost = &VirtualHost{Name: name}
			s.vhosts[name] = vhost
		}
		vhost.Add(service)
	}

	// and replace the list
	service.VirtualHosts = newHosts
}

func (s *ServiceRegistry) RemoveService(name string) error {
	s.Lock()
	defer s.Unlock()

	svc, ok := s.svcs[name]
	if ok {
		log.Debugf("Removing Service %s", svc.Name)
		delete(s.svcs, name)
		svc.stop()

		for host := range s.vhosts {
			delete(s.vhosts, host)
		}

		return nil
	}
	return ErrNoService
}

func (s *ServiceRegistry) ServiceStats(serviceName string) (ServiceStat, error) {
	s.Lock()
	defer s.Unlock()

	service, ok := s.svcs[serviceName]
	if !ok {
		return ServiceStat{}, ErrNoService
	}
	return service.Stats(), nil
}

func (s *ServiceRegistry) ServiceConfig(serviceName string) (client.ServiceConfig, error) {
	s.Lock()
	defer s.Unlock()

	service, ok := s.svcs[serviceName]
	if !ok {
		return client.ServiceConfig{}, ErrNoService
	}
	return service.Config(), nil
}

func (s *ServiceRegistry) BackendStats(serviceName, backendName string) (BackendStat, error) {
	s.Lock()
	defer s.Unlock()

	service, ok := s.svcs[serviceName]
	if !ok {
		return BackendStat{}, ErrNoService
	}

	for _, backend := range service.Backends {
		if backendName == backend.Name {
			return backend.Stats(), nil
		}
	}
	return BackendStat{}, ErrNoBackend
}

// Add or update a Backend on an existing Service.
func (s *ServiceRegistry) AddBackend(svcName string, backendCfg client.BackendConfig) error {
	s.Lock()
	defer s.Unlock()

	service, ok := s.svcs[svcName]
	if !ok {
		return ErrNoService
	}

	log.Debugf("Adding Backend %s/%s", service.Name, backendCfg.Name)
	service.add(NewBackend(backendCfg))
	return nil
}

// Remove a Backend from an existing Service.
func (s *ServiceRegistry) RemoveBackend(svcName, backendName string) error {
	s.Lock()
	defer s.Unlock()

	log.Debugf("Removing Backend %s/%s", svcName, backendName)
	service, ok := s.svcs[svcName]
	if !ok {
		return ErrNoService
	}

	if !service.remove(backendName) {
		return ErrNoBackend
	}
	return nil
}

func (s *ServiceRegistry) Stats() []ServiceStat {
	s.Lock()
	defer s.Unlock()

	stats := []ServiceStat{}
	for _, service := range s.svcs {
		stats = append(stats, service.Stats())
	}

	return stats
}

func (s *ServiceRegistry) Config() []client.ServiceConfig {
	s.Lock()
	defer s.Unlock()

	var configs []client.ServiceConfig
	for _, service := range s.svcs {
		configs = append(configs, service.Config())
	}

	return configs
}

func (s *ServiceRegistry) String() string {
	return string(marshal(s.Config()))
}
<commit_msg>Return a service even if all backends are down<commit_after>package main

import (
	"fmt"
	"reflect"
	"sort"
	"sync"

	"github.com/litl/galaxy/log"
	"github.com/litl/galaxy/shuttle/client"
)

var (
	ErrNoService        = fmt.Errorf("service does not exist")
	ErrNoBackend        = fmt.Errorf("backend does not exist")
	ErrDuplicateService = fmt.Errorf("service already exists")
	ErrDuplicateBackend = fmt.Errorf("backend already exists")
)

type VirtualHost struct {
	sync.Mutex
	Name string
	// All services registered under this vhost name.
	services []*Service
	// The last one we returned so we can RoundRobin them.
	last int
}

func (v *VirtualHost) Len() int {
	v.Lock()
	defer v.Unlock()
	return len(v.services)
}

// Insert a service
// do nothing if the service already is registered
func (v *VirtualHost) Add(svc *Service) {
	v.Lock()
	defer v.Unlock()
	for _, s := range v.services {
		if s.Name == svc.Name {
			log.Debugf("Service %s already registered in VirtualHost %s", svc.Name, v.Name)
			return
		}
	}

	// TODO: is this the best place to log these?
	svcCfg := svc.Config()
	for _, backend := range svcCfg.Backends {
		log.Printf("Adding HTTP endpoint http://%s to %s", backend.Addr, v.Name)
	}
	v.services = append(v.services, svc)
}

func (v *VirtualHost) Remove(svc *Service) {
	v.Lock()
	defer v.Unlock()

	found := -1
	for i, s := range v.services {
		if s.Name == svc.Name {
			found = i
			break
		}
	}

	if found < 0 {
		log.Debugf("Service %s not found under VirtualHost %s", svc.Name, v.Name)
		return
	}

	// safe way to get the backends info for logging
	svcCfg := svc.Config()

	// Now removing this Service
	for _, backend := range svcCfg.Backends {
		log.Printf("Removing HTTP endpoint http://%s from %s", backend.Addr, v.Name)
	}

	v.services = append(v.services[:found], v.services[found+1:]...)
}

// Return a *Service for this VirtualHost
func (v *VirtualHost) Service() *Service {
	v.Lock()
	defer v.Unlock()

	if len(v.services) == 0 {
		log.Warnf("No Services registered for VirtualHost %s", v.Name)
		return nil
	}

	// start cycling through the services in case one has no backends available
	for i := 1; i <= len(v.services); i++ {
		idx := (v.last + i) % len(v.services)
		if v.services[idx].Available() > 0 {
			v.last = idx
			return v.services[idx]
		}
	}

	// even if all backends are down, return a service so that the request can
	// be processed normally (we may have a custom 502 error page for this)
	return v.services[v.last]
}

//TODO: notify or prevent vhost name conflicts between services.
// ServiceRegistry is a global container for all configured services.
type ServiceRegistry struct {
	sync.Mutex
	svcs map[string]*Service
	// Multiple services may respond from a single vhost
	vhosts map[string]*VirtualHost
}

// Return a service by name.
func (s *ServiceRegistry) GetService(name string) *Service {
	s.Lock()
	defer s.Unlock()
	return s.svcs[name]
}

// Return a service that handles a particular vhost by name.
func (s *ServiceRegistry) GetVHostService(name string) *Service {
	s.Lock()
	defer s.Unlock()

	if vhost := s.vhosts[name]; vhost != nil {
		return vhost.Service()
	}
	return nil
}

func (s *ServiceRegistry) VHostsLen() int {
	s.Lock()
	defer s.Unlock()
	return len(s.vhosts)
}

// Add a new service to the Registry.
// Do not replace an existing service.
func (s *ServiceRegistry) AddService(cfg client.ServiceConfig) error {
	s.Lock()
	defer s.Unlock()

	log.Debug("Adding service:", cfg.Name)
	if _, ok := s.svcs[cfg.Name]; ok {
		log.Debug("Service already exists:", cfg.Name)
		return ErrDuplicateService
	}

	service := NewService(cfg)
	s.svcs[service.Name] = service

	for _, name := range cfg.VirtualHosts {
		vhost := s.vhosts[name]
		if vhost == nil {
			vhost = &VirtualHost{Name: name}
			s.vhosts[name] = vhost
		}
		vhost.Add(service)
	}

	return service.start()
}

// Replace the service's configuration, or update its list of backends.
// Replacing a configuration will shutdown the existing service, and start a
// new one, which will cause the listening socket to be temporarily
// unavailable.
func (s *ServiceRegistry) UpdateService(newCfg client.ServiceConfig) error {
	s.Lock()
	defer s.Unlock()

	log.Debug("Updating Service:", newCfg.Name)
	service, ok := s.svcs[newCfg.Name]
	if !ok {
		log.Debug("Service not found:", newCfg.Name)
		return ErrNoService
	}

	currentCfg := service.Config()

	// Lots of looping here (including fetching the Config, but the cardinality
	// of Backends shouldn't be very large, and the default RoundRobin balancing
	// is much simpler with a slice.

	// we're going to update just the backends for this config
	// get a map of what's already running
	currentBackends := make(map[string]client.BackendConfig)
	for _, backendCfg := range currentCfg.Backends {
		currentBackends[backendCfg.Name] = backendCfg
	}

	// Keep existing backends when they have equivalent config.
	// Update changed backends, and add new ones.
	for _, newBackend := range newCfg.Backends {
		current, ok := currentBackends[newBackend.Name]
		if ok && current.Equal(newBackend) {
			log.Debugf("Backend %s/%s unchanged", service.Name, current.Name)
			// no change for this one
			delete(currentBackends, current.Name)
			continue
		}

		// we need to remove and re-add this backend
		log.Debugf("Updating Backend %s/%s", service.Name, newBackend.Name)
		service.remove(newBackend.Name)
		service.add(NewBackend(newBackend))

		delete(currentBackends, newBackend.Name)
	}

	// remove any left over backends
	for name := range currentBackends {
		log.Debugf("Removing Backend %s/%s", service.Name, name)
		service.remove(name)
	}

	if currentCfg.Equal(newCfg) {
		log.Debugf("Service Unchanged %s", service.Name)
		return nil
	}

	// replace error pages if there's any change
	if !reflect.DeepEqual(service.errPagesCfg, newCfg.ErrorPages) {
		log.Debugf("Updating ErrorPages")
		service.errPagesCfg = newCfg.ErrorPages
		service.errorPages.Update(newCfg.ErrorPages)
	}

	s.updateVHosts(service, newCfg.VirtualHosts)

	return nil
}

// update the VirtualHost entries for this service
// only to be called from UpdateService.
func (s *ServiceRegistry) updateVHosts(service *Service, newHosts []string) {
	// We could just clear the vhosts and the new list since we're doing
	// this all while the registry is locked, but because we want sane log
	// messages about adding remove endpoints, we have to diff the slices
	// anyway.

	oldHosts := service.VirtualHosts
	sort.Strings(oldHosts)
	sort.Strings(newHosts)

	// find the relative compliments of each set of hostnames
	var remove, add []string
	i, j := 0, 0
	for i < len(oldHosts) && j < len(newHosts) {
		if oldHosts[i] != newHosts[j] {
			if oldHosts[i] < newHosts[j] {
				// oldHosts[i] can't be in newHosts
				remove = append(remove, oldHosts[i])
				i++
				continue
			} else {
				// newHosts[j] can't be in oldHosts
				add = append(add, newHosts[j])
				j++
				continue
			}
		}
		i++
		j++
	}
	if i < len(oldHosts) {
		// there's more!
		remove = append(remove, oldHosts[i:]...)
	}
	if j < len(newHosts) {
		add = append(add, newHosts[j:]...)
	}

	// remove existing vhost entries for this service, and add new ones
	for _, name := range remove {
		vhost := s.vhosts[name]
		if vhost != nil {
			vhost.Remove(service)
		}
		if vhost.Len() == 0 {
			log.Println("Removing empty VirtualHost", name)
			delete(s.vhosts, name)
		}
	}

	for _, name := range add {
		vhost := s.vhosts[name]
		if vhost == nil {
			vhost = &VirtualHost{Name: name}
			s.vhosts[name] = vhost
		}
		vhost.Add(service)
	}

	// and replace the list
	service.VirtualHosts = newHosts
}

func (s *ServiceRegistry) RemoveService(name string) error {
	s.Lock()
	defer s.Unlock()

	svc, ok := s.svcs[name]
	if ok {
		log.Debugf("Removing Service %s", svc.Name)
		delete(s.svcs, name)
		svc.stop()

		for host := range s.vhosts {
			delete(s.vhosts, host)
		}

		return nil
	}
	return ErrNoService
}

func (s *ServiceRegistry) ServiceStats(serviceName string) (ServiceStat, error) {
	s.Lock()
	defer s.Unlock()

	service, ok := s.svcs[serviceName]
	if !ok {
		return ServiceStat{}, ErrNoService
	}
	return service.Stats(), nil
}

func (s *ServiceRegistry) ServiceConfig(serviceName string) (client.ServiceConfig, error) {
	s.Lock()
	defer s.Unlock()

	service, ok := s.svcs[serviceName]
	if !ok {
		return client.ServiceConfig{}, ErrNoService
	}
	return service.Config(), nil
}

func (s *ServiceRegistry) BackendStats(serviceName, backendName string) (BackendStat, error) {
	s.Lock()
	defer s.Unlock()

	service, ok := s.svcs[serviceName]
	if !ok {
		return BackendStat{}, ErrNoService
	}

	for _, backend := range service.Backends {
		if backendName == backend.Name {
			return backend.Stats(), nil
		}
	}
	return BackendStat{}, ErrNoBackend
}

// Add or update a Backend on an existing Service.
func (s *ServiceRegistry) AddBackend(svcName string, backendCfg client.BackendConfig) error {
	s.Lock()
	defer s.Unlock()

	service, ok := s.svcs[svcName]
	if !ok {
		return ErrNoService
	}

	log.Debugf("Adding Backend %s/%s", service.Name, backendCfg.Name)
	service.add(NewBackend(backendCfg))
	return nil
}

// Remove a Backend from an existing Service.
func (s *ServiceRegistry) RemoveBackend(svcName, backendName string) error {
	s.Lock()
	defer s.Unlock()

	log.Debugf("Removing Backend %s/%s", svcName, backendName)
	service, ok := s.svcs[svcName]
	if !ok {
		return ErrNoService
	}

	if !service.remove(backendName) {
		return ErrNoBackend
	}
	return nil
}

func (s *ServiceRegistry) Stats() []ServiceStat {
	s.Lock()
	defer s.Unlock()

	stats := []ServiceStat{}
	for _, service := range s.svcs {
		stats = append(stats, service.Stats())
	}

	return stats
}

func (s *ServiceRegistry) Config() []client.ServiceConfig {
	s.Lock()
	defer s.Unlock()

	var configs []client.ServiceConfig
	for _, service := range s.svcs {
		configs = append(configs, service.Config())
	}

	return configs
}

func (s *ServiceRegistry) String() string {
	return string(marshal(s.Config()))
}
<|endoftext|>


<commit_before>package sched

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"sync"
	"time"

	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/tsaf/conf"
	"github.com/StackExchange/tsaf/expr"
)

type Schedule struct {
	sync.Mutex

	Conf          *conf.Conf
	Status        map[AlertKey]*State
	Notifications map[AlertKey]map[string]time.Time
	Silence       []*Silence

	cache     *opentsdb.Cache
	runStates map[AlertKey]Status
	nc        chan interface{}
}

type Silence struct {
	Start, End time.Time
	Text       string
	match      map[string]*regexp.Regexp
}

func (s *Silence) Silenced(tags opentsdb.TagSet) bool {
	now := time.Now()
	if now.Before(s.Start) || now.After(s.End) {
		return false
	}
	for k, re := range s.match {
		tagv, ok := tags[k]
		if !ok || !re.MatchString(tagv) {
			return false
		}
	}
	return true
}

// Silenced returns all currently silenced AlertKeys and the time they will
// be unsilenced.
func (s *Schedule) Silenced() map[AlertKey]time.Time {
	aks := make(map[AlertKey]time.Time)
	for _, si := range s.Silence {
		for ak, st := range s.Status {
			if si.Silenced(st.Group) {
				if aks[ak].Before(si.End) {
					aks[ak] = si.End
				}
				break
			}
		}
	}
	return aks
}

func (s *Schedule) AddSilence(start, end time.Time, text string, confirm bool) ([]AlertKey, error) {
	if start.IsZero() || end.IsZero() {
		return nil, fmt.Errorf("both start and end must be specified")
	}
	if start.After(end) {
		return nil, fmt.Errorf("start time must be before end time")
	}
	tags, err := opentsdb.ParseTags(text)
	if err != nil {
		return nil, err
	} else if len(tags) == 0 {
		return nil, fmt.Errorf("empty text")
	}
	si := Silence{
		Start: start,
		End:   end,
		Text:  text,
		match: make(map[string]*regexp.Regexp),
	}
	for k, v := range tags {
		re, err := regexp.Compile(v)
		if err != nil {
			return nil, err
		}
		si.match[k] = re
	}
	if confirm {
		s.Lock()
		s.Silence = append(s.Silence, &si)
		s.Unlock()
		return nil, nil
	}
	var aks []AlertKey
	for ak, st := range s.Status {
		if si.Silenced(st.Group) {
			aks = append(aks, ak)
			break
		}
	}
	return aks, nil
}

func (s *Schedule) MarshalJSON() ([]byte, error) {
	t := struct {
		Alerts map[string]*conf.Alert
		Status map[string]*State
	}{
		s.Conf.Alerts,
		make(map[string]*State),
	}
	for k, v := range s.Status {
		if v.Last().Status < stWarning {
			continue
		}
		t.Status[k.String()] = v
	}
	return json.Marshal(&t)
}

var DefaultSched = &Schedule{}

// Loads a configuration into the default schedule
func Load(c *conf.Conf) {
	DefaultSched.Load(c)
}

// Runs the default schedule.
func Run() error {
	return DefaultSched.Run()
}

func (s *Schedule) Load(c *conf.Conf) {
	s.Conf = c
	s.RestoreState()
}

// Restores notification and alert state from the file on disk.
func (s *Schedule) RestoreState() {
	s.Lock()
	defer s.Unlock()
	s.cache = opentsdb.NewCache(s.Conf.TsdbHost)
	s.Notifications = nil
	s.Status = make(map[AlertKey]*State)
	f, err := os.Open(s.Conf.StateFile)
	if err != nil {
		log.Println(err)
		return
	}
	dec := gob.NewDecoder(f)
	notifications := make(map[AlertKey]map[string]time.Time)
	if err := dec.Decode(&notifications); err != nil {
		log.Println(err)
		return
	}
	for {
		var ak AlertKey
		var st State
		if err := dec.Decode(&ak); err == io.EOF {
			break
		} else if err != nil {
			log.Println(err)
			return
		}
		if err := dec.Decode(&st); err != nil {
			log.Println(err)
			return
		}
		if a, present := s.Conf.Alerts[ak.Name]; !present {
			log.Println("sched: alert no longer present, ignoring:", ak)
			continue
		} else if a.Squelched(st.Group) {
			log.Println("sched: alert now squelched:", ak)
			continue
		} else {
			t := a.Unknown
			if t == 0 {
				t = s.Conf.Unknown
			}
			if t == 0 && st.Last().Status == stUnknown {
				st.Append(stNormal)
			}
		}
		s.Status[ak] = &st
		for name, t := range notifications[ak] {
			n, present := s.Conf.Notifications[name]
			if !present {
				log.Println("sched: notification not present during restore:", name)
				continue
			}
			s.AddNotification(ak, n, t)
		}
	}
}

func (s *Schedule) Save() {
	s.Lock()
	defer s.Unlock()
	f, err := os.Create(s.Conf.StateFile)
	if err != nil {
		log.Println(err)
		return
	}
	enc := gob.NewEncoder(f)
	if err := enc.Encode(s.Notifications); err != nil {
		log.Println(err)
		return
	}
	for k, v := range s.Status {
		enc.Encode(k)
		enc.Encode(v)
	}
	if err := f.Close(); err != nil {
		log.Println(err)
		return
	}
	log.Println("sched: wrote state to", s.Conf.StateFile)
}

func (s *Schedule) Run() error {
	go func() {
		for {
			time.Sleep(time.Minute)
			s.Save()
		}
	}()
	s.nc = make(chan interface{}, 1)
	go s.Poll()
	for {
		wait := time.After(s.Conf.CheckFrequency)
		if s.Conf.CheckFrequency < time.Second {
			return fmt.Errorf("sched: frequency must be > 1 second")
		}
		if s.Conf == nil {
			return fmt.Errorf("sched: nil configuration")
		}
		start := time.Now()
		log.Printf("starting run at %v\n", start)
		s.Check()
		log.Printf("run at %v took %v\n", start, time.Since(start))
		<-wait
	}
}

// Poll dispatches notification checks when needed.
func (s *Schedule) Poll() {
	var timeout time.Duration
	for {
		// Wait for one of these two.
		select {
		case <-time.After(timeout):
		case <-s.nc:
		}
		timeout = s.CheckNotifications()
	}
}

// CheckNotifications processes past notification events. It returns the
func (s *Schedule) CheckNotifications() time.Duration {
	s.Lock()
	defer s.Unlock()
	timeout := time.Hour
	notifications := s.Notifications
	s.Notifications = nil
	for ak, ns := range notifications {
		for name, t := range ns {
			n, present := s.Conf.Notifications[name]
			if !present {
				continue
			}
			remaining := t.Add(n.Timeout).Sub(time.Now())
			if remaining > 0 {
				if remaining < timeout {
					timeout = remaining
				}
				s.AddNotification(ak, n, t)
				continue
			}
			st, present := s.Status[ak]
			if !present {
				continue
			}
			a, present := s.Conf.Alerts[ak.Name]
			if !present {
				continue
			}
			s.Notify(st, a, n)
			if n.Timeout < timeout {
				timeout = n.Timeout
			}
		}
	}
	return timeout
}

func (s *Schedule) Check() {
	s.Lock()
	defer s.Unlock()
	s.runStates = make(map[AlertKey]Status)
	s.cache = opentsdb.NewCache(s.Conf.TsdbHost)
	for _, a := range s.Conf.Alerts {
		s.CheckAlert(a)
	}
	s.CheckUnknown()
	checkNotify := false
	for ak, status := range s.runStates {
		state := s.Status[ak]
		last := state.Append(status)
		a := s.Conf.Alerts[ak.Name]
		if status > stNormal {
			var subject = new(bytes.Buffer)
			if err := s.ExecuteSubject(subject, a, state); err != nil {
				log.Println(err)
			}
			state.Subject = subject.String()
		}
		// On state increase, clear old notifications and notify current.
		// On state decrease, and if the old alert was already acknowledged, notify current.
		// If the old alert was not acknowledged, do nothing.
		// Do nothing if state did not change.
		notify := func(notifications map[string]*conf.Notification) {
			for _, n := range notifications {
				s.Notify(state, a, n)
				checkNotify = true
			}
		}
		notifyCurrent := func() {
			switch status {
			case stCritical, stUnknown:
				notify(a.CritNotification)
			case stWarning:
				notify(a.WarnNotification)
			}
		}
		clearOld := func() {
			delete(s.Notifications, ak)
		}
		if status > last {
			clearOld()
			notifyCurrent()
		} else if status < last {
			if _, hasOld := s.Notifications[ak]; hasOld {
				notifyCurrent()
			}
		}
	}
	if checkNotify {
		s.nc <- true
	}
}

func (s *Schedule) CheckUnknown() {
	for ak, st := range s.Status {
		t := s.Conf.Alerts[ak.Name].Unknown
		if t == 0 {
			t = s.Conf.Unknown
		}
		if t == 0 {
			continue
		}
		if time.Since(st.Touched) < t {
			continue
		}
		s.runStates[ak] = stUnknown
	}
}

func (s *Schedule) CheckAlert(a *conf.Alert) {
	crits := s.CheckExpr(a, a.Crit, stCritical, nil)
	warns := s.CheckExpr(a, a.Warn, stWarning, crits)
	log.Printf("checking alert %v: %v crits, %v warns", a.Name, len(crits), len(warns))
}

func (s *Schedule) CheckExpr(a *conf.Alert, e *expr.Expr, checkStatus Status, ignore []AlertKey) (alerts []AlertKey) {
	if e == nil {
		return
	}
	results, _, err := e.Execute(s.cache, nil)
	if err != nil {
		// todo: do something here?
		log.Println(err)
		return
	}
Loop:
	for _, r := range results {
		if a.Squelched(r.Group) {
			continue
		}
		ak := AlertKey{a.Name, r.Group.String()}
		for _, v := range ignore {
			if ak == v {
				continue Loop
			}
		}
		state := s.Status[ak]
		if state == nil {
			state = &State{
				Group: r.Group,
			}
			s.Status[ak] = state
		}
		state.Touch()
		status := checkStatus
		state.Computations = r.Computations
		if r.Value.(expr.Number) != 0 {
			state.Expr = e.String()
			alerts = append(alerts, ak)
		} else {
			status = stNormal
		}
		if status > s.runStates[ak] {
			s.runStates[ak] = status
		}
	}
	return
}

func (s *Schedule) Notify(st *State, a *conf.Alert, n *conf.Notification) {
	subject := new(bytes.Buffer)
	if err := s.ExecuteSubject(subject, a, st); err != nil {
		log.Println(err)
	}
	body := new(bytes.Buffer)
	if err := s.ExecuteBody(body, a, st); err != nil {
		log.Println(err)
	}
	n.Notify(subject.Bytes(), body.Bytes(), s.Conf.EmailFrom, s.Conf.SmtpHost)
	if n.Next == nil {
		return
	}
	s.AddNotification(AlertKey{Name: a.Name, Group: st.Group.String()}, n, time.Now().UTC())
}

func (s *Schedule) AddNotification(ak AlertKey, n *conf.Notification, started time.Time) {
	if s.Notifications == nil {
		s.Notifications = make(map[AlertKey]map[string]time.Time)
	}
	if s.Notifications[ak] == nil {
		s.Notifications[ak] = make(map[string]time.Time)
	}
	s.Notifications[ak][n.Name] = started
}

type AlertKey struct {
	Name  string
	Group string
}

func (a AlertKey) String() string {
	return a.Name + a.Group
}

type State struct {
	// Most recent event last.
	History      []Event
	Touched      time.Time
	Expr         string
	Group        opentsdb.TagSet
	Computations expr.Computations
	Subject      string
}

func (s *Schedule) Acknowledge(ak AlertKey) {
	s.Lock()
	delete(s.Notifications, ak)
	s.Unlock()
}

func (s *State) Touch() {
	s.Touched = time.Now().UTC()
}

// Appends status to the history if the status is different than the latest
// status. Returns the previous status.
func (s *State) Append(status Status) Status {
	last := s.Last()
	if len(s.History) == 0 || s.Last().Status != status {
		s.History = append(s.History, Event{status, time.Now().UTC()})
	}
	return last.Status
}

func (s *State) Last() Event {
	if len(s.History) == 0 {
		return Event{}
	}
	return s.History[len(s.History)-1]
}

type Event struct {
	Status Status
	Time   time.Time
}

type Status int

const (
	stNone Status = iota
	stNormal
	stWarning
	stCritical
	stUnknown
)

func (s Status) String() string {
	switch s {
	case stNormal:
		return "normal"
	case stWarning:
		return "warning"
	case stCritical:
		return "critical"
	case stUnknown:
		return "unknown"
	default:
		return "none"
	}
}
<commit_msg>Complete doc comment<commit_after>package sched

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"sync"
	"time"

	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/tsaf/conf"
	"github.com/StackExchange/tsaf/expr"
)

type Schedule struct {
	sync.Mutex

	Conf          *conf.Conf
	Status        map[AlertKey]*State
	Notifications map[AlertKey]map[string]time.Time
	Silence       []*Silence

	cache     *opentsdb.Cache
	runStates map[AlertKey]Status
	nc        chan interface{}
}

type Silence struct {
	Start, End time.Time
	Text       string
	match      map[string]*regexp.Regexp
}

func (s *Silence) Silenced(tags opentsdb.TagSet) bool {
	now := time.Now()
	if now.Before(s.Start) || now.After(s.End) {
		return false
	}
	for k, re := range s.match {
		tagv, ok := tags[k]
		if !ok || !re.MatchString(tagv) {
			return false
		}
	}
	return true
}

// Silenced returns all currently silenced AlertKeys and the time they will
// be unsilenced.
func (s *Schedule) Silenced() map[AlertKey]time.Time {
	aks := make(map[AlertKey]time.Time)
	for _, si := range s.Silence {
		for ak, st := range s.Status {
			if si.Silenced(st.Group) {
				if aks[ak].Before(si.End) {
					aks[ak] = si.End
				}
				break
			}
		}
	}
	return aks
}

func (s *Schedule) AddSilence(start, end time.Time, text string, confirm bool) ([]AlertKey, error) {
	if start.IsZero() || end.IsZero() {
		return nil, fmt.Errorf("both start and end must be specified")
	}
	if start.After(end) {
		return nil, fmt.Errorf("start time must be before end time")
	}
	tags, err := opentsdb.ParseTags(text)
	if err != nil {
		return nil, err
	} else if len(tags) == 0 {
		return nil, fmt.Errorf("empty text")
	}
	si := Silence{
		Start: start,
		End:   end,
		Text:  text,
		match: make(map[string]*regexp.Regexp),
	}
	for k, v := range tags {
		re, err := regexp.Compile(v)
		if err != nil {
			return nil, err
		}
		si.match[k] = re
	}
	if confirm {
		s.Lock()
		s.Silence = append(s.Silence, &si)
		s.Unlock()
		return nil, nil
	}
	var aks []AlertKey
	for ak, st := range s.Status {
		if si.Silenced(st.Group) {
			aks = append(aks, ak)
			break
		}
	}
	return aks, nil
}

func (s *Schedule) MarshalJSON() ([]byte, error) {
	t := struct {
		Alerts map[string]*conf.Alert
		Status map[string]*State
	}{
		s.Conf.Alerts,
		make(map[string]*State),
	}
	for k, v := range s.Status {
		if v.Last().Status < stWarning {
			continue
		}
		t.Status[k.String()] = v
	}
	return json.Marshal(&t)
}

var DefaultSched = &Schedule{}

// Loads a configuration into the default schedule
func Load(c *conf.Conf) {
	DefaultSched.Load(c)
}

// Runs the default schedule.
func Run() error {
	return DefaultSched.Run()
}

func (s *Schedule) Load(c *conf.Conf) {
	s.Conf = c
	s.RestoreState()
}

// Restores notification and alert state from the file on disk.
func (s *Schedule) RestoreState() {
	s.Lock()
	defer s.Unlock()
	s.cache = opentsdb.NewCache(s.Conf.TsdbHost)
	s.Notifications = nil
	s.Status = make(map[AlertKey]*State)
	f, err := os.Open(s.Conf.StateFile)
	if err != nil {
		log.Println(err)
		return
	}
	dec := gob.NewDecoder(f)
	notifications := make(map[AlertKey]map[string]time.Time)
	if err := dec.Decode(&notifications); err != nil {
		log.Println(err)
		return
	}
	for {
		var ak AlertKey
		var st State
		if err := dec.Decode(&ak); err == io.EOF {
			break
		} else if err != nil {
			log.Println(err)
			return
		}
		if err := dec.Decode(&st); err != nil {
			log.Println(err)
			return
		}
		if a, present := s.Conf.Alerts[ak.Name]; !present {
			log.Println("sched: alert no longer present, ignoring:", ak)
			continue
		} else if a.Squelched(st.Group) {
			log.Println("sched: alert now squelched:", ak)
			continue
		} else {
			t := a.Unknown
			if t == 0 {
				t = s.Conf.Unknown
			}
			if t == 0 && st.Last().Status == stUnknown {
				st.Append(stNormal)
			}
		}
		s.Status[ak] = &st
		for name, t := range notifications[ak] {
			n, present := s.Conf.Notifications[name]
			if !present {
				log.Println("sched: notification not present during restore:", name)
				continue
			}
			s.AddNotification(ak, n, t)
		}
	}
}

func (s *Schedule) Save() {
	s.Lock()
	defer s.Unlock()
	f, err := os.Create(s.Conf.StateFile)
	if err != nil {
		log.Println(err)
		return
	}
	enc := gob.NewEncoder(f)
	if err := enc.Encode(s.Notifications); err != nil {
		log.Println(err)
		return
	}
	for k, v := range s.Status {
		enc.Encode(k)
		enc.Encode(v)
	}
	if err := f.Close(); err != nil {
		log.Println(err)
		return
	}
	log.Println("sched: wrote state to", s.Conf.StateFile)
}

func (s *Schedule) Run() error {
	go func() {
		for {
			time.Sleep(time.Minute)
			s.Save()
		}
	}()
	s.nc = make(chan interface{}, 1)
	go s.Poll()
	for {
		wait := time.After(s.Conf.CheckFrequency)
		if s.Conf.CheckFrequency < time.Second {
			return fmt.Errorf("sched: frequency must be > 1 second")
		}
		if s.Conf == nil {
			return fmt.Errorf("sched: nil configuration")
		}
		start := time.Now()
		log.Printf("starting run at %v\n", start)
		s.Check()
		log.Printf("run at %v took %v\n", start, time.Since(start))
		<-wait
	}
}

// Poll dispatches notification checks when needed.
func (s *Schedule) Poll() {
	var timeout time.Duration
	for {
		// Wait for one of these two.
		select {
		case <-time.After(timeout):
		case <-s.nc:
		}
		timeout = s.CheckNotifications()
	}
}

// CheckNotifications processes past notification events. It returns the
// duration until the soonest notification triggers.
func (s *Schedule) CheckNotifications() time.Duration {
	s.Lock()
	defer s.Unlock()
	timeout := time.Hour
	notifications := s.Notifications
	s.Notifications = nil
	for ak, ns := range notifications {
		for name, t := range ns {
			n, present := s.Conf.Notifications[name]
			if !present {
				continue
			}
			remaining := t.Add(n.Timeout).Sub(time.Now())
			if remaining > 0 {
				if remaining < timeout {
					timeout = remaining
				}
				s.AddNotification(ak, n, t)
				continue
			}
			st, present := s.Status[ak]
			if !present {
				continue
			}
			a, present := s.Conf.Alerts[ak.Name]
			if !present {
				continue
			}
			s.Notify(st, a, n)
			if n.Timeout < timeout {
				timeout = n.Timeout
			}
		}
	}
	return timeout
}

func (s *Schedule) Check() {
	s.Lock()
	defer s.Unlock()
	s.runStates = make(map[AlertKey]Status)
	s.cache = opentsdb.NewCache(s.Conf.TsdbHost)
	for _, a := range s.Conf.Alerts {
		s.CheckAlert(a)
	}
	s.CheckUnknown()
	checkNotify := false
	for ak, status := range s.runStates {
		state := s.Status[ak]
		last := state.Append(status)
		a := s.Conf.Alerts[ak.Name]
		if status > stNormal {
			var subject = new(bytes.Buffer)
			if err := s.ExecuteSubject(subject, a, state); err != nil {
				log.Println(err)
			}
			state.Subject = subject.String()
		}
		// On state increase, clear old notifications and notify current.
		// On state decrease, and if the old alert was already acknowledged, notify current.
		// If the old alert was not acknowledged, do nothing.
		// Do nothing if state did not change.
		notify := func(notifications map[string]*conf.Notification) {
			for _, n := range notifications {
				s.Notify(state, a, n)
				checkNotify = true
			}
		}
		notifyCurrent := func() {
			switch status {
			case stCritical, stUnknown:
				notify(a.CritNotification)
			case stWarning:
				notify(a.WarnNotification)
			}
		}
		clearOld := func() {
			delete(s.Notifications, ak)
		}
		if status > last {
			clearOld()
			notifyCurrent()
		} else if status < last {
			if _, hasOld := s.Notifications[ak]; hasOld {
				notifyCurrent()
			}
		}
	}
	if checkNotify {
		s.nc <- true
	}
}

func (s *Schedule) CheckUnknown() {
	for ak, st := range s.Status {
		t := s.Conf.Alerts[ak.Name].Unknown
		if t == 0 {
			t = s.Conf.Unknown
		}
		if t == 0 {
			continue
		}
		if time.Since(st.Touched) < t {
			continue
		}
		s.runStates[ak] = stUnknown
	}
}

func (s *Schedule) CheckAlert(a *conf.Alert) {
	crits := s.CheckExpr(a, a.Crit, stCritical, nil)
	warns := s.CheckExpr(a, a.Warn, stWarning, crits)
	log.Printf("checking alert %v: %v crits, %v warns", a.Name, len(crits), len(warns))
}

func (s *Schedule) CheckExpr(a *conf.Alert, e *expr.Expr, checkStatus Status, ignore []AlertKey) (alerts []AlertKey) {
	if e == nil {
		return
	}
	results, _, err := e.Execute(s.cache, nil)
	if err != nil {
		// todo: do something here?
		log.Println(err)
		return
	}
Loop:
	for _, r := range results {
		if a.Squelched(r.Group) {
			continue
		}
		ak := AlertKey{a.Name, r.Group.String()}
		for _, v := range ignore {
			if ak == v {
				continue Loop
			}
		}
		state := s.Status[ak]
		if state == nil {
			state = &State{
				Group: r.Group,
			}
			s.Status[ak] = state
		}
		state.Touch()
		status := checkStatus
		state.Computations = r.Computations
		if r.Value.(expr.Number) != 0 {
			state.Expr = e.String()
			alerts = append(alerts, ak)
		} else {
			status = stNormal
		}
		if status > s.runStates[ak] {
			s.runStates[ak] = status
		}
	}
	return
}

func (s *Schedule) Notify(st *State, a *conf.Alert, n *conf.Notification) {
	subject := new(bytes.Buffer)
	if err := s.ExecuteSubject(subject, a, st); err != nil {
		log.Println(err)
	}
	body := new(bytes.Buffer)
	if err := s.ExecuteBody(body, a, st); err != nil {
		log.Println(err)
	}
	n.Notify(subject.Bytes(), body.Bytes(), s.Conf.EmailFrom, s.Conf.SmtpHost)
	if n.Next == nil {
		return
	}
	s.AddNotification(AlertKey{Name: a.Name, Group: st.Group.String()}, n, time.Now().UTC())
}

func (s *Schedule) AddNotification(ak AlertKey, n *conf.Notification, started time.Time) {
	if s.Notifications == nil {
		s.Notifications = make(map[AlertKey]map[string]time.Time)
	}
	if s.Notifications[ak] == nil {
		s.Notifications[ak] = make(map[string]time.Time)
	}
	s.Notifications[ak][n.Name] = started
}

type AlertKey struct {
	Name  string
	Group string
}

func (a AlertKey) String() string {
	return a.Name + a.Group
}

type State struct {
	// Most recent event last.
	History      []Event
	Touched      time.Time
	Expr         string
	Group        opentsdb.TagSet
	Computations expr.Computations
	Subject      string
}

func (s *Schedule) Acknowledge(ak AlertKey) {
	s.Lock()
	delete(s.Notifications, ak)
	s.Unlock()
}

func (s *State) Touch() {
	s.Touched = time.Now().UTC()
}

// Appends status to the history if the status is different than the latest
// status. Returns the previous status.
func (s *State) Append(status Status) Status {
	last := s.Last()
	if len(s.History) == 0 || s.Last().Status != status {
		s.History = append(s.History, Event{status, time.Now().UTC()})
	}
	return last.Status
}

func (s *State) Last() Event {
	if len(s.History) == 0 {
		return Event{}
	}
	return s.History[len(s.History)-1]
}

type Event struct {
	Status Status
	Time   time.Time
}

type Status int

const (
	stNone Status = iota
	stNormal
	stWarning
	stCritical
	stUnknown
)

func (s Status) String() string {
	switch s {
	case stNormal:
		return "normal"
	case stWarning:
		return "warning"
	case stCritical:
		return "critical"
	case stUnknown:
		return "unknown"
	default:
		return "none"
	}
}
<|endoftext|>


<commit_before>package schema

import (
	"github.com/graphql-go/graphql"
	"github.com/graphql-go/relay"
	"github.com/mleonard87/merknera/repository"
)

var gameType *graphql.Object
var gameConnectionDefinition *relay.GraphQLConnectionDefinitions

func GameConnectionDefinition() *relay.GraphQLConnectionDefinitions {
	if gameConnectionDefinition == nil {
		gameConnectionDefinition = relay.ConnectionDefinitions(relay.ConnectionConfig{
			Name:     "Game",
			NodeType: GameType(),
		})
	}

	return gameConnectionDefinition
}

func GameType() *graphql.Object {
	if gameType == nil {
		gameType = graphql.NewObject(
			graphql.ObjectConfig{
				Name:        "Game",
				Description: "A game played between one or more bots depending on the game type.",
				Fields: graphql.Fields{
					"id": relay.GlobalIDField("Game", nil),
					"gameId": &graphql.Field{
						Type:        graphql.Int,
						Description: "The unique ID of the game.",
						Resolve: func(p graphql.ResolveParams) (interface{}, error) {
							if g, ok := p.Source.(repository.Game); ok {
								return g.Id, nil
							}
							return nil, nil
						},
					},
					"gameType": &graphql.Field{
						Type:        GameTypeType(),
						Description: "The mnemonic used to represent this game type.",
						Resolve: func(p graphql.ResolveParams) (interface{}, error) {
							if g, ok := p.Source.(repository.Game); ok {
								return g.GameType()
							}
							return nil, nil
						},
					},
					"players": &graphql.Field{
						Type:        graphql.NewList(GameBotType()),
						Description: "The bots playing this game against each other.",
						Resolve: func(p graphql.ResolveParams) (interface{}, error) {
							if g, ok := p.Source.(repository.Game); ok {
								return g.Players()
							}
							return nil, nil
						},
					},
					"status": &graphql.Field{
						Type:        graphql.String,
						Description: "The user-friendly name of this game type.",
						Resolve: func(p graphql.ResolveParams) (interface{}, error) {
							if g, ok := p.Source.(repository.Game); ok {
								return string(g.Status), nil
							}
							return nil, nil
						},
					},
					"moves": &graphql.Field{
						Type:        graphql.NewList(GameMoveType()),
						Description: "The moves played for this game, order by time ascending.",
						Resolve: func(p graphql.ResolveParams) (interface{}, error) {
							if g, ok := p.Source.(repository.Game); ok {
								return g.Moves()
							}
							return nil, nil
						},
					},
					"winningMove": &graphql.Field{
						Type:        GameMoveType(),
						Description: "The winning move of this game if the game is complete.",
						Resolve: func(p graphql.ResolveParams) (interface{}, error) {
							if g, ok := p.Source.(repository.Game); ok {
								wm, err := g.WinningMove()
								if err != nil {
									return nil, nil
								}
								if wm.Id == 0 {
									return nil, nil
								}
								return wm, nil
							}
							return nil, nil
						},
					},
				},
				Interfaces: []*graphql.Interface{
					nodeDefinitions.NodeInterface,
				},
			},
		)
	}

	return gameType
}
<commit_msg>Add a "totalCount" connection field to games.<commit_after>package schema

import (
	"github.com/graphql-go/graphql"
	"github.com/graphql-go/relay"
	"github.com/mleonard87/merknera/repository"
)

var gameType *graphql.Object
var gameConnectionDefinition *relay.GraphQLConnectionDefinitions

func GameConnectionDefinition() *relay.GraphQLConnectionDefinitions {
	if gameConnectionDefinition == nil {
		gameConnectionDefinition = relay.ConnectionDefinitions(relay.ConnectionConfig{
			Name:     "Game",
			NodeType: GameType(),
			ConnectionFields: graphql.Fields{
				"totalCount": &graphql.Field{
					Type:        graphql.Int,
					Description: "The total number of games.",
					Resolve: func(p graphql.ResolveParams) (interface{}, error) {
						games, err := repository.ListGames()
						if err != nil {
							return nil, err
						}
						return len(games), nil
					},
				},
			},
		})
	}

	return gameConnectionDefinition
}

func GameType() *graphql.Object {
	if gameType == nil {
		gameType = graphql.NewObject(
			graphql.ObjectConfig{
				Name:        "Game",
				Description: "A game played between one or more bots depending on the game type.",
				Fields: graphql.Fields{
					"id": relay.GlobalIDField("Game", nil),
					"gameId": &graphql.Field{
						Type:        graphql.Int,
						Description: "The unique ID of the game.",
						Resolve: func(p graphql.ResolveParams) (interface{}, error) {
							if g, ok := p.Source.(repository.Game); ok {
								return g.Id, nil
							}
							return nil, nil
						},
					},
					"gameType": &graphql.Field{
						Type:        GameTypeType(),
						Description: "The mnemonic used to represent this game type.",
						Resolve: func(p graphql.ResolveParams) (interface{}, error) {
							if g, ok := p.Source.(repository.Game); ok {
								return g.GameType()
							}
							return nil, nil
						},
					},
					"players": &graphql.Field{
						Type:        graphql.NewList(GameBotType()),
						Description: "The bots playing this game against each other.",
						Resolve: func(p graphql.ResolveParams) (interface{}, error) {
							if g, ok := p.Source.(repository.Game); ok {
								return g.Players()
							}
							return nil, nil
						},
					},
					"status": &graphql.Field{
						Type:        graphql.String,
						Description: "The user-friendly name of this game type.",
						Resolve: func(p graphql.ResolveParams) (interface{}, error) {
							if g, ok := p.Source.(repository.Game); ok {
								return string(g.Status), nil
							}
							return nil, nil
						},
					},
					"moves": &graphql.Field{
						Type:        graphql.NewList(GameMoveType()),
						Description: "The moves played for this game, order by time ascending.",
						Resolve: func(p graphql.ResolveParams) (interface{}, error) {
							if g, ok := p.Source.(repository.Game); ok {
								return g.Moves()
							}
							return nil, nil
						},
					},
					"winningMove": &graphql.Field{
						Type:        GameMoveType(),
						Description: "The winning move of this game if the game is complete.",
						Resolve: func(p graphql.ResolveParams) (interface{}, error) {
							if g, ok := p.Source.(repository.Game); ok {
								wm, err := g.WinningMove()
								if err != nil {
									return nil, nil
								}
								if wm.Id == 0 {
									return nil, nil
								}
								return wm, nil
							}
							return nil, nil
						},
					},
				},
				Interfaces: []*graphql.Interface{
					nodeDefinitions.NodeInterface,
				},
			},
		)
	}

	return gameType
}
<|endoftext|>


<commit_before>package goldb

import (
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/errors"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"github.com/syndtr/goleveldb/leveldb/util"
)

type Storage struct {
	Context
	dir string
	db  *leveldb.DB
	op  *opt.Options
	mx  sync.Mutex
}

func NewStorage(dir string, op *opt.Options) (s *Storage) {
	dir = strings.TrimSuffix(dir, "/")

	s = &Storage{
		dir: dir,
		op:  op,
	}

	if err := s.Open(); err != nil {
		if errors.IsCorrupted(err) {
			// try to recover files
			if err := s.Recover(); err != nil {
				log.Println("!!! db.Storage.Recover-ERROR: ", err)
			}
			if err := s.Open(); err != nil {
				panic(err)
			}
		} else {
			panic(err)
		}
	}
	return
}

func (s *Storage) Open() error {
	db, err := leveldb.OpenFile(s.dir, s.op)
	if err != nil {
		return err
	}
	s.db = db
	s.Context.qCtx = db
	return nil
}

func (s *Storage) Recover() error {
	if db, err := leveldb.RecoverFile(s.dir, nil); err != nil {
		return err
	} else {
		return db.Close()
	}
}

func (s *Storage) Close() error {
	if s.db != nil {
		if err := s.db.Close(); err != leveldb.ErrClosed {
			return err
		}
	}
	return nil
}

func (s *Storage) Drop() error {
	if err := s.Close(); err != nil {
		return err
	}
	return os.RemoveAll(s.dir)
}

func (s *Storage) Size() (size int64) {
	s.rmx.RLock()
	defer s.rmx.RUnlock()

	filepath.Walk(s.dir, func(_ string, info os.FileInfo, err error) error {
		if info != nil && !info.IsDir() {
			size += info.Size()
		}
		return err
	})
	return
}

func (s *Storage) Truncate() error {
	s.mx.Lock()
	defer s.mx.Unlock()

	s.rmx.Lock() // wait for all readers
	defer s.rmx.Unlock()

	if err := s.Drop(); err != nil {
		return err
	}
	return s.Open()
}

// Exec executes transaction.
// The executing transaction can be discard by methods tx.Fail(err) or by panic(err)
func (s *Storage) Exec(fn func(tx *Transaction)) (err error) {
	s.mx.Lock()
	defer s.mx.Unlock()

	t := &Transaction{}
	t.tr, t.err = s.db.OpenTransaction()
	t.Context.qCtx = t.tr
	t.Context.ReadOptions = s.ReadOptions
	t.Context.WriteOptions = s.WriteOptions

	defer func() {
		if e, _ := recover().(error); e != nil {
			t.Discard()
			err = e
		}
	}()
	if t.err != nil {
		return t.err
	}
	fn(t)
	if t.err == nil {
		t.Commit()
	} else {
		t.Discard()
	}
	return t.err
}

func (s *Storage) Vacuum() (err error) {
	s.mx.Lock()
	defer s.mx.Unlock()

	tmpDir := s.dir + ".reindex"
	oldDir := s.dir + ".old"

	defer os.RemoveAll(tmpDir)
	os.RemoveAll(tmpDir)
	os.RemoveAll(oldDir)

	dbOld := s.db
	dbNew, err := leveldb.OpenFile(tmpDir, s.op)
	if err != nil {
		return
	}

	iterator := dbOld.NewIterator(&util.Range{}, s.ReadOptions)

	var tr *leveldb.Transaction
	defer func() {
		iterator.Release()
		if err == nil {
			err = iterator.Error()
		}
		if tr != nil {
			tr.Discard()
		}
	}()
	for i := 0; iterator.Next(); i++ {
		if err = iterator.Error(); err != nil {
			return
		}
		if i%10000 == 0 {
			if tr != nil {
				if err = tr.Commit(); err != nil {
					return
				}
			}
			if tr, err = dbNew.OpenTransaction(); err != nil {
				return
			}
		}
		// put values to new DB
		key := iterator.Key()
		val := iterator.Value()
		if err = tr.Put(key, val, s.WriteOptions); err != nil {
			return
		}
	}
	if tr != nil {
		if err = tr.Commit(); err != nil {
			return
		}
		tr = nil
	}

	if err = dbNew.Close(); err != nil {
		return
	}

	s.rmx.Lock() // wait for all readers
	defer s.rmx.Unlock()

	if err = os.Rename(s.dir, oldDir); err != nil {
		return
	}
	if err = os.Rename(tmpDir, s.dir); err != nil {
		return
	}

	// reopen db
	dbNew, err = leveldb.OpenFile(s.dir, s.op)
	if err != nil {
		return
	}
	s.Context.qCtx = dbNew
	s.db = dbNew
	dbOld.Close()

	os.RemoveAll(oldDir)

	return
}

func (s *Storage) Put(key, data []byte) error {
	return s.Exec(func(tr *Transaction) {
		tr.Put(key, data)
	})
}

func (s *Storage) PutID(key []byte, id uint64) error {
	return s.Exec(func(tr *Transaction) {
		tr.PutID(key, id)
	})
}

func (s *Storage) PutInt(key []byte, num int64) error {
	return s.Exec(func(tr *Transaction) {
		tr.PutInt(key, num)
	})
}

func (s *Storage) PutVar(key []byte, v interface{}) error {
	return s.Exec(func(tr *Transaction) {
		tr.PutVar(key, v)
	})
}

func (s *Storage) Del(key []byte) error {
	return s.Exec(func(tr *Transaction) {
		tr.Del(key)
	})
}

func (s *Storage) RemoveByQuery(q *Query) error {
	return s.Exec(func(tr *Transaction) {
		tr.Fetch(q, func(key, value []byte) error {
			tr.Del(key)
			return nil
		})
	})
}
<commit_msg>refactor Vacuum<commit_after>package goldb

import (
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/errors"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"github.com/syndtr/goleveldb/leveldb/util"
)

type Storage struct {
	Context
	dir string
	db  *leveldb.DB
	op  *opt.Options
	mx  sync.Mutex
}

func NewStorage(dir string, op *opt.Options) (s *Storage) {
	dir = strings.TrimSuffix(dir, "/")

	s = &Storage{
		dir: dir,
		op:  op,
	}

	if err := s.Open(); err != nil {
		if errors.IsCorrupted(err) {
			// try to recover files
			if err := s.Recover(); err != nil {
				log.Println("!!! db.Storage.Recover-ERROR: ", err)
			}
			if err := s.Open(); err != nil {
				panic(err)
			}
		} else {
			panic(err)
		}
	}
	return
}

func (s *Storage) Open() error {
	db, err := leveldb.OpenFile(s.dir, s.op)
	if err != nil {
		return err
	}
	s.db = db
	s.Context.qCtx = db
	return nil
}

func (s *Storage) Recover() error {
	if db, err := leveldb.RecoverFile(s.dir, nil); err != nil {
		return err
	} else {
		return db.Close()
	}
}

func (s *Storage) Close() error {
	if s.db != nil {
		if err := s.db.Close(); err != leveldb.ErrClosed {
			return err
		}
	}
	return nil
}

func (s *Storage) Drop() error {
	if err := s.Close(); err != nil {
		return err
	}
	return os.RemoveAll(s.dir)
}

func (s *Storage) Size() (size int64) {
	s.rmx.RLock()
	defer s.rmx.RUnlock()

	filepath.Walk(s.dir, func(_ string, info os.FileInfo, err error) error {
		if info != nil && !info.IsDir() {
			size += info.Size()
		}
		return err
	})
	return
}

func (s *Storage) Truncate() error {
	s.mx.Lock()
	defer s.mx.Unlock()

	s.rmx.Lock() // wait for all readers
	defer s.rmx.Unlock()

	if err := s.Drop(); err != nil {
		return err
	}
	return s.Open()
}

// Exec executes transaction.
// The executing transaction can be discard by methods tx.Fail(err) or by panic(err)
func (s *Storage) Exec(fn func(tx *Transaction)) (err error) {
	s.mx.Lock()
	defer s.mx.Unlock()

	t := &Transaction{}
	t.tr, t.err = s.db.OpenTransaction()
	t.Context.qCtx = t.tr
	t.Context.ReadOptions = s.ReadOptions
	t.Context.WriteOptions = s.WriteOptions

	defer func() {
		if e, _ := recover().(error); e != nil {
			t.Discard()
			err = e
		}
	}()
	if t.err != nil {
		return t.err
	}
	fn(t)
	if t.err == nil {
		t.Commit()
	} else {
		t.Discard()
	}
	return t.err
}

func (s *Storage) Vacuum() (err error) {
	s.mx.Lock()
	defer s.mx.Unlock()

	tmpDir := s.dir + ".tmp"
	oldDir := s.dir + ".old"

	defer os.RemoveAll(tmpDir)
	os.RemoveAll(tmpDir)
	os.RemoveAll(oldDir)

	// copy db-data to new tmpDB
	if err = s.copyDataToNewDB(tmpDir); err != nil {
		return
	}

	s.rmx.Lock() // wait for all readers
	defer s.rmx.Unlock()

	// close old db
	if err = s.db.Close(); err != nil {
		return
	}

	// move db dirs
	if err = os.Rename(s.dir, oldDir); err != nil {
		return
	}
	if err = os.Rename(tmpDir, s.dir); err != nil {
		return
	}

	// open new db
	if err = s.Open(); err != nil {
		return
	}

	os.RemoveAll(oldDir)

	return
}

func (s *Storage) copyDataToNewDB(dir string) (err error) {
	db, err := leveldb.OpenFile(dir, s.op)
	if err != nil {
		return
	}
	defer db.Close()

	iterator := s.db.NewIterator(&util.Range{}, s.ReadOptions)
	defer iterator.Release()

	var tr *leveldb.Transaction
	defer func() {
		if err == nil {
			err = iterator.Error()
		}
		if tr != nil {
			tr.Discard()
		}
	}()
	for i := 0; iterator.Next(); i++ {
		if err = iterator.Error(); err != nil {
			return
		}
		if i%10000 == 0 {
			if tr != nil {
				if err = tr.Commit(); err != nil {
					return
				}
			}
			if tr, err = db.OpenTransaction(); err != nil {
				return
			}
		}
		// put values to new DB
		key := iterator.Key()
		val := iterator.Value()
		if err = tr.Put(key, val, s.WriteOptions); err != nil {
			return
		}
	}
	if tr != nil {
		if err = tr.Commit(); err != nil {
			return
		}
		tr = nil
	}
	return
}

func (s *Storage) Put(key, data []byte) error {
	return s.Exec(func(tr *Transaction) {
		tr.Put(key, data)
	})
}

func (s *Storage) PutID(key []byte, id uint64) error {
	return s.Exec(func(tr *Transaction) {
		tr.PutID(key, id)
	})
}

func (s *Storage) PutInt(key []byte, num int64) error {
	return s.Exec(func(tr *Transaction) {
		tr.PutInt(key, num)
	})
}

func (s *Storage) PutVar(key []byte, v interface{}) error {
	return s.Exec(func(tr *Transaction) {
		tr.PutVar(key, v)
	})
}

func (s *Storage) Del(key []byte) error {
	return s.Exec(func(tr *Transaction) {
		tr.Del(key)
	})
}

func (s *Storage) RemoveByQuery(q *Query) error {
	return s.Exec(func(tr *Transaction) {
		tr.Fetch(q, func(key, value []byte) error {
			tr.Del(key)
			return nil
		})
	})
}
<|endoftext|>


<commit_before>package filestore_util

import (
	errs "errors"
	"fmt"
	"io"

	. "github.com/ipfs/go-ipfs/filestore"

	cmds "github.com/ipfs/go-ipfs/commands"
	"github.com/ipfs/go-ipfs/core"
	//"gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
	k "github.com/ipfs/go-ipfs/blocks/key"
	//ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
	b "github.com/ipfs/go-ipfs/blocks/blockstore"
	node "github.com/ipfs/go-ipfs/merkledag"
	"github.com/ipfs/go-ipfs/pin"
)

type DeleteOpts struct {
	Direct     bool
	Force      bool
	IgnorePins bool
}

func Delete(req cmds.Request, out io.Writer, node *core.IpfsNode, fs *Datastore, opts DeleteOpts, keyList ...k.Key) error {
	keys := make(map[k.Key]struct{})
	for _, k := range keyList {
		keys[k] = struct{}{}
	}

	//
	// First check files
	//
	errors := false
	for _, k := range keyList {
		dagNode, dataObj, err := fsGetNode(k.DsKey(), fs)
		if err != nil {
			fmt.Fprintf(out, "%s: %s\n", k, err.Error())
			delete(keys, k)
			errors = true
			continue
		}
		if !opts.Direct && !dataObj.WholeFile() {
			fmt.Fprintf(out, "%s: part of another file, use --direct to delete\n", k)
			delete(keys, k)
			errors = true
			continue
		}
		if dagNode != nil && !opts.Direct {
			err = getChildren(out, dagNode, fs, node.Blockstore, keys)
			if err != nil {
				errors = true
			}
		}
	}
	if !opts.Force && errors {
		return errs.New("Errors during precheck.")
	}

	//
	// Now check pins
	//
	pinned := make(map[k.Key]pin.PinMode)
	if !opts.IgnorePins {
		walkPins(node.Pinning, fs, node.Blockstore, func(key k.Key, mode pin.PinMode) bool {
			_, ok := keys[key]
			if !ok {
				// Hack to make sure mangled hashes are unpinned
				// (see issue #2601)
				_, ok = keys[k.KeyFromDsKey(key.DsKey())]
			}
			if ok {
				if mode == pin.NotPinned {
					// an indirect pin
					fmt.Fprintf(out, "%s: indirectly pinned\n", key)
					errors = true
					return true
				} else {
					pinned[key] = mode
					return false
				}
			} else {
				if opts.Force {
					// do not recurse and thus do not check indirect pins
					return false
				} else {
					return true
				}
			}
		})
		if !opts.Force && errors {
			return errs.New("Errors during pin-check.")
		}
	}

	//
	//
	//
	for key, _ := range keys {
		err := fs.DeleteDirect(key.DsKey())
		if err != nil {
			fmt.Fprintf(out, "%s: %s\n", key, err.Error())
		}
		fmt.Fprintf(out, "deleted %s\n", key)
	}

	for key, mode := range pinned {
		stillExists, err := node.Blockstore.Has(key)
		if err != nil {
			fmt.Fprintf(out, "skipping pin %s: %s\n", err.Error())
			continue
		} else if stillExists {
			fmt.Fprintf(out, "skipping pin %s: object still exists outside filestore\n", key)
			continue
		}
		node.Pinning.RemovePinWithMode(key, mode)
		fmt.Fprintf(out, "unpinned %s\n", key)
	}
	if len(pinned) > 0 {
		err := node.Pinning.Flush()
		if err != nil {
			return err
		}
	}

	if errors {
		return errs.New("Errors deleting some keys.")
	}
	return nil
}

func getChildren(out io.Writer, node *node.Node, fs *Datastore, bs b.Blockstore, keys map[k.Key]struct{}) error {
	errors := false
	for _, link := range node.Links {
		key := k.Key(link.Hash)
		if _, ok := keys[key]; ok {
			continue
		}
		n, _, status := getNode(key.DsKey(), key, fs, bs)
		if AnError(status) {
			fmt.Fprintf(out, "%s: error retrieving key", key)
			errors = true
		}
		keys[key] = struct{}{}
		if n != nil {
			err := getChildren(out, n, fs, bs, keys)
			if err != nil {
				errors = true
			}
		}
	}
	if errors {
		return errs.New("Could net get all children.")
	}
	return nil
}
<commit_msg>"filestore rm": Do not delete blocks that are shared with another pinned node.<commit_after>package filestore_util

import (
	errs "errors"
	"fmt"
	"io"

	. "github.com/ipfs/go-ipfs/filestore"

	cmds "github.com/ipfs/go-ipfs/commands"
	"github.com/ipfs/go-ipfs/core"
	//"gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
	k "github.com/ipfs/go-ipfs/blocks/key"
	//ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
	b "github.com/ipfs/go-ipfs/blocks/blockstore"
	node "github.com/ipfs/go-ipfs/merkledag"
	"github.com/ipfs/go-ipfs/pin"
)

type DeleteOpts struct {
	Direct     bool
	Force      bool
	IgnorePins bool
}

type delInfo int

const (
	DirectlySpecified   delInfo = 1
	IndirectlySpecified delInfo = 2
)

func Delete(req cmds.Request, out io.Writer, node *core.IpfsNode, fs *Datastore, opts DeleteOpts, keyList ...k.Key) error {
	keys := make(map[k.Key]delInfo)
	for _, k := range keyList {
		keys[k] = DirectlySpecified
	}

	//
	// First check files
	//
	errors := false
	for _, k := range keyList {
		dagNode, dataObj, err := fsGetNode(k.DsKey(), fs)
		if err != nil {
			fmt.Fprintf(out, "%s: %s\n", k, err.Error())
			delete(keys, k)
			errors = true
			continue
		}
		if !opts.Direct && !dataObj.WholeFile() {
			fmt.Fprintf(out, "%s: part of another file, use --direct to delete\n", k)
			delete(keys, k)
			errors = true
			continue
		}
		if dagNode != nil && !opts.Direct {
			err = getChildren(out, dagNode, fs, node.Blockstore, keys)
			if err != nil {
				errors = true
			}
		}
	}
	if !opts.Force && errors {
		return errs.New("Errors during precheck.")
	}

	//
	// Now check pins
	//
	pinned := make(map[k.Key]pin.PinMode)
	if !opts.IgnorePins {
		walkPins(node.Pinning, fs, node.Blockstore, func(key k.Key, mode pin.PinMode) bool {
			dm, ok := keys[key]
			if !ok {
				// Hack to make sure mangled hashes are unpinned
				// (see issue #2601)
				dm, ok = keys[k.KeyFromDsKey(key.DsKey())]
			}
			if ok {
				if mode == pin.NotPinned && dm == DirectlySpecified {
					// an indirect pin
					fmt.Fprintf(out, "%s: indirectly pinned\n", key)
					if !opts.Force {
						errors = true
					}
					return true
				} else if mode == pin.NotPinned && dm == IndirectlySpecified {
					fmt.Fprintf(out, "skipping indirectly pinned block: %s\n", key)
					delete(keys, key)
					return true
				} else {
					pinned[key] = mode
					return false
				}
			} else {
				if opts.Force && opts.Direct {
					// do not recurse and thus do not check indirect pins
					return false
				} else {
					return true
				}
			}
		})
		if !opts.Force && errors {
			return errs.New("Errors during pin-check.")
		}
	}

	//
	//
	//
	for key, _ := range keys {
		err := fs.DeleteDirect(key.DsKey())
		if err != nil {
			fmt.Fprintf(out, "%s: %s\n", key, err.Error())
		}
		fmt.Fprintf(out, "deleted %s\n", key)
	}

	for key, mode := range pinned {
		stillExists, err := node.Blockstore.Has(key)
		if err != nil {
			fmt.Fprintf(out, "skipping pin %s: %s\n", err.Error())
			continue
		} else if stillExists {
			fmt.Fprintf(out, "skipping pin %s: object still exists outside filestore\n", key)
			continue
		}
		node.Pinning.RemovePinWithMode(key, mode)
		fmt.Fprintf(out, "unpinned %s\n", key)
	}
	if len(pinned) > 0 {
		err := node.Pinning.Flush()
		if err != nil {
			return err
		}
	}

	if errors {
		return errs.New("Errors deleting some keys.")
	}
	return nil
}

func getChildren(out io.Writer, node *node.Node, fs *Datastore, bs b.Blockstore, keys map[k.Key]delInfo) error {
	errors := false
	for _, link := range node.Links {
		key := k.Key(link.Hash)
		if _, ok := keys[key]; ok {
			continue
		}
		n, _, status := getNode(key.DsKey(), key, fs, bs)
		if AnError(status) {
			fmt.Fprintf(out, "%s: error retrieving key", key)
			errors = true
		}
		keys[key] = IndirectlySpecified
		if n != nil {
			err := getChildren(out, n, fs, bs, keys)
			if err != nil {
				errors = true
			}
		}
	}
	if errors {
		return errs.New("Could net get all children.")
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html"
	"net/http"
)

func serve1() {
	http.HandleFunc("/user", func(w http.ResponseWriter, r *http.Request) {
		r.ParseForm()
		username := r.Form.Get("username")
		if !isValidUsername(username) {
			// BAD: a request parameter is incorporated without validation into the response
			fmt.Fprintf(w, "%q is an unknown user", html.EscapeString(username))
		} else {
			// TODO: do something exciting
		}
	})
	http.ListenAndServe(":80", nil)
}
<commit_msg>Update bad / good message for CWE 079<commit_after>package main

import (
	"fmt"
	"html"
	"net/http"
)

func serve1() {
	http.HandleFunc("/user", func(w http.ResponseWriter, r *http.Request) {
		r.ParseForm()
		username := r.Form.Get("username")
		if !isValidUsername(username) {
			// GOOD: a request parameter is escaped before being put into the response
			fmt.Fprintf(w, "%q is an unknown user", html.EscapeString(username))
		} else {
			// TODO: do something exciting
		}
	})
	http.ListenAndServe(":80", nil)
}
<|endoftext|>


<commit_before>package phputil

import (
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/sha512"
	"encoding/hex"
	"fmt"
	"hash/crc32"
	"html"
	"io"
	"log"
	"os"
	"strings"
	"time"

	"golang.org/x/crypto/blowfish"
)

func AddcSlashes(s string, c string) string {
	r := strings.Split(c, "..")
	var res string
	min := []rune(r[0])
	max := []rune(r[len(r)-1])
	for _, v := range s {
		if min[0] <= v && v <= max[0] {
			res += "\\" + string(v)
		} else {
			res += string(v)
		}
	}

	return res
}

func AddSlashes(s string) string {
	r := strings.NewReplacer("'", "\\'", "\"", "\\\"", "\\", "\\\\")
	return r.Replace(s)
}

func Bin2Hex(s string) string {
	b := []byte(s[:])
	return hex.EncodeToString(b)
}

func Chop(s ...string) string {
	return trim(s, 1)
}

func Chr(i int) string {
	for i < 0 {
		i += 256
	}
	i %= 256
	return string(rune(i))
}

func ChunkSplit(s string, l int, sep string) string {
	if len(s) < l {
		return s
	}

	res := s[:l] + sep
	tail := s[l:]

	for len(tail) > l {
		res += tail[:l] + sep
		tail = tail[l:]
	}
	res += tail

	return res
}

// TODO
func ConvertCyrString(s string, from string, to string) string {
	return ""
}

// TODO
func ConvertUudecode(s string) string {
	return ""
}

// TODO
func ConvertUuencode(s string) string {
	return ""
}

func CountChars(s string, i int) map[int]int {

	r := countChars(s)

	switch i {
	case 0:
		return r
	case 1:
		for n := 0; n < 255; n++ {
			if r[n] == 0 {
				delete(r, n)
			}
		}
		return r
	case 2:
		for n := 0; n < 255; n++ {
			if r[n] > 0 {
				delete(r, n)
			}
		}
		return r
	}

	return r
}

func CountChars34(s string, i int) string {
	r := countChars(s)
	var res string
	switch i {
	case 3:
		for n := 0; n < 255; n++ {
			if r[n] > 0 {
				res += string(n)
			}
		}
		return res
	case 4:
		for n := 0; n < 255; n++ {
			if r[n] == 0 {
				res += string(n)
			}
		}
		return res
	}
	return res
}

func countChars(s string) map[int]int {
	r := make(map[int]int)
	for i := 0; i < 255; i++ {
		r[i] = 0
	}

	for _, v := range s {
		r[int(v)]++
	}
	return r
}

func Crc32(s string) uint32 {
	return crc32.ChecksumIEEE([]byte(s))
}

// TODO
func Crypt(s string, salt string) string {
	var res string

	switch len(salt) {
	/**
	case 2:
		// CRYPT_STD_DES
		c, _ := des.NewCipher([]byte(salt))
		encrypted := make([]byte, des.BlockSize)
		// DES で暗号化をおこなう
		c.Encrypt(encrypted, []byte(s))
		return string(encrypted)
	case 9:
	// CRYPT_EXT_DES
	**/
	case 12:
		// CRYPT_MD5
		h := md5.New()
		h.Write([]byte(s))
		return hex.EncodeToString(h.Sum(nil))
	case 22:
		// TODO: PADDING salt. CRYPT_BLOWFISH
		cipher, _ := blowfish.NewSaltedCipher([]byte("key"), []byte(salt))
		if len(s)%blowfish.BlockSize != 0 {
			os.Exit(1)
		}
		var encrypted []byte
		cipher.Encrypt(encrypted, []byte(s))
		return string(encrypted)
	case 16:
		// TODO: if condition. CRYPT_SHA256
		if true {
			c := sha256.Sum256([]byte(s))
			return hex.EncodeToString(c[:])
		} else {
			// CRYPT_SHA51
			c := sha512.Sum512([]byte(s))
			return hex.EncodeToString(c[:])
		}

	}
	return res
}

func echo(s string) {
	fmt.Print(s)
}

func Explode(d string, s string, l int) []string {
	return strings.SplitN(s, d, l)
}

func Fprintf(w io.Writer, f string, a ...interface{}) int {
	n, _ := fmt.Fprintf(w, f, a[:]...)
	return n
}

// TODO
func GetHtmlTranslationTable(t int, f int, e string) map[string]string {
	var res map[string]string
	return res
}

// TODO
func Hebrev(s string, m int) string {
	return ""
}

// TODO
func Hebrevc(s string, m int) string {
	return ""
}

func Hex2Bin(s string) string {
	b, _ := hex.DecodeString(s)
	return string(b)
}

func HtmlEntityDecode(s string) string {
	return html.UnescapeString(s)
}

func HtmlEntities(s string) string {
	return html.EscapeString(s)
}

// TODO Fix. It's not strict.
func HtmlspecialcharsDecode(s string) string {
	return html.UnescapeString(s)
}

// TODO Fix. It's not strict.
func Htmlspecialchars(s string) string {
	return html.EscapeString(s)
}

func Implode(d string, s []string) string {
	return strings.Join(s, d)
}

func Join(d string, s []string) string {
	return Implode(d, s)
}

func Levenshtein(s string, t string) int {
	d := make([][]int, len(s)+1)
	for i := range d {
		d[i] = make([]int, len(t)+1)
	}
	for i := range d {
		d[i][0] = i
	}
	for j := range d[0] {
		d[0][j] = j
	}
	for j := 1; j <= len(t); j++ {
		for i := 1; i <= len(s); i++ {
			if s[i-1] == t[j-1] {
				d[i][j] = d[i-1][j-1]
			} else {
				min := d[i-1][j]
				if d[i][j-1] < min {
					min = d[i][j-1]
				}
				if d[i-1][j-1] < min {
					min = d[i-1][j-1]
				}
				d[i][j] = min + 1
			}
		}
	}
	return d[len(s)][len(t)]
}

// TODO
func Localeconv() {

}

func Ltrim(s ...string) string {
	return trim(s, 2)
}

func Md5File(s string) string {
	f, err := os.Open(s)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	h := md5.New()
	if _, err := io.Copy(h, f); err != nil {
		log.Fatal(err)
	}

	return hex.EncodeToString(h.Sum(nil))
}

func Md5(s string) string {
	h := md5.New()
	io.WriteString(h, s)
	return hex.EncodeToString(h.Sum(nil))
}

// TODO: http://php.net/manual/ja/function.metaphone.php
func Metaphone(s string) string {
	return ""
}

// TODO: http://php.net/manual/ja/function.money-format.php
func MoneyFormat(f string, m int) string {
	return ""
}

// TODO: http://php.net/manual/ja/function.nl-langinfo.php
func NlLanginfo() string {
	return ""
}

func Nl2br(s string) string {
	r := strings.NewReplacer("\n\r", "\n", "\r\n", "\n", "\r", "\n", "\n", "
\n")
	return r.Replace(s)
}

// TODO:
func NumberFormat(s string) string {
	return ""
}

func Ord(s string) int {
	return int(s[0])
}

func ParseStr(s string) map[string][]string {
	res := make(map[string][]string)
	// key=v
	queries := strings.Split(s, "&")
	for _, v := range queries {
		// 0:key , 1:v
		query := strings.Split(v, "=")
		if t := strings.Index(query[0], "[]"); t != -1 {
			// ak = key
			ak := query[0][:t]
			vv := strings.Replace(query[1], "+", " ", -1)
			res[ak] = append(res[ak], vv)
		} else {
			ak := query[0]
			res[ak] = append(res[ak], query[1])
		}
	}

	return res
}

func Print(s string) int {
	fmt.Print(s)
	return 1
}

func Printf(f string, a ...interface{}) {
	fmt.Printf(f, a...)
}

// TODO: http://php.net/manual/ja/function.quoted-printable-decode.php
func QuotedPrintableDecode(s string) string {
	return ""
}

// TODO: http://php.net/manual/ja/function.quoted-printable-encode.php
func QuotedPrintableEncode(s string) string {
	return ""
}

func Quotemeta(s string) string {
	r := strings.NewReplacer(
		`.`, `\.`,
		`\`, `\\`,
		`+`, `\+`,
		`*`, `\*`,
		`?`, `\?`,
		`[`, `\[`,
		`^`, `\^`,
		`]`, `\]`,
		`(`, `\(`,
		`$`, `\$`,
		`)`, `\)`,
	)
	return r.Replace(s)
}

func Rtrim(s ...string) string {
	return trim(s, 1)
}

// TODO: http://php.net/manual/ja/function.setlocale.php
func SetLocate(i int, l string) {
	loc := time.FixedZone(l, 9*60*60)
	time.Local = loc
}

// TODO
func Sha1File(s string) string {
	f, err := os.Open(s)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	h := sha1.New()
	if _, err := io.Copy(h, f); err != nil {
		log.Fatal(err)
	}
	return hex.EncodeToString(h.Sum(nil))
}

func Sha1(s string) string {
	h := sha1.New()
	h.Write([]byte(s))
	return hex.EncodeToString(h.Sum(nil))
}

// TODO http://php.net/manual/ja/function.similar-text.php
func SimilarText(f string, s string) int {
	return 0
}

func trimfunc(i int) func(s string, d string) string {
	if i == 1 {
		return func(s string, d string) string {
			return strings.TrimSuffix(s, d)
		}
	} else if i == 2 {
		return func(s string, d string) string {
			return strings.TrimPrefix(s, d)
		}
	} else {
		// TODO
		return func(s string, d string) string {
			return strings.TrimPrefix(s, d)
		}
	}
}

func trim(s []string, i int) string {
	trims := trimfunc(i)
	if len(s) == 2 {
		r := strings.Split(s[1], "..")
		if len(r) == 2 {
			min := []rune(r[0])
			max := []rune(r[len(r)-1])
			str := s[0]
			t := len(str)
			for {
				for i := min[0]; i <= max[0]; i++ {
					str = trims(str, string(i))
				}
				if t == len(str) {
					break
				}
				t = len(str)
			}
			return str
		}
		return trims(s[0], s[1])
	}

	r := s[0]
	t := len(r)
	suffix := [5]string{" ", "\t", "\n", "\r", "\x0B"}

	for {
		for _, v := range suffix {
			r = trims(r, v)
		}
		if t == len(r) {
			break
		}
		t = len(r)
	}

	return r
}

func Ucfirst(s string) string {
	first := strings.ToUpper(string(s[0]))
	return first + s[1:]
}

func Lcfirst(s string) string {
	first := strings.ToLower(string(s[0]))
	return first + s[1:]
}

func Ucwords(p ...string) string {
	d := " "
	if len(p) > 1 {
		d = p[1]
	}
	words := strings.Split(p[0], d)
	var res string

	if len(words) > 1 {
		for i := 0; i < len(words); i++ {
			res += Ucfirst(words[i]) + d
		}
		res = strings.TrimSuffix(res, d)
	} else {
		res = p[0]
	}

	return res
}
<commit_msg>soudex<commit_after>package phputil

import (
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/sha512"
	"encoding/hex"
	"fmt"
	"hash/crc32"
	"html"
	"io"
	"log"
	"os"
	"strings"
	"time"

	"golang.org/x/crypto/blowfish"
)

func AddcSlashes(s string, c string) string {
	r := strings.Split(c, "..")
	var res string
	min := []rune(r[0])
	max := []rune(r[len(r)-1])
	for _, v := range s {
		if min[0] <= v && v <= max[0] {
			res += "\\" + string(v)
		} else {
			res += string(v)
		}
	}

	return res
}

func AddSlashes(s string) string {
	r := strings.NewReplacer("'", "\\'", "\"", "\\\"", "\\", "\\\\")
	return r.Replace(s)
}

func Bin2Hex(s string) string {
	b := []byte(s[:])
	return hex.EncodeToString(b)
}

func Chop(s ...string) string {
	return trim(s, 1)
}

func Chr(i int) string {
	for i < 0 {
		i += 256
	}
	i %= 256
	return string(rune(i))
}

func ChunkSplit(s string, l int, sep string) string {
	if len(s) < l {
		return s
	}

	res := s[:l] + sep
	tail := s[l:]

	for len(tail) > l {
		res += tail[:l] + sep
		tail = tail[l:]
	}
	res += tail

	return res
}

// TODO
func ConvertCyrString(s string, from string, to string) string {
	return ""
}

// TODO
func ConvertUudecode(s string) string {
	return ""
}

// TODO
func ConvertUuencode(s string) string {
	return ""
}

func CountChars(s string, i int) map[int]int {

	r := countChars(s)

	switch i {
	case 0:
		return r
	case 1:
		for n := 0; n < 255; n++ {
			if r[n] == 0 {
				delete(r, n)
			}
		}
		return r
	case 2:
		for n := 0; n < 255; n++ {
			if r[n] > 0 {
				delete(r, n)
			}
		}
		return r
	}

	return r
}

func CountChars34(s string, i int) string {
	r := countChars(s)
	var res string
	switch i {
	case 3:
		for n := 0; n < 255; n++ {
			if r[n] > 0 {
				res += string(n)
			}
		}
		return res
	case 4:
		for n := 0; n < 255; n++ {
			if r[n] == 0 {
				res += string(n)
			}
		}
		return res
	}
	return res
}

func countChars(s string) map[int]int {
	r := make(map[int]int)
	for i := 0; i < 255; i++ {
		r[i] = 0
	}

	for _, v := range s {
		r[int(v)]++
	}
	return r
}

func Crc32(s string) uint32 {
	return crc32.ChecksumIEEE([]byte(s))
}

// TODO
func Crypt(s string, salt string) string {
	var res string

	switch len(salt) {
	/**
	case 2:
		// CRYPT_STD_DES
		c, _ := des.NewCipher([]byte(salt))
		encrypted := make([]byte, des.BlockSize)
		// DES で暗号化をおこなう
		c.Encrypt(encrypted, []byte(s))
		return string(encrypted)
	case 9:
	// CRYPT_EXT_DES
	**/
	case 12:
		// CRYPT_MD5
		h := md5.New()
		h.Write([]byte(s))
		return hex.EncodeToString(h.Sum(nil))
	case 22:
		// TODO: PADDING salt. CRYPT_BLOWFISH
		cipher, _ := blowfish.NewSaltedCipher([]byte("key"), []byte(salt))
		if len(s)%blowfish.BlockSize != 0 {
			os.Exit(1)
		}
		var encrypted []byte
		cipher.Encrypt(encrypted, []byte(s))
		return string(encrypted)
	case 16:
		// TODO: if condition. CRYPT_SHA256
		if true {
			c := sha256.Sum256([]byte(s))
			return hex.EncodeToString(c[:])
		} else {
			// CRYPT_SHA51
			c := sha512.Sum512([]byte(s))
			return hex.EncodeToString(c[:])
		}

	}
	return res
}

func echo(s string) {
	fmt.Print(s)
}

func Explode(d string, s string, l int) []string {
	return strings.SplitN(s, d, l)
}

func Fprintf(w io.Writer, f string, a ...interface{}) int {
	n, _ := fmt.Fprintf(w, f, a[:]...)
	return n
}

// TODO
func GetHtmlTranslationTable(t int, f int, e string) map[string]string {
	var res map[string]string
	return res
}

// TODO
func Hebrev(s string, m int) string {
	return ""
}

// TODO
func Hebrevc(s string, m int) string {
	return ""
}

func Hex2Bin(s string) string {
	b, _ := hex.DecodeString(s)
	return string(b)
}

func HtmlEntityDecode(s string) string {
	return html.UnescapeString(s)
}

func HtmlEntities(s string) string {
	return html.EscapeString(s)
}

// TODO Fix. It's not strict.
func HtmlspecialcharsDecode(s string) string {
	return html.UnescapeString(s)
}

// TODO Fix. It's not strict.
func Htmlspecialchars(s string) string {
	return html.EscapeString(s)
}

func Implode(d string, s []string) string {
	return strings.Join(s, d)
}

func Join(d string, s []string) string {
	return Implode(d, s)
}

func Levenshtein(s string, t string) int {
	d := make([][]int, len(s)+1)
	for i := range d {
		d[i] = make([]int, len(t)+1)
	}
	for i := range d {
		d[i][0] = i
	}
	for j := range d[0] {
		d[0][j] = j
	}
	for j := 1; j <= len(t); j++ {
		for i := 1; i <= len(s); i++ {
			if s[i-1] == t[j-1] {
				d[i][j] = d[i-1][j-1]
			} else {
				min := d[i-1][j]
				if d[i][j-1] < min {
					min = d[i][j-1]
				}
				if d[i-1][j-1] < min {
					min = d[i-1][j-1]
				}
				d[i][j] = min + 1
			}
		}
	}
	return d[len(s)][len(t)]
}

// TODO
func Localeconv() {

}

func Ltrim(s ...string) string {
	return trim(s, 2)
}

func Md5File(s string) string {
	f, err := os.Open(s)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	h := md5.New()
	if _, err := io.Copy(h, f); err != nil {
		log.Fatal(err)
	}

	return hex.EncodeToString(h.Sum(nil))
}

func Md5(s string) string {
	h := md5.New()
	io.WriteString(h, s)
	return hex.EncodeToString(h.Sum(nil))
}

// TODO: http://php.net/manual/ja/function.metaphone.php
func Metaphone(s string) string {
	return ""
}

// TODO: http://php.net/manual/ja/function.money-format.php
func MoneyFormat(f string, m int) string {
	return ""
}

// TODO: http://php.net/manual/ja/function.nl-langinfo.php
func NlLanginfo() string {
	return ""
}

func Nl2br(s string) string {
	r := strings.NewReplacer("\n\r", "\n", "\r\n", "\n", "\r", "\n", "\n", "
\n")
	return r.Replace(s)
}

// TODO:
func NumberFormat(s string) string {
	return ""
}

func Ord(s string) int {
	return int(s[0])
}

func ParseStr(s string) map[string][]string {
	res := make(map[string][]string)
	// key=v
	queries := strings.Split(s, "&")
	for _, v := range queries {
		// 0:key , 1:v
		query := strings.Split(v, "=")
		if t := strings.Index(query[0], "[]"); t != -1 {
			// ak = key
			ak := query[0][:t]
			vv := strings.Replace(query[1], "+", " ", -1)
			res[ak] = append(res[ak], vv)
		} else {
			ak := query[0]
			res[ak] = append(res[ak], query[1])
		}
	}

	return res
}

func Print(s string) int {
	fmt.Print(s)
	return 1
}

func Printf(f string, a ...interface{}) {
	fmt.Printf(f, a...)
}

// TODO: http://php.net/manual/ja/function.quoted-printable-decode.php
func QuotedPrintableDecode(s string) string {
	return ""
}

// TODO: http://php.net/manual/ja/function.quoted-printable-encode.php
func QuotedPrintableEncode(s string) string {
	return ""
}

func Quotemeta(s string) string {
	r := strings.NewReplacer(
		`.`, `\.`,
		`\`, `\\`,
		`+`, `\+`,
		`*`, `\*`,
		`?`, `\?`,
		`[`, `\[`,
		`^`, `\^`,
		`]`, `\]`,
		`(`, `\(`,
		`$`, `\$`,
		`)`, `\)`,
	)
	return r.Replace(s)
}

func Rtrim(s ...string) string {
	return trim(s, 1)
}

// TODO: http://php.net/manual/ja/function.setlocale.php
func SetLocate(i int, l string) {
	loc := time.FixedZone(l, 9*60*60)
	time.Local = loc
}

// TODO
func Sha1File(s string) string {
	f, err := os.Open(s)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	h := sha1.New()
	if _, err := io.Copy(h, f); err != nil {
		log.Fatal(err)
	}
	return hex.EncodeToString(h.Sum(nil))
}

func Sha1(s string) string {
	h := sha1.New()
	h.Write([]byte(s))
	return hex.EncodeToString(h.Sum(nil))
}

// TODO http://php.net/manual/ja/function.similar-text.php
func SimilarText(f string, s string) int {
	return 0
}

// TODO http://php.net/manual/ja/function.soundex.php
func Soundex(s string) string {
	return ""
}

func trimfunc(i int) func(s string, d string) string {
	if i == 1 {
		return func(s string, d string) string {
			return strings.TrimSuffix(s, d)
		}
	} else if i == 2 {
		return func(s string, d string) string {
			return strings.TrimPrefix(s, d)
		}
	} else {
		// TODO
		return func(s string, d string) string {
			return strings.TrimPrefix(s, d)
		}
	}
}

func trim(s []string, i int) string {
	trims := trimfunc(i)
	if len(s) == 2 {
		r := strings.Split(s[1], "..")
		if len(r) == 2 {
			min := []rune(r[0])
			max := []rune(r[len(r)-1])
			str := s[0]
			t := len(str)
			for {
				for i := min[0]; i <= max[0]; i++ {
					str = trims(str, string(i))
				}
				if t == len(str) {
					break
				}
				t = len(str)
			}
			return str
		}
		return trims(s[0], s[1])
	}

	r := s[0]
	t := len(r)
	suffix := [5]string{" ", "\t", "\n", "\r", "\x0B"}

	for {
		for _, v := range suffix {
			r = trims(r, v)
		}
		if t == len(r) {
			break
		}
		t = len(r)
	}

	return r
}

func Ucfirst(s string) string {
	first := strings.ToUpper(string(s[0]))
	return first + s[1:]
}

func Lcfirst(s string) string {
	first := strings.ToLower(string(s[0]))
	return first + s[1:]
}

func Ucwords(p ...string) string {
	d := " "
	if len(p) > 1 {
		d = p[1]
	}
	words := strings.Split(p[0], d)
	var res string

	if len(words) > 1 {
		for i := 0; i < len(words); i++ {
			res += Ucfirst(words[i]) + d
		}
		res = strings.TrimSuffix(res, d)
	} else {
		res = p[0]
	}

	return res
}
<|endoftext|>