<commit_before>package build

import (
	"fmt"
	"neon/util"
	"reflect"
	"strings"
)

// character for expressions
const (
	// character that start expressions
	CHAR_EXPRESSION = "="
	// expression curly brace
	CURLY_EXPRESSION = "{"
	// tag separator
	TAG_SEPARATOR = " "
	// field might not be provided
	FIELD_OPTIONAL = "optional"
	// field is a file name that is expanded for user home
	FIELD_FILE = "file"
	// field is an expression
	FIELD_EXPRESSION = "expression"
	// field should be rapped in a slice of its type
	FIELD_WRAP = "wrap"
	// field is a list of steps
	FIELD_STEPS = "steps"
	// field has a different name
	FIELD_NAME = "name"
)

// Map that gives constructor for given task name
var TaskMap = make(map[string]TaskDesc)

// AddTask adds a task in the map:
// - task: description of the task
func AddTask(task TaskDesc) {
	if _, ok := TaskMap[task.Name]; ok {
		panic(fmt.Errorf("task '%s' already defined", task.Name))
	}
	TaskMap[task.Name] = task
}

// A task descriptor is made of a task constructor and an help string
type TaskDesc struct {
	Name string
	Args reflect.Type
	Func TaskFunc
	Help string
}

// Type for task arguments as parsed in build file
type TaskArgs map[interface{}]interface{}

// Type for task function called to run task
type TaskFunc func(ctx *Context, args interface{}) error

// Validate task arguments against task arguments definition
// - args: task arguments parsed in build file
// - typ: type of the arguments
// Return: an error (detailing the fault) if arguments are illegal
func ValidateTaskArgs(args TaskArgs, typ reflect.Type) error {
	if typ.Kind() != reflect.Struct {
		return fmt.Errorf("params must be a pointer on a struct")
	}
	var fields []string
	// iterate on fields of the parameters types and check argument types
	for i := 0; i < typ.NumField(); i++ {
		field := typ.Field(i)
		name := strings.ToLower(field.Name)
		if field.Tag.Get(FIELD_NAME) != "" {
			name = field.Tag.Get(FIELD_NAME)
		}
		fields = append(fields, name)
		// check field is not missing
		if _, ok := args[name]; !ok {
			if !FieldIs(field, FIELD_OPTIONAL) {
				return fmt.Errorf("missing mandatory field '%s'", name)
			}
		}
		value := args[name]
		// parse steps fields
		if FieldIs(field, "steps") && value != nil {
			steps, err := NewSteps(value)
			if err != nil {
				return fmt.Errorf("parsing field '%s': %v", name, err)
			}
			args[name] = steps
		}
		// check field type
		if !CheckType(field, value) {
			return fmt.Errorf("field '%s' must be of type '%s' ('%s' provided)",
				name, field.Type, reflect.TypeOf(value))
		}
	}
	// check that we don't have unknown args
	if !(typ.NumField() == 0 && len(args) == 1) {
		for name := range args {
			found := false
			for _, n := range fields {
				if n == name {
					found = true
					continue
				}
			}
			if !found {
				return fmt.Errorf("unknown parameter '%s'", name)
			}
		}
	}
	return nil
}

// CheckType checks that given value is compatible with field type
// - field: the field of the parameters as reflect.StructField
// - value: the value of the argument
// Return: a bool that tells if type is OK
func CheckType(field reflect.StructField, value interface{}) bool {
	valueType := reflect.TypeOf(value)
	// if field is optional and argument nil, it's OK
	if FieldIs(field, FIELD_OPTIONAL) && value == nil {
		return true
	}
	// if argument is an expression it's OK whatever the type
	if valueType.Kind() == reflect.String &&
		(IsExpression(value.(string)) ||
			FieldIs(field, FIELD_EXPRESSION)) {
		return true
	}
	// if type of field is slice of the type of the argument and wrap, it's OK
	if field.Type.Kind() == reflect.Slice &&
		reflect.SliceOf(valueType) == field.Type &&
		FieldIs(field, FIELD_WRAP) {
		return true
	}
	// if type is slice and argument is steps, it's OK
	if field.Type.Kind() == reflect.Slice &&
		FieldIs(field, FIELD_STEPS) {
		return true
	}
	// check that value is of given type
	return IsValueOfType(value, field.Type)
}

// IsValueOfType tells if a value is of given type
// - value: the value to test as an interface{}
// - type: the type to check as a reflect.Type
// Return: a bool telling if value is of given type
func IsValueOfType(value interface{}, typ reflect.Type) bool {
	// if value is of given type it's true
	if reflect.TypeOf(value) == typ {
		return true
	}
	// if value and type are slices, test elements
	if reflect.TypeOf(value).Kind() == reflect.Slice && typ.Kind() == reflect.Slice {
		return IsValueOfType(reflect.ValueOf(value).Index(0).Interface(), typ.Elem())
	}
	// if value and type are maps, test key and value
	if reflect.TypeOf(value).Kind() == reflect.Map && typ.Kind() == reflect.Map {
		key := reflect.ValueOf(value).MapKeys()[0].Interface()
		val := reflect.ValueOf(value).MapIndex(reflect.ValueOf(key)).Interface()
		return IsValueOfType(key, typ.Key()) && IsValueOfType(val, typ.Elem())
	}
	// else return false
	return false
}

// Build task arguments from task params and return it
// - args: task arguments parsed in build file
// - typ: the type of the arguments
// - context: the build context to evaluate arguments into
// Return:
// - result: as an interface{}
// - error: if something went wrong
func EvaluateTaskArgs(args TaskArgs, typ reflect.Type, context *Context) (interface{}, error) {
	var err error
	value := reflect.New(typ).Elem()
	for i := 0; i < value.NumField(); i++ {
		name := strings.ToLower(typ.Field(i).Name)
		field := typ.Field(i)
		if field.Tag.Get(FIELD_NAME) != "" {
			name = field.Tag.Get(FIELD_NAME)
		}
		if args[name] != nil {
			val := args[name]
			// evaluate expressions in context
			if reflect.TypeOf(val).Kind() == reflect.String &&
				(IsExpression(val.(string)) || FieldIs(field, FIELD_EXPRESSION)) {
				str := val.(string)
				if IsExpression(str) {
					str = str[1:]
				}
				val, err = context.EvaluateExpression(str)
				if err != nil {
					return nil, err
				}
				expected := field.Type
				actual := reflect.TypeOf(val)
				if actual != expected {
					// we accept type check if expected is slice of interfaces and actual is slice
					if !(expected.Kind() == reflect.Slice && actual.Kind() == reflect.Slice) {
						return nil, fmt.Errorf("bad expression return type, expected '%v' but '%v' was returned",
							expected, actual)
					}
				}
			}
			// evaluate arguments
			val, err = context.EvaluateObject(val)
			if err != nil {
				return nil, err
			}
			// evaluate strings to replace "={expression}" with its value
			if reflect.TypeOf(val).Kind() == reflect.String {
				str := val.(string)
				// replace '\=' with '='
				if strings.HasPrefix(str, `\`+CHAR_EXPRESSION) {
					str = str[1:]
				}
				// expand home if field tagged 'file'
				if FieldIs(field, FIELD_FILE) {
					str = util.ExpandUserHome(str)
				}
				val = str
			}
			// wrap values if necessary
			if FieldIs(field, FIELD_WRAP) && !(reflect.TypeOf(val).Kind() == reflect.Slice) {
				slice := reflect.New(field.Type).Elem()
				slice = reflect.Append(slice, reflect.ValueOf(val))
				val = slice.Interface()
			}
			// put value in params
			CopyValue(reflect.ValueOf(val), value.Field(i))
		}
	}
	return value.Interface(), nil
}

// CopyValue copy given value in another
// - orig: origin value
// - dest: destination value
func CopyValue(orig, dest reflect.Value) {
	// loop on slices
	if orig.Type().Kind() == reflect.Slice && dest.Type().Kind() == reflect.Slice {
		new := reflect.MakeSlice(dest.Type(), orig.Len(), orig.Len())
		for i := 0; i < orig.Len(); i++ {
			CopyValue(orig.Index(i), new.Index(i))
		}
		dest.Set(new)
	} else
	// loop on maps
	if orig.Type().Kind() == reflect.Map && dest.Type().Kind() == reflect.Map {
		keyType := dest.Type().Key()
		valType := dest.Type().Elem()
		new := reflect.MakeMap(reflect.MapOf(keyType, valType))
		for _, key := range orig.MapKeys() {
			if key.Type().Kind() == reflect.Interface {
				key = key.Elem()
			}
			val := orig.MapIndex(key)
			if val.Type().Kind() == reflect.Interface {
				val = val.Elem()
			}
			new.SetMapIndex(key, val)
		}
		dest.Set(new)
	} else
	// get value of interfaces
	if orig.Kind() == reflect.Interface {
		CopyValue(orig.Elem(), dest)
	} else
	// other types
	{
		dest.Set(orig)
	}
}

// FieldIs tells if given field tag contains quality
// - field: the struct field
// - quality: the tested quality (such as "optional")
func FieldIs(field reflect.StructField, quality string) bool {
	tag := string(field.Tag)
	qualities := strings.Split(tag, TAG_SEPARATOR)
	for _, q := range qualities {
		if q == quality {
			return true
		}
	}
	return false
}

// IsExpression tells if given string is an expression
// - s: the string to test
// Return: a bool that tells if the string is an expression
func IsExpression(s string) bool {
	return s[0:1] == CHAR_EXPRESSION && s[1:2] != CURLY_EXPRESSION
}
<commit_msg>Fixed type check<commit_after>package build

import (
	"fmt"
	"neon/util"
	"reflect"
	"strings"
)

// character for expressions
const (
	// character that start expressions
	CHAR_EXPRESSION = "="
	// expression curly brace
	CURLY_EXPRESSION = "{"
	// tag separator
	TAG_SEPARATOR = " "
	// field might not be provided
	FIELD_OPTIONAL = "optional"
	// field is a file name that is expanded for user home
	FIELD_FILE = "file"
	// field is an expression
	FIELD_EXPRESSION = "expression"
	// field should be rapped in a slice of its type
	FIELD_WRAP = "wrap"
	// field is a list of steps
	FIELD_STEPS = "steps"
	// field has a different name
	FIELD_NAME = "name"
)

// Map that gives constructor for given task name
var TaskMap = make(map[string]TaskDesc)

// AddTask adds a task in the map:
// - task: description of the task
func AddTask(task TaskDesc) {
	if _, ok := TaskMap[task.Name]; ok {
		panic(fmt.Errorf("task '%s' already defined", task.Name))
	}
	TaskMap[task.Name] = task
}

// A task descriptor is made of a task constructor and an help string
type TaskDesc struct {
	Name string
	Args reflect.Type
	Func TaskFunc
	Help string
}

// Type for task arguments as parsed in build file
type TaskArgs map[interface{}]interface{}

// Type for task function called to run task
type TaskFunc func(ctx *Context, args interface{}) error

// Validate task arguments against task arguments definition
// - args: task arguments parsed in build file
// - typ: type of the arguments
// Return: an error (detailing the fault) if arguments are illegal
func ValidateTaskArgs(args TaskArgs, typ reflect.Type) error {
	if typ.Kind() != reflect.Struct {
		return fmt.Errorf("params must be a pointer on a struct")
	}
	var fields []string
	// iterate on fields of the parameters types and check argument types
	for i := 0; i < typ.NumField(); i++ {
		field := typ.Field(i)
		name := strings.ToLower(field.Name)
		if field.Tag.Get(FIELD_NAME) != "" {
			name = field.Tag.Get(FIELD_NAME)
		}
		fields = append(fields, name)
		// check field is not missing
		if _, ok := args[name]; !ok {
			if !FieldIs(field, FIELD_OPTIONAL) {
				return fmt.Errorf("missing mandatory field '%s'", name)
			}
		}
		value := args[name]
		// parse steps fields
		if FieldIs(field, "steps") && value != nil {
			steps, err := NewSteps(value)
			if err != nil {
				return fmt.Errorf("parsing field '%s': %v", name, err)
			}
			args[name] = steps
		}
		// check field type
		if !CheckType(field, value) {
			return fmt.Errorf("field '%s' must be of type '%s' ('%s' provided)",
				name, field.Type, reflect.TypeOf(value))
		}
	}
	// check that we don't have unknown args
	if !(typ.NumField() == 0 && len(args) == 1) {
		for name := range args {
			found := false
			for _, n := range fields {
				if n == name {
					found = true
					continue
				}
			}
			if !found {
				return fmt.Errorf("unknown parameter '%s'", name)
			}
		}
	}
	return nil
}

// CheckType checks that given value is compatible with field type
// - field: the field of the parameters as reflect.StructField
// - value: the value of the argument
// Return: a bool that tells if type is OK
func CheckType(field reflect.StructField, value interface{}) bool {
	valueType := reflect.TypeOf(value)
	// if field is optional and argument nil, it's OK
	if FieldIs(field, FIELD_OPTIONAL) && value == nil {
		return true
	}
	// if argument is an expression it's OK whatever the type
	if valueType.Kind() == reflect.String &&
		(IsExpression(value.(string)) ||
			FieldIs(field, FIELD_EXPRESSION)) {
		return true
	}
	// if type of field is slice of the type of the argument and wrap, it's OK
	if field.Type.Kind() == reflect.Slice &&
		reflect.SliceOf(valueType) == field.Type &&
		FieldIs(field, FIELD_WRAP) {
		return true
	}
	// if type is slice and argument is steps, it's OK
	if field.Type.Kind() == reflect.Slice &&
		FieldIs(field, FIELD_STEPS) {
		return true
	}
	// check that value is of given type
	return IsValueOfType(value, field.Type)
}

// IsValueOfType tells if a value is of given type
// - value: the value to test as an interface{}
// - type: the type to check as a reflect.Type
// Return: a bool telling if value is of given type
func IsValueOfType(value interface{}, typ reflect.Type) bool {
	// if value is of given type it's true
	if reflect.TypeOf(value) == typ {
		return true
	}
	// if value and type are slices, test elements
	if reflect.TypeOf(value).Kind() == reflect.Slice && typ.Kind() == reflect.Slice {
		return IsValueOfType(reflect.ValueOf(value).Index(0).Interface(), typ.Elem())
	}
	// if value and type are maps, test key and value
	if reflect.TypeOf(value).Kind() == reflect.Map && typ.Kind() == reflect.Map {
		key := reflect.ValueOf(value).MapKeys()[0].Interface()
		val := reflect.ValueOf(value).MapIndex(reflect.ValueOf(key)).Interface()
		return IsValueOfType(key, typ.Key()) && IsValueOfType(val, typ.Elem())
	}
	// else return false
	return false
}

// Build task arguments from task params and return it
// - args: task arguments parsed in build file
// - typ: the type of the arguments
// - context: the build context to evaluate arguments into
// Return:
// - result: as an interface{}
// - error: if something went wrong
func EvaluateTaskArgs(args TaskArgs, typ reflect.Type, context *Context) (interface{}, error) {
	var err error
	value := reflect.New(typ).Elem()
	for i := 0; i < value.NumField(); i++ {
		name := strings.ToLower(typ.Field(i).Name)
		field := typ.Field(i)
		if field.Tag.Get(FIELD_NAME) != "" {
			name = field.Tag.Get(FIELD_NAME)
		}
		if args[name] != nil {
			val := args[name]
			// evaluate expressions in context
			if reflect.TypeOf(val).Kind() == reflect.String &&
				(IsExpression(val.(string)) || FieldIs(field, FIELD_EXPRESSION)) {
				str := val.(string)
				if IsExpression(str) {
					str = str[1:]
				}
				val, err = context.EvaluateExpression(str)
				if err != nil {
					return nil, err
				}
				expected := field.Type
				actual := reflect.TypeOf(val)
				if actual != expected {
					// we accept if expected is slice of interfaces and actual is slice
					if !(expected.Kind() == reflect.Slice && actual.Kind() == reflect.Slice) &&
						// or if slice and wrap
						!(expected == reflect.SliceOf(actual) && FieldIs(field, FIELD_WRAP)) {
						return nil, fmt.Errorf("bad expression return type, expected '%v' but '%v' was returned",
							expected, actual)
					}
				}
			}
			// evaluate arguments
			val, err = context.EvaluateObject(val)
			if err != nil {
				return nil, err
			}
			// evaluate strings to replace "={expression}" with its value
			if reflect.TypeOf(val).Kind() == reflect.String {
				str := val.(string)
				// replace '\=' with '='
				if strings.HasPrefix(str, `\`+CHAR_EXPRESSION) {
					str = str[1:]
				}
				// expand home if field tagged 'file'
				if FieldIs(field, FIELD_FILE) {
					str = util.ExpandUserHome(str)
				}
				val = str
			}
			// wrap values if necessary
			if FieldIs(field, FIELD_WRAP) && !(reflect.TypeOf(val).Kind() == reflect.Slice) {
				slice := reflect.New(field.Type).Elem()
				slice = reflect.Append(slice, reflect.ValueOf(val))
				val = slice.Interface()
			}
			// put value in params
			CopyValue(reflect.ValueOf(val), value.Field(i))
		}
	}
	return value.Interface(), nil
}

// CopyValue copy given value in another
// - orig: origin value
// - dest: destination value
func CopyValue(orig, dest reflect.Value) {
	// loop on slices
	if orig.Type().Kind() == reflect.Slice && dest.Type().Kind() == reflect.Slice {
		new := reflect.MakeSlice(dest.Type(), orig.Len(), orig.Len())
		for i := 0; i < orig.Len(); i++ {
			CopyValue(orig.Index(i), new.Index(i))
		}
		dest.Set(new)
	} else
	// loop on maps
	if orig.Type().Kind() == reflect.Map && dest.Type().Kind() == reflect.Map {
		keyType := dest.Type().Key()
		valType := dest.Type().Elem()
		new := reflect.MakeMap(reflect.MapOf(keyType, valType))
		for _, key := range orig.MapKeys() {
			if key.Type().Kind() == reflect.Interface {
				key = key.Elem()
			}
			val := orig.MapIndex(key)
			if val.Type().Kind() == reflect.Interface {
				val = val.Elem()
			}
			new.SetMapIndex(key, val)
		}
		dest.Set(new)
	} else
	// get value of interfaces
	if orig.Kind() == reflect.Interface {
		CopyValue(orig.Elem(), dest)
	} else
	// other types
	{
		dest.Set(orig)
	}
}

// FieldIs tells if given field tag contains quality
// - field: the struct field
// - quality: the tested quality (such as "optional")
func FieldIs(field reflect.StructField, quality string) bool {
	tag := string(field.Tag)
	qualities := strings.Split(tag, TAG_SEPARATOR)
	for _, q := range qualities {
		if q == quality {
			return true
		}
	}
	return false
}

// IsExpression tells if given string is an expression
// - s: the string to test
// Return: a bool that tells if the string is an expression
func IsExpression(s string) bool {
	return s[0:1] == CHAR_EXPRESSION && s[1:2] != CURLY_EXPRESSION
}
<|endoftext|>


<commit_before>// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package storage // import "miniflux.app/storage"

import (
	"errors"
	"fmt"
	"time"

	"miniflux.app/logger"
	"miniflux.app/model"
	"miniflux.app/timer"

	"github.com/lib/pq"
)

// CountUnreadEntries returns the number of unread entries.
func (s *Storage) CountUnreadEntries(userID int64) int {
	builder := s.NewEntryQueryBuilder(userID)
	builder.WithStatus(model.EntryStatusUnread)

	n, err := builder.CountEntries()
	if err != nil {
		logger.Error("unable to count unread entries for user #%d: %v", userID, err)
		return 0
	}

	return n
}

// NewEntryQueryBuilder returns a new EntryQueryBuilder
func (s *Storage) NewEntryQueryBuilder(userID int64) *EntryQueryBuilder {
	return NewEntryQueryBuilder(s, userID)
}

// UpdateEntryContent updates entry content.
func (s *Storage) UpdateEntryContent(entry *model.Entry) error {
	tx, err := s.db.Begin()
	if err != nil {
		return err
	}

	_, err = tx.Exec(`UPDATE entries SET content=$1 WHERE id=$2 AND user_id=$3`, entry.Content, entry.ID, entry.UserID)
	if err != nil {
		tx.Rollback()
		return fmt.Errorf(`unable to update content of entry #%d: %v`, entry.ID, err)
	}

	query := `
		UPDATE entries
		SET document_vectors = to_tsvector(substring(title || ' ' || coalesce(content, '') for 1000000))
		WHERE id=$1 AND user_id=$2
	`
	_, err = tx.Exec(query, entry.ID, entry.UserID)
	if err != nil {
		tx.Rollback()
		return fmt.Errorf(`unable to update content of entry #%d: %v`, entry.ID, err)
	}

	return tx.Commit()
}

// createEntry add a new entry.
func (s *Storage) createEntry(entry *model.Entry) error {
	query := `
		INSERT INTO entries
		(title, hash, url, comments_url, published_at, content, author, user_id, feed_id, document_vectors)
		VALUES
		($1, $2, $3, $4, $5, $6, $7, $8, $9, to_tsvector(substring($1 || ' ' || coalesce($6, '') for 1000000)))
		RETURNING id, status
	`
	err := s.db.QueryRow(
		query,
		entry.Title,
		entry.Hash,
		entry.URL,
		entry.CommentsURL,
		entry.Date,
		entry.Content,
		entry.Author,
		entry.UserID,
		entry.FeedID,
	).Scan(&entry.ID, &entry.Status)

	if err != nil {
		return fmt.Errorf("unable to create entry %q (feed #%d): %v", entry.URL, entry.FeedID, err)
	}

	for i := 0; i < len(entry.Enclosures); i++ {
		entry.Enclosures[i].EntryID = entry.ID
		entry.Enclosures[i].UserID = entry.UserID
		err := s.CreateEnclosure(entry.Enclosures[i])
		if err != nil {
			return err
		}
	}

	return nil
}

// updateEntry updates an entry when a feed is refreshed.
// Note: we do not update the published date because some feeds do not contains any date,
// it default to time.Now() which could change the order of items on the history page.
func (s *Storage) updateEntry(entry *model.Entry) error {
	query := `
		UPDATE entries SET
		title=$1, url=$2, comments_url=$3, content=$4, author=$5,
		document_vectors=to_tsvector(substring($1 || ' ' || coalesce($4, '') for 1000000))
		WHERE user_id=$6 AND feed_id=$7 AND hash=$8
		RETURNING id
	`
	err := s.db.QueryRow(
		query,
		entry.Title,
		entry.URL,
		entry.CommentsURL,
		entry.Content,
		entry.Author,
		entry.UserID,
		entry.FeedID,
		entry.Hash,
	).Scan(&entry.ID)

	if err != nil {
		return fmt.Errorf(`unable to update entry %q: %v`, entry.URL, err)
	}

	for _, enclosure := range entry.Enclosures {
		enclosure.UserID = entry.UserID
		enclosure.EntryID = entry.ID
	}

	return s.UpdateEnclosures(entry.Enclosures)
}

// entryExists checks if an entry already exists based on its hash when refreshing a feed.
func (s *Storage) entryExists(entry *model.Entry) bool {
	var result int
	query := `SELECT count(*) as c FROM entries WHERE user_id=$1 AND feed_id=$2 AND hash=$3`
	s.db.QueryRow(query, entry.UserID, entry.FeedID, entry.Hash).Scan(&result)
	return result >= 1
}

// cleanupEntries deletes from the database entries marked as "removed" and not visible anymore in the feed.
func (s *Storage) cleanupEntries(feedID int64, entryHashes []string) error {
	query := `
		DELETE FROM entries
		WHERE feed_id=$1 AND
		id IN (SELECT id FROM entries WHERE feed_id=$2 AND status=$3 AND NOT (hash=ANY($4)))
	`
	if _, err := s.db.Exec(query, feedID, feedID, model.EntryStatusRemoved, pq.Array(entryHashes)); err != nil {
		return fmt.Errorf("unable to cleanup entries: %v", err)
	}

	return nil
}

// UpdateEntries updates a list of entries while refreshing a feed.
func (s *Storage) UpdateEntries(userID, feedID int64, entries model.Entries, updateExistingEntries bool) (err error) {
	var entryHashes []string
	for _, entry := range entries {
		entry.UserID = userID
		entry.FeedID = feedID

		if s.entryExists(entry) {
			if updateExistingEntries {
				err = s.updateEntry(entry)
			}
		} else {
			err = s.createEntry(entry)
		}

		if err != nil {
			return err
		}

		entryHashes = append(entryHashes, entry.Hash)
	}

	if err := s.cleanupEntries(feedID, entryHashes); err != nil {
		logger.Error("[Storage:CleanupEntries] feed #%d: %v", feedID, err)
	}

	return nil
}

// ArchiveEntries changes the status of read items to "removed" after 60 days.
func (s *Storage) ArchiveEntries() error {
	query := `
		UPDATE entries SET status='removed'
		WHERE id=ANY(SELECT id FROM entries WHERE status='read' AND starred is false AND published_at < now () - '60 days'::interval LIMIT 500)
	`
	if _, err := s.db.Exec(query); err != nil {
		return fmt.Errorf("unable to archive read entries: %v", err)
	}

	return nil
}

// SetEntriesStatus update the status of the given list of entries.
func (s *Storage) SetEntriesStatus(userID int64, entryIDs []int64, status string) error {
	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:SetEntriesStatus] userID=%d, entryIDs=%v, status=%s", userID, entryIDs, status))

	query := `UPDATE entries SET status=$1 WHERE user_id=$2 AND id=ANY($3)`
	result, err := s.db.Exec(query, status, userID, pq.Array(entryIDs))
	if err != nil {
		return fmt.Errorf("unable to update entries statuses %v: %v", entryIDs, err)
	}

	count, err := result.RowsAffected()
	if err != nil {
		return fmt.Errorf("unable to update these entries %v: %v", entryIDs, err)
	}

	if count == 0 {
		return errors.New("nothing has been updated")
	}

	return nil
}

// ToggleBookmark toggles entry bookmark value.
func (s *Storage) ToggleBookmark(userID int64, entryID int64) error {
	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:ToggleBookmark] userID=%d, entryID=%d", userID, entryID))

	query := `UPDATE entries SET starred = NOT starred WHERE user_id=$1 AND id=$2`
	result, err := s.db.Exec(query, userID, entryID)
	if err != nil {
		return fmt.Errorf("unable to toggle bookmark flag for entry #%d: %v", entryID, err)
	}

	count, err := result.RowsAffected()
	if err != nil {
		return fmt.Errorf("unable to toogle bookmark flag for entry #%d: %v", entryID, err)
	}

	if count == 0 {
		return errors.New("nothing has been updated")
	}

	return nil
}

// FlushHistory set all entries with the status "read" to "removed".
func (s *Storage) FlushHistory(userID int64) error {
	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:FlushHistory] userID=%d", userID))

	query := `UPDATE entries SET status=$1 WHERE user_id=$2 AND status=$3 AND starred='f'`
	_, err := s.db.Exec(query, model.EntryStatusRemoved, userID, model.EntryStatusRead)
	if err != nil {
		return fmt.Errorf("unable to flush history: %v", err)
	}

	return nil
}

// MarkAllAsRead updates all user entries to the read status.
func (s *Storage) MarkAllAsRead(userID int64) error {
	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:MarkAllAsRead] userID=%d", userID))

	query := `UPDATE entries SET status=$1 WHERE user_id=$2 AND status=$3`
	result, err := s.db.Exec(query, model.EntryStatusRead, userID, model.EntryStatusUnread)
	if err != nil {
		return fmt.Errorf("unable to mark all entries as read: %v", err)
	}

	count, _ := result.RowsAffected()
	logger.Debug("[Storage:MarkAllAsRead] %d items marked as read", count)

	return nil
}

// MarkFeedAsRead updates all feed entries to the read status.
func (s *Storage) MarkFeedAsRead(userID, feedID int64, before time.Time) error {
	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:MarkFeedAsRead] userID=%d, feedID=%d, before=%v", userID, feedID, before))

	query := `
		UPDATE entries
		SET status=$1
		WHERE user_id=$2 AND feed_id=$3 AND status=$4 AND published_at < $5
	`

	result, err := s.db.Exec(query, model.EntryStatusRead, userID, feedID, model.EntryStatusUnread, before)
	if err != nil {
		return fmt.Errorf("unable to mark feed entries as read: %v", err)
	}

	count, _ := result.RowsAffected()
	logger.Debug("[Storage:MarkFeedAsRead] %d items marked as read", count)

	return nil
}

// MarkCategoryAsRead updates all category entries to the read status.
func (s *Storage) MarkCategoryAsRead(userID, categoryID int64, before time.Time) error {
	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:MarkCategoryAsRead] userID=%d, categoryID=%d, before=%v", userID, categoryID, before))

	query := `
		UPDATE entries
		SET status=$1
		WHERE
		user_id=$2 AND status=$3 AND published_at < $4 AND feed_id IN (SELECT id FROM feeds WHERE user_id=$2 AND category_id=$5)
	`

	result, err := s.db.Exec(query, model.EntryStatusRead, userID, model.EntryStatusUnread, before, categoryID)
	if err != nil {
		return fmt.Errorf("unable to mark category entries as read: %v", err)
	}

	count, _ := result.RowsAffected()
	logger.Debug("[Storage:MarkCategoryAsRead] %d items marked as read", count)

	return nil
}

// EntryURLExists returns true if an entry with this URL already exists.
func (s *Storage) EntryURLExists(userID int64, entryURL string) bool {
	var result int
	query := `SELECT count(*) as c FROM entries WHERE user_id=$1 AND url=$2`
	s.db.QueryRow(query, userID, entryURL).Scan(&result)
	return result >= 1
}
<commit_msg>Archive more read entries<commit_after>// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package storage // import "miniflux.app/storage"

import (
	"errors"
	"fmt"
	"time"

	"miniflux.app/logger"
	"miniflux.app/model"
	"miniflux.app/timer"

	"github.com/lib/pq"
)

// CountUnreadEntries returns the number of unread entries.
func (s *Storage) CountUnreadEntries(userID int64) int {
	builder := s.NewEntryQueryBuilder(userID)
	builder.WithStatus(model.EntryStatusUnread)

	n, err := builder.CountEntries()
	if err != nil {
		logger.Error("unable to count unread entries for user #%d: %v", userID, err)
		return 0
	}

	return n
}

// NewEntryQueryBuilder returns a new EntryQueryBuilder
func (s *Storage) NewEntryQueryBuilder(userID int64) *EntryQueryBuilder {
	return NewEntryQueryBuilder(s, userID)
}

// UpdateEntryContent updates entry content.
func (s *Storage) UpdateEntryContent(entry *model.Entry) error {
	tx, err := s.db.Begin()
	if err != nil {
		return err
	}

	_, err = tx.Exec(`UPDATE entries SET content=$1 WHERE id=$2 AND user_id=$3`, entry.Content, entry.ID, entry.UserID)
	if err != nil {
		tx.Rollback()
		return fmt.Errorf(`unable to update content of entry #%d: %v`, entry.ID, err)
	}

	query := `
		UPDATE entries
		SET document_vectors = to_tsvector(substring(title || ' ' || coalesce(content, '') for 1000000))
		WHERE id=$1 AND user_id=$2
	`
	_, err = tx.Exec(query, entry.ID, entry.UserID)
	if err != nil {
		tx.Rollback()
		return fmt.Errorf(`unable to update content of entry #%d: %v`, entry.ID, err)
	}

	return tx.Commit()
}

// createEntry add a new entry.
func (s *Storage) createEntry(entry *model.Entry) error {
	query := `
		INSERT INTO entries
		(title, hash, url, comments_url, published_at, content, author, user_id, feed_id, document_vectors)
		VALUES
		($1, $2, $3, $4, $5, $6, $7, $8, $9, to_tsvector(substring($1 || ' ' || coalesce($6, '') for 1000000)))
		RETURNING id, status
	`
	err := s.db.QueryRow(
		query,
		entry.Title,
		entry.Hash,
		entry.URL,
		entry.CommentsURL,
		entry.Date,
		entry.Content,
		entry.Author,
		entry.UserID,
		entry.FeedID,
	).Scan(&entry.ID, &entry.Status)

	if err != nil {
		return fmt.Errorf("unable to create entry %q (feed #%d): %v", entry.URL, entry.FeedID, err)
	}

	for i := 0; i < len(entry.Enclosures); i++ {
		entry.Enclosures[i].EntryID = entry.ID
		entry.Enclosures[i].UserID = entry.UserID
		err := s.CreateEnclosure(entry.Enclosures[i])
		if err != nil {
			return err
		}
	}

	return nil
}

// updateEntry updates an entry when a feed is refreshed.
// Note: we do not update the published date because some feeds do not contains any date,
// it default to time.Now() which could change the order of items on the history page.
func (s *Storage) updateEntry(entry *model.Entry) error {
	query := `
		UPDATE entries SET
		title=$1, url=$2, comments_url=$3, content=$4, author=$5,
		document_vectors=to_tsvector(substring($1 || ' ' || coalesce($4, '') for 1000000))
		WHERE user_id=$6 AND feed_id=$7 AND hash=$8
		RETURNING id
	`
	err := s.db.QueryRow(
		query,
		entry.Title,
		entry.URL,
		entry.CommentsURL,
		entry.Content,
		entry.Author,
		entry.UserID,
		entry.FeedID,
		entry.Hash,
	).Scan(&entry.ID)

	if err != nil {
		return fmt.Errorf(`unable to update entry %q: %v`, entry.URL, err)
	}

	for _, enclosure := range entry.Enclosures {
		enclosure.UserID = entry.UserID
		enclosure.EntryID = entry.ID
	}

	return s.UpdateEnclosures(entry.Enclosures)
}

// entryExists checks if an entry already exists based on its hash when refreshing a feed.
func (s *Storage) entryExists(entry *model.Entry) bool {
	var result int
	query := `SELECT count(*) as c FROM entries WHERE user_id=$1 AND feed_id=$2 AND hash=$3`
	s.db.QueryRow(query, entry.UserID, entry.FeedID, entry.Hash).Scan(&result)
	return result >= 1
}

// cleanupEntries deletes from the database entries marked as "removed" and not visible anymore in the feed.
func (s *Storage) cleanupEntries(feedID int64, entryHashes []string) error {
	query := `
		DELETE FROM entries
		WHERE feed_id=$1 AND
		id IN (SELECT id FROM entries WHERE feed_id=$2 AND status=$3 AND NOT (hash=ANY($4)))
	`
	if _, err := s.db.Exec(query, feedID, feedID, model.EntryStatusRemoved, pq.Array(entryHashes)); err != nil {
		return fmt.Errorf("unable to cleanup entries: %v", err)
	}

	return nil
}

// UpdateEntries updates a list of entries while refreshing a feed.
func (s *Storage) UpdateEntries(userID, feedID int64, entries model.Entries, updateExistingEntries bool) (err error) {
	var entryHashes []string
	for _, entry := range entries {
		entry.UserID = userID
		entry.FeedID = feedID

		if s.entryExists(entry) {
			if updateExistingEntries {
				err = s.updateEntry(entry)
			}
		} else {
			err = s.createEntry(entry)
		}

		if err != nil {
			return err
		}

		entryHashes = append(entryHashes, entry.Hash)
	}

	if err := s.cleanupEntries(feedID, entryHashes); err != nil {
		logger.Error("[Storage:CleanupEntries] feed #%d: %v", feedID, err)
	}

	return nil
}

// ArchiveEntries changes the status of read items to "removed" after 60 days.
func (s *Storage) ArchiveEntries() error {
	query := `
		UPDATE entries SET status='removed'
		WHERE id=ANY(SELECT id FROM entries WHERE status='read' AND starred is false AND published_at < now () - '60 days'::interval LIMIT 5000)
	`
	if _, err := s.db.Exec(query); err != nil {
		return fmt.Errorf("unable to archive read entries: %v", err)
	}

	return nil
}

// SetEntriesStatus update the status of the given list of entries.
func (s *Storage) SetEntriesStatus(userID int64, entryIDs []int64, status string) error {
	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:SetEntriesStatus] userID=%d, entryIDs=%v, status=%s", userID, entryIDs, status))

	query := `UPDATE entries SET status=$1 WHERE user_id=$2 AND id=ANY($3)`
	result, err := s.db.Exec(query, status, userID, pq.Array(entryIDs))
	if err != nil {
		return fmt.Errorf("unable to update entries statuses %v: %v", entryIDs, err)
	}

	count, err := result.RowsAffected()
	if err != nil {
		return fmt.Errorf("unable to update these entries %v: %v", entryIDs, err)
	}

	if count == 0 {
		return errors.New("nothing has been updated")
	}

	return nil
}

// ToggleBookmark toggles entry bookmark value.
func (s *Storage) ToggleBookmark(userID int64, entryID int64) error {
	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:ToggleBookmark] userID=%d, entryID=%d", userID, entryID))

	query := `UPDATE entries SET starred = NOT starred WHERE user_id=$1 AND id=$2`
	result, err := s.db.Exec(query, userID, entryID)
	if err != nil {
		return fmt.Errorf("unable to toggle bookmark flag for entry #%d: %v", entryID, err)
	}

	count, err := result.RowsAffected()
	if err != nil {
		return fmt.Errorf("unable to toogle bookmark flag for entry #%d: %v", entryID, err)
	}

	if count == 0 {
		return errors.New("nothing has been updated")
	}

	return nil
}

// FlushHistory set all entries with the status "read" to "removed".
func (s *Storage) FlushHistory(userID int64) error {
	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:FlushHistory] userID=%d", userID))

	query := `UPDATE entries SET status=$1 WHERE user_id=$2 AND status=$3 AND starred='f'`
	_, err := s.db.Exec(query, model.EntryStatusRemoved, userID, model.EntryStatusRead)
	if err != nil {
		return fmt.Errorf("unable to flush history: %v", err)
	}

	return nil
}

// MarkAllAsRead updates all user entries to the read status.
func (s *Storage) MarkAllAsRead(userID int64) error {
	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:MarkAllAsRead] userID=%d", userID))

	query := `UPDATE entries SET status=$1 WHERE user_id=$2 AND status=$3`
	result, err := s.db.Exec(query, model.EntryStatusRead, userID, model.EntryStatusUnread)
	if err != nil {
		return fmt.Errorf("unable to mark all entries as read: %v", err)
	}

	count, _ := result.RowsAffected()
	logger.Debug("[Storage:MarkAllAsRead] %d items marked as read", count)

	return nil
}

// MarkFeedAsRead updates all feed entries to the read status.
func (s *Storage) MarkFeedAsRead(userID, feedID int64, before time.Time) error {
	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:MarkFeedAsRead] userID=%d, feedID=%d, before=%v", userID, feedID, before))

	query := `
		UPDATE entries
		SET status=$1
		WHERE user_id=$2 AND feed_id=$3 AND status=$4 AND published_at < $5
	`

	result, err := s.db.Exec(query, model.EntryStatusRead, userID, feedID, model.EntryStatusUnread, before)
	if err != nil {
		return fmt.Errorf("unable to mark feed entries as read: %v", err)
	}

	count, _ := result.RowsAffected()
	logger.Debug("[Storage:MarkFeedAsRead] %d items marked as read", count)

	return nil
}

// MarkCategoryAsRead updates all category entries to the read status.
func (s *Storage) MarkCategoryAsRead(userID, categoryID int64, before time.Time) error {
	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Storage:MarkCategoryAsRead] userID=%d, categoryID=%d, before=%v", userID, categoryID, before))

	query := `
		UPDATE entries
		SET status=$1
		WHERE
		user_id=$2 AND status=$3 AND published_at < $4 AND feed_id IN (SELECT id FROM feeds WHERE user_id=$2 AND category_id=$5)
	`

	result, err := s.db.Exec(query, model.EntryStatusRead, userID, model.EntryStatusUnread, before, categoryID)
	if err != nil {
		return fmt.Errorf("unable to mark category entries as read: %v", err)
	}

	count, _ := result.RowsAffected()
	logger.Debug("[Storage:MarkCategoryAsRead] %d items marked as read", count)

	return nil
}

// EntryURLExists returns true if an entry with this URL already exists.
func (s *Storage) EntryURLExists(userID int64, entryURL string) bool {
	var result int
	query := `SELECT count(*) as c FROM entries WHERE user_id=$1 AND url=$2`
	s.db.QueryRow(query, userID, entryURL).Scan(&result)
	return result >= 1
}
<|endoftext|>


<commit_before>package stream

import (
	"encoding/xml"
	"github.com/dotdoom/goxmpp/stream/elements"
)

var GlobalElementFactory = elements.NewFactory()

type Stream struct {
	XMLName xml.Name
	ID      string
	From    string
	To      string
	Version string
}

type Element interface{}

// An entry point for decoding elements in response to features
// anouncment from server, before sission is opened
func handleFeature(sw *Wrapper) {
	processStreamElements(sw.StreamDecoder, GlobalElementFactory, func(handler Element) bool {
		// TODO: need to check sw for the state when all required features processed and exit the loop
		unmarshalStreamElement(handler, sw)
		return true
	})
}

// This is an entry point for decode stanzas
func NextStanza(sw *Wrapper) Element {
	var stanza Element

	processStreamElements(sw.StreamDecoder, GlobalElementFactory, func(handler Element) bool {
		stanza = unmarshalStreamElement(handler, sw)
		return false // We need process stanzas one by one
	})

	return stanza
}
<commit_msg>Fix typo<commit_after>package stream

import (
	"encoding/xml"
	"github.com/dotdoom/goxmpp/stream/elements"
)

var GlobalElementFactory = elements.NewFactory()

type Stream struct {
	XMLName xml.Name
	ID      string
	From    string
	To      string
	Version string
}

type Element interface{}

// An entry point for decoding elements in response to features
// announcment from server, before session is opened
func handleFeature(sw *Wrapper) {
	processStreamElements(sw.StreamDecoder, GlobalElementFactory, func(handler Element) bool {
		// TODO: need to check sw for the state when all required features processed and exit the loop
		unmarshalStreamElement(handler, sw)
		return true
	})
}

// This is an entry point for decode stanzas
func NextStanza(sw *Wrapper) Element {
	var stanza Element

	processStreamElements(sw.StreamDecoder, GlobalElementFactory, func(handler Element) bool {
		stanza = unmarshalStreamElement(handler, sw)
		return false // We need process stanzas one by one
	})

	return stanza
}
<|endoftext|>


<commit_before>package handler

import (
	"fmt"
	"net/http"
	"net/url"
	"time"

	"github.com/mattbostock/leavediary/model"
)

func Dashboard(w http.ResponseWriter, r *http.Request) {
	user := currentUser(r)

	if user.ID == 0 { // no current user; not logged in
		http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
		return
	}

	if len(user.Jobs) == 0 {
		log.Infoln("No jobs defined")
		http.Redirect(w, r, "/settings", http.StatusTemporaryRedirect)
		return
	}

	job := user.CurrentJob()
	currentLeaveAllowance, err := job.CurrentLeaveAllowance()
	if err != nil {
		internalError(w, err)
		return
	}
	if currentLeaveAllowance.ID == 0 {
		log.Infoln("No current leave allowance")
		http.Redirect(w, r, "/settings", http.StatusTemporaryRedirect)
		return
	}

	remainingMinutes, err := currentLeaveAllowance.RemainingTime()
	if err != nil {
		internalError(w, err)
		return
	}
	remainingDays := float32(remainingMinutes) / 60 / 24

	var nextOnLeave model.LeaveRequest
	var pastRequests, upcomingRequests []model.LeaveRequest

	requests, err := job.RequestsLastYearAndFuture()
	if err != nil {
		internalError(w, err)
		return
	}

	for _, req := range requests {
		if req.Before(time.Now().In(user.TZLocation())) {
			pastRequests = append(pastRequests, req)
		}
	}
	for _, req := range requests {
		if req.After(time.Now().AddDate(0, 0, -1).In(user.TZLocation())) {
			upcomingRequests = append(upcomingRequests, req)
		}
	}

	if len(upcomingRequests) > 0 {
		nextOnLeave = upcomingRequests[0]
	}

	calendarURL := &url.URL{
		Scheme: "https",
		Host:   r.Host,
		Path:   fmt.Sprintf("/export/ics/%s", job.ExportSecret),
	}

	webCalURLNoScheme := &url.URL{}
	*webCalURLNoScheme = *calendarURL
	webCalURLNoScheme.Scheme = ""

	output.HTML(w, http.StatusOK, "dashboard", &struct {
		CurrentLeaveAllowance model.LeaveAllowance
		CalendarURL           *url.URL
		NextOnLeave           model.LeaveRequest
		PastRequests          []model.LeaveRequest
		UpcomingRequests      []model.LeaveRequest
		RemainingDays         float32
		User                  model.User
		WebCalURLNoScheme     *url.URL
	}{
		currentLeaveAllowance,
		calendarURL,
		nextOnLeave,
		pastRequests,
		upcomingRequests,
		remainingDays,
		user,
		webCalURLNoScheme,
	})
}
<commit_msg>Dashboard handler: Redirect to allowance<commit_after>package handler

import (
	"fmt"
	"net/http"
	"net/url"
	"time"

	"github.com/mattbostock/leavediary/model"
)

func Dashboard(w http.ResponseWriter, r *http.Request) {
	user := currentUser(r)

	if user.ID == 0 { // no current user; not logged in
		http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
		return
	}

	if len(user.Jobs) == 0 {
		log.Infoln("No jobs defined")
		http.Redirect(w, r, "/settings", http.StatusTemporaryRedirect)
		return
	}

	job := user.CurrentJob()
	currentLeaveAllowance, err := job.CurrentLeaveAllowance()
	if err != nil {
		internalError(w, err)
		return
	}
	if currentLeaveAllowance.ID == 0 {
		log.Infoln("No current leave allowance")
		http.Redirect(w, r, "/allowance", http.StatusTemporaryRedirect)
		return
	}

	remainingMinutes, err := currentLeaveAllowance.RemainingTime()
	if err != nil {
		internalError(w, err)
		return
	}
	remainingDays := float32(remainingMinutes) / 60 / 24

	var nextOnLeave model.LeaveRequest
	var pastRequests, upcomingRequests []model.LeaveRequest

	requests, err := job.RequestsLastYearAndFuture()
	if err != nil {
		internalError(w, err)
		return
	}

	for _, req := range requests {
		if req.Before(time.Now().In(user.TZLocation())) {
			pastRequests = append(pastRequests, req)
		}
	}
	for _, req := range requests {
		if req.After(time.Now().AddDate(0, 0, -1).In(user.TZLocation())) {
			upcomingRequests = append(upcomingRequests, req)
		}
	}

	if len(upcomingRequests) > 0 {
		nextOnLeave = upcomingRequests[0]
	}

	calendarURL := &url.URL{
		Scheme: "https",
		Host:   r.Host,
		Path:   fmt.Sprintf("/export/ics/%s", job.ExportSecret),
	}

	webCalURLNoScheme := &url.URL{}
	*webCalURLNoScheme = *calendarURL
	webCalURLNoScheme.Scheme = ""

	output.HTML(w, http.StatusOK, "dashboard", &struct {
		CurrentLeaveAllowance model.LeaveAllowance
		CalendarURL           *url.URL
		NextOnLeave           model.LeaveRequest
		PastRequests          []model.LeaveRequest
		UpcomingRequests      []model.LeaveRequest
		RemainingDays         float32
		User                  model.User
		WebCalURLNoScheme     *url.URL
	}{
		currentLeaveAllowance,
		calendarURL,
		nextOnLeave,
		pastRequests,
		upcomingRequests,
		remainingDays,
		user,
		webCalURLNoScheme,
	})
}
<|endoftext|>


<commit_before>// Copyright 2014 The zhgo Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package db

import (
	"errors"
	"fmt"
	"log"
	"reflect"
	"strings"
)

// query struct
type Query struct {
	//Table
	table Table

	//Sql
	sql Sql

	//Execute
	execute Execute
}

// Select fields.
// Row() or Rows() will return []map[string]interface{} after the method called, otherwith return []EntityStruct.
func (q *Query) Fields(fields ...string) *Query {
	if len(fields) == 0 {
		fields = append(fields, "*")
	}

	q.sql.Select = strings.Join(fields, ", ")
	return q
}

// join table
func (q *Query) Join() *Query {

	return q
}

// Where, 通常查询以Where()方法开始, 后面跟随0个或多个的And(),Or()
func (q *Query) Where(field string, m string, val ...interface{}) *Query {
	return q.whereNode("AND", field, m, val...) //the first where logical must be AND.
}

// AND
func (q *Query) And(field string, m string, val ...interface{}) *Query {
	return q.whereNode("AND", field, m, val...)
}

// OR
func (q *Query) Or(field string, m string, val ...interface{}) *Query {
	return q.whereNode("OR", field, m, val...)
}

// Append where sql, private method
// l logic, for example: AND, OR
// field
// m condition, for example: =, <, <=, >, >=, <>, LLIKE, RLIKE, LIKE, IN, NOT IN, (
// val argument value, 可能是字符串, 也可能是数组, 也可能Where对象
func (q *Query) whereNode(l string, field string, m string, val ...interface{}) *Query {
	switch m {
	case "=", "<", "<=", ">", ">=", "<>", "LIKE":
		q.sql.Where += fmt.Sprintf(" %s %s %s ?", l, field, m)
		q.sql.Args = append(q.sql.Args, val[0])
	case "IN", "NOT IN":
		w := make([]string, len(val))
		for i, v := range val {
			w[i] = "?"
			q.sql.Args = append(q.sql.Args, v)
		}
		q.sql.Where += fmt.Sprintf(" %s %s %s (%s) ", l, field, m, strings.Join(w, ","))
	}

	return q
}

func (q *Query) Order() *Query {
	return q
}

func (q *Query) Group() *Query {

	return q
}

func (q *Query) Having() *Query {

	return q
}

func (q *Query) Limit(offset, rows uint64) *Query {

	return q
}

func (q *Query) Lock() *Query {

	return q
}

// First row, first column
func (q *Query) Scaler(ptr interface{}) error {

	return nil
}

// get one row
func (q *Query) Row(ptr interface{}) error {
	//get first row only
	q.sql.Offset = 0
	q.sql.Rows = 1

	return q.execute.Row(ptr, q.toString(), q.sql.Args)
}

// get all rows, return []struct or []map[string]interface{} or [][]interface{}
func (q *Query) Rows(ptr interface{}) error {
	return q.execute.Rows(ptr, q.toString(), q.sql.Args)
}

// Insert
func (q *Query) Insert(data *[]interface{}) (int64, error) {
	if len(*data) == 0 {
		err := errors.New("empty insert data")
		log.Printf("%s\n", err)
		return 0, err
	}

	var insertVal []string
	insertArgs := []interface{}{}
	insertArgsValue := reflect.Indirect(reflect.ValueOf(&insertArgs))

	for _, item := range *data {
		sli := []string{}

		//convert struct fields to sql string
		itemValue := reflect.ValueOf(item)
		itemType := itemValue.Type()
		for i := 0; i < itemType.NumField(); i++ {
			itemField := itemType.Field(i)
			if !itemField.Anonymous && itemField.Tag.Get("json") != "" && itemField.Tag.Get("pk") == "" {
				sli = append(sli, "?")
				v := itemValue.FieldByIndex([]int{i})
				insertArgsValue.Set(reflect.Append(insertArgsValue, v))
			}
		}

		insertVal = append(insertVal, "("+strings.Join(sli, ", ")+")")
	}

	insertSql := fmt.Sprintf("INSERT INTO `%s` (`%s`) VALUES %s", q.table.Name, strings.Join(q.table.Fields, "`, `"), strings.Join(insertVal, ","))

	log.Printf("%s\n", insertSql)
	log.Printf("%#v\n", insertArgs)

	result, err := q.execute.Exec(insertSql, insertArgs)
	if err != nil {
		log.Printf("%s\n", err)
		return 0, err
	}

	return result.LastInsertId()
}

// Update
func (q *Query) Update(data *map[string]interface{}) (int64, error) {
	//convert map to sql string
	updateBody := []string{}
	updateArgs := []interface{}{}

	for key, element := range *data {
		updateBody = append(updateBody, key+" = ?")
		updateArgs = append(updateArgs, element)
	}

	updateSql := fmt.Sprintf("UPDATE %s SET %s WHERE 1 %s", q.table.Name, strings.Join(updateBody, ", "), q.sql.Where)

	updateArgs = append(updateArgs, q.sql.Args...)

	log.Printf("%s\n", updateSql)
	log.Printf("%#v\n", updateArgs)

	result, err := q.execute.Exec(updateSql, updateArgs)
	if err != nil {
		log.Printf("%s\n", err)
		return 0, err
	}

	return result.RowsAffected()
}

// Delete
func (q *Query) Delete() (int64, error) {
	deleteSql := fmt.Sprintf("DELETE FROM %s WHERE 1 %s", q.table.Name, q.sql.Where)

	log.Printf("%s\n", deleteSql)
	log.Printf("%#v\n", q.sql.Args)

	result, err := q.execute.Exec(deleteSql, q.sql.Args)
	if err != nil {
		log.Printf("%s\n", err)
		return 0, err
	}

	return result.RowsAffected()
}

// connect all sql department to a corect sql string.
func (q *Query) toString() string {
	//create sql
	var sel string
	if len(q.sql.Select) > 0 {
		sel = q.sql.Select //返回[]map[string]interface{}
	} else {
		sel = fmt.Sprintf("a.`%s`, a.`%s`", q.table.Primary, strings.Join(q.table.Fields, "`, a.`")) //返回[]EntityStruct
	}

	sql := fmt.Sprintf("SELECT %s FROM %s a %s WHERE 1 %s %s %s %s LIMIT %d, %d %s", sel, q.table.Name, q.sql.Join, q.sql.Where, q.sql.Order, q.sql.Group, q.sql.Having, q.sql.Offset, q.sql.Rows, q.sql.ForUpdate)

	log.Printf("%s\n", sql)
	log.Printf("%#v\n", q.sql.Args)

	return sql
}
<commit_msg>Update repo.<commit_after>// Copyright 2014 The zhgo Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package db

import (
	"errors"
	"fmt"
	"log"
	"reflect"
	"strings"
)

// query struct
type Query struct {
	//Table
	table Table

	//Sql
	sql Sql

	//Execute
	execute Execute
}

// Select fields.
// Row() or Rows() will return []map[string]interface{} after the method called, otherwith return []EntityStruct.
func (q *Query) Fields(fields ...string) *Query {
	if len(fields) == 0 {
		fields = append(fields, "*")
	}

	q.sql.Select = strings.Join(fields, ", ")
	return q
}

// join table
func (q *Query) Join() *Query {

	return q
}

// Where, 通常查询以Where()方法开始, 后面跟随0个或多个的And(),Or()
func (q *Query) Where(field string, m string, val ...interface{}) *Query {
	return q.whereNode("AND", field, m, val...) //the first where logical must be AND.
}

// AND
func (q *Query) And(field string, m string, val ...interface{}) *Query {
	return q.whereNode("AND", field, m, val...)
}

// OR
func (q *Query) Or(field string, m string, val ...interface{}) *Query {
	return q.whereNode("OR", field, m, val...)
}

// Append where sql, private method
// l logic, for example: AND, OR
// field
// m condition, for example: =, <, <=, >, >=, <>, LLIKE, RLIKE, LIKE, IN, NOT IN, (
// val argument value, 可能是字符串, 也可能是数组, 也可能Where对象
func (q *Query) whereNode(l string, field string, m string, val ...interface{}) *Query {
	switch m {
	case "=", "<", "<=", ">", ">=", "<>", "LIKE":
		q.sql.Where += fmt.Sprintf(" %s %s %s ?", l, field, m)
		q.sql.Args = append(q.sql.Args, val[0])
	case "IN", "NOT IN":
		w := make([]string, len(val))
		for i, v := range val {
			w[i] = "?"
			q.sql.Args = append(q.sql.Args, v)
		}
		q.sql.Where += fmt.Sprintf(" %s %s %s (%s) ", l, field, m, strings.Join(w, ","))
	}

	return q
}

func (q *Query) Order() *Query {
	return q
}

func (q *Query) Group() *Query {

	return q
}

func (q *Query) Having() *Query {

	return q
}

func (q *Query) Limit(offset, rows uint64) *Query {

	return q
}

func (q *Query) Lock() *Query {

	return q
}

// First row, first column
func (q *Query) Scaler(ptr interface{}) error {

	return nil
}

// get one row
func (q *Query) Row(ptr interface{}) error {
	//get first row only
	q.sql.Offset = 0
	q.sql.Rows = 1

	return q.execute.Row(ptr, q.toString(), q.sql.Args)
}

// get all rows, return []struct or []map[string]interface{} or [][]interface{}
func (q *Query) Rows(ptr interface{}) error {
	return q.execute.Rows(ptr, q.toString(), q.sql.Args)
}

// Insert
func (q *Query) Insert(data *[]interface{}) (int64, error) {
	if len(*data) == 0 {
		err := errors.New("empty insert data")
		log.Printf("%s\n", err)
		return 0, err
	}

	var insertVal []string
	insertArgs := []interface{}{}
	insertArgsValue := reflect.Indirect(reflect.ValueOf(&insertArgs))

	for _, item := range *data {
		sli := []string{}

		//convert struct fields to sql string
		itemValue := reflect.ValueOf(item)
		itemType := itemValue.Type()
		for i := 0; i < itemType.NumField(); i++ {
			itemField := itemType.Field(i)
			if !itemField.Anonymous && itemField.Tag.Get("pk") == "" { // && itemField.Tag.Get("json") != ""
				sli = append(sli, "?")
				v := itemValue.FieldByIndex([]int{i})
				insertArgsValue.Set(reflect.Append(insertArgsValue, v))
			}
		}

		insertVal = append(insertVal, "("+strings.Join(sli, ", ")+")")
	}

	insertSql := fmt.Sprintf("INSERT INTO `%s` (`%s`) VALUES %s", q.table.Name, strings.Join(q.table.Fields, "`, `"), strings.Join(insertVal, ","))

	log.Printf("%s\n", insertSql)
	log.Printf("%#v\n", insertArgs)

	result, err := q.execute.Exec(insertSql, insertArgs)
	if err != nil {
		log.Printf("%s\n", err)
		return 0, err
	}

	return result.LastInsertId()
}

// Update
func (q *Query) Update(data *map[string]interface{}) (int64, error) {
	//convert map to sql string
	updateBody := []string{}
	updateArgs := []interface{}{}

	for key, element := range *data {
		updateBody = append(updateBody, key+" = ?")
		updateArgs = append(updateArgs, element)
	}

	updateSql := fmt.Sprintf("UPDATE %s SET %s WHERE 1 %s", q.table.Name, strings.Join(updateBody, ", "), q.sql.Where)

	updateArgs = append(updateArgs, q.sql.Args...)

	log.Printf("%s\n", updateSql)
	log.Printf("%#v\n", updateArgs)

	result, err := q.execute.Exec(updateSql, updateArgs)
	if err != nil {
		log.Printf("%s\n", err)
		return 0, err
	}

	return result.RowsAffected()
}

// Delete
func (q *Query) Delete() (int64, error) {
	deleteSql := fmt.Sprintf("DELETE FROM %s WHERE 1 %s", q.table.Name, q.sql.Where)

	log.Printf("%s\n", deleteSql)
	log.Printf("%#v\n", q.sql.Args)

	result, err := q.execute.Exec(deleteSql, q.sql.Args)
	if err != nil {
		log.Printf("%s\n", err)
		return 0, err
	}

	return result.RowsAffected()
}

// connect all sql department to a corect sql string.
func (q *Query) toString() string {
	//create sql
	var sel string
	if len(q.sql.Select) > 0 {
		sel = q.sql.Select //返回[]map[string]interface{}
	} else {
		sel = fmt.Sprintf("a.`%s`, a.`%s`", q.table.Primary, strings.Join(q.table.Fields, "`, a.`")) //返回[]EntityStruct
	}

	sql := fmt.Sprintf("SELECT %s FROM %s a %s WHERE 1 %s %s %s %s LIMIT %d, %d %s", sel, q.table.Name, q.sql.Join, q.sql.Where, q.sql.Order, q.sql.Group, q.sql.Having, q.sql.Offset, q.sql.Rows, q.sql.ForUpdate)

	log.Printf("%s\n", sql)
	log.Printf("%#v\n", q.sql.Args)

	return sql
}
<|endoftext|>


<commit_before>package qb

import (
	"fmt"
	"strings"
)

const defaultDelimiter = "\n"

// NewQuery creates a new query and returns its pointer
func NewQuery() *Query {
	return &Query{
		clauses:      []string{},
		bindings:     []interface{}{},
		delimiter:    defaultDelimiter,
		bindingIndex: 0,
	}
}

// Query is the base abstraction for sql queries
type Query struct {
	clauses      []string
	bindings     []interface{}
	delimiter    string
	bindingIndex int
}

// SetDelimiter sets the delimiter of query
func (q *Query) SetDelimiter(delimiter string) {
	q.delimiter = delimiter
}

// AddClause appends a new clause to current query
func (q *Query) AddClause(clause string) {
	q.clauses = append(q.clauses, clause)
}

// AddBinding appends a new binding to current query
func (q *Query) AddBinding(bindings ...interface{}) {
	for _, v := range bindings {
		q.bindings = append(q.bindings, v)
	}
}

// Clauses returns all clauses of current query
func (q *Query) Clauses() []string {
	return q.clauses
}

// Bindings returns all bindings of current query
func (q *Query) Bindings() []interface{} {
	return q.bindings
}

// Placeholder generates a placeholder for binding.
// If the driver is postgres generates "$i" where i is incremental integer
// Otherwise it generates "?"
func (q *Query) placeholder(driver string) string {
	if driver == "postgres" {
		q.bindingIndex++
		return fmt.Sprintf("$%d", q.bindingIndex)
	}
	return "?"
}

// QuestionMarks generates temporary question marks for each binding in the query
// This is for builder to be able to temporarily put question marks
// The driver is unknown in builder
func (q *Query) QuestionMarks(values ...interface{}) []string {
	marks := []string{}
	for _ = range values {
		marks = append(marks, "?")
	}
	return marks
}

// SQL returns the query struct sql statement
func (q *Query) SQL() string {

	if len(q.clauses) > 0 {
		sql := fmt.Sprintf("%s;", strings.Join(q.clauses, q.delimiter))
		return sql
	}

	return ""
}
<commit_msg>remove internal placeholder function of query<commit_after>package qb

import (
	"fmt"
	"strings"
)

const defaultDelimiter = "\n"

// NewQuery creates a new query and returns its pointer
func NewQuery() *Query {
	return &Query{
		clauses:      []string{},
		bindings:     []interface{}{},
		delimiter:    defaultDelimiter,
		bindingIndex: 0,
	}
}

// Query is the base abstraction for sql queries
type Query struct {
	clauses      []string
	bindings     []interface{}
	delimiter    string
	bindingIndex int
}

// SetDelimiter sets the delimiter of query
func (q *Query) SetDelimiter(delimiter string) {
	q.delimiter = delimiter
}

// AddClause appends a new clause to current query
func (q *Query) AddClause(clause string) {
	q.clauses = append(q.clauses, clause)
}

// AddBinding appends a new binding to current query
func (q *Query) AddBinding(bindings ...interface{}) {
	for _, v := range bindings {
		q.bindings = append(q.bindings, v)
	}
}

// Clauses returns all clauses of current query
func (q *Query) Clauses() []string {
	return q.clauses
}

// Bindings returns all bindings of current query
func (q *Query) Bindings() []interface{} {
	return q.bindings
}

// QuestionMarks generates temporary question marks for each binding in the query
// This is for builder to be able to temporarily put question marks
// The driver is unknown in builder
func (q *Query) QuestionMarks(values ...interface{}) []string {
	marks := []string{}
	for _ = range values {
		marks = append(marks, "?")
	}
	return marks
}

// SQL returns the query struct sql statement
func (q *Query) SQL() string {

	if len(q.clauses) > 0 {
		sql := fmt.Sprintf("%s;", strings.Join(q.clauses, q.delimiter))
		return sql
	}

	return ""
}
<|endoftext|>


<commit_before>package dht

import (
	"sync"

	peer "github.com/jbenet/go-ipfs/p2p/peer"
	queue "github.com/jbenet/go-ipfs/p2p/peer/queue"
	"github.com/jbenet/go-ipfs/routing"
	eventlog "github.com/jbenet/go-ipfs/thirdparty/eventlog"
	u "github.com/jbenet/go-ipfs/util"
	pset "github.com/jbenet/go-ipfs/util/peerset"
	todoctr "github.com/jbenet/go-ipfs/util/todocounter"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ctxgroup "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup"
)

var maxQueryConcurrency = AlphaValue

type dhtQuery struct {
	dht         *IpfsDHT
	key         u.Key     // the key we're querying for
	qfunc       queryFunc // the function to execute per peer
	concurrency int       // the concurrency parameter
}

type dhtQueryResult struct {
	value         []byte          // GetValue
	peer          peer.PeerInfo   // FindPeer
	providerPeers []peer.PeerInfo // GetProviders
	closerPeers   []peer.PeerInfo // *
	success       bool
}

// constructs query
func (dht *IpfsDHT) newQuery(k u.Key, f queryFunc) *dhtQuery {
	return &dhtQuery{
		key:         k,
		dht:         dht,
		qfunc:       f,
		concurrency: maxQueryConcurrency,
	}
}

// QueryFunc is a function that runs a particular query with a given peer.
// It returns either:
// - the value
// - a list of peers potentially better able to serve the query
// - an error
type queryFunc func(context.Context, peer.ID) (*dhtQueryResult, error)

// Run runs the query at hand. pass in a list of peers to use first.
func (q *dhtQuery) Run(ctx context.Context, peers []peer.ID) (*dhtQueryResult, error) {
	runner := newQueryRunner(ctx, q)
	return runner.Run(peers)
}

type dhtQueryRunner struct {
	query          *dhtQuery        // query to run
	peersSeen      *pset.PeerSet    // all peers queried. prevent querying same peer 2x
	peersToQuery   *queue.ChanQueue // peers remaining to be queried
	peersRemaining todoctr.Counter  // peersToQuery + currently processing

	result *dhtQueryResult // query result
	errs   u.MultiErr      // result errors. maybe should be a map[peer.ID]error

	rateLimit chan struct{} // processing semaphore
	log       eventlog.EventLogger

	cg ctxgroup.ContextGroup
	sync.RWMutex
}

func newQueryRunner(ctx context.Context, q *dhtQuery) *dhtQueryRunner {
	return &dhtQueryRunner{
		query:          q,
		peersToQuery:   queue.NewChanQueue(ctx, queue.NewXORDistancePQ(q.key)),
		peersRemaining: todoctr.NewSyncCounter(),
		peersSeen:      pset.New(),
		rateLimit:      make(chan struct{}, q.concurrency),
		cg:             ctxgroup.WithContext(ctx),
	}
}

func (r *dhtQueryRunner) Run(peers []peer.ID) (*dhtQueryResult, error) {
	log := log.Prefix("dht(%s).Query(%s).Run(%d)", r.query.dht.self, r.query.key, len(peers))
	r.log = log
	log.Debug("enter")
	defer log.Debug("end")

	log.Debugf("Run query with %d peers.", len(peers))
	if len(peers) == 0 {
		log.Warning("Running query with no peers!")
		return nil, nil
	}

	// setup concurrency rate limiting
	for i := 0; i < r.query.concurrency; i++ {
		r.rateLimit <- struct{}{}
	}

	// add all the peers we got first.
	for _, p := range peers {
		r.addPeerToQuery(r.cg.Context(), p)
	}

	// go do this thing.
	// do it as a child func to make sure Run exits
	// ONLY AFTER spawn workers has exited.
	log.Debugf("go spawn workers")
	r.cg.AddChildFunc(r.spawnWorkers)

	// so workers are working.

	// wait until they're done.
	err := routing.ErrNotFound

	select {
	case <-r.peersRemaining.Done():
		log.Debug("all peers ended")
		r.cg.Close()
		r.RLock()
		defer r.RUnlock()

		err = routing.ErrNotFound

		// if every query to every peer failed, something must be very wrong.
		if len(r.errs) > 0 && len(r.errs) == r.peersSeen.Size() {
			log.Debugf("query errs: %s", r.errs)
			err = r.errs[0]
		}

	case <-r.cg.Closed():
		log.Debug("r.cg.Closed()")

		r.RLock()
		defer r.RUnlock()
		err = r.cg.Context().Err() // collect the error.
	}

	if r.result != nil && r.result.success {
		log.Debug("success: %s", r.result)
		return r.result, nil
	}

	log.Debug("failure: %s", err)
	return nil, err
}

func (r *dhtQueryRunner) addPeerToQuery(ctx context.Context, next peer.ID) {
	// if new peer is ourselves...
	if next == r.query.dht.self {
		r.log.Debug("addPeerToQuery skip self")
		return
	}

	if !r.peersSeen.TryAdd(next) {
		r.log.Debugf("addPeerToQuery skip seen %s", next)
		return
	}

	r.log.Debugf("addPeerToQuery adding %s", next)
	r.peersRemaining.Increment(1)
	select {
	case r.peersToQuery.EnqChan <- next:
	case <-ctx.Done():
	}
}

func (r *dhtQueryRunner) spawnWorkers(parent ctxgroup.ContextGroup) {
	log := r.log.Prefix("spawnWorkers")
	log.Debugf("begin")
	defer log.Debugf("end")

	for {

		select {
		case <-r.peersRemaining.Done():
			return

		case <-r.cg.Closing():
			return

		case p, more := <-r.peersToQuery.DeqChan:
			if !more {
				return // channel closed.
			}
			log.Debugf("spawning worker for: %v", p)

			// do it as a child func to make sure Run exits
			// ONLY AFTER spawn workers has exited.
			parent.AddChildFunc(func(cg ctxgroup.ContextGroup) {
				r.queryPeer(cg, p)
			})
		}
	}
}

func (r *dhtQueryRunner) queryPeer(cg ctxgroup.ContextGroup, p peer.ID) {
	log := r.log.Prefix("queryPeer(%s)", p)
	log.Debugf("spawned")
	defer log.Debugf("finished")

	// make sure we rate limit concurrency.
	select {
	case <-r.rateLimit:
	case <-cg.Closing():
		r.peersRemaining.Decrement(1)
		return
	}

	// ok let's do this!
	log.Debugf("running")

	// make sure we do this when we exit
	defer func() {
		// signal we're done proccessing peer p
		log.Debugf("completed")
		r.peersRemaining.Decrement(1)
		r.rateLimit <- struct{}{}
	}()

	// make sure we're connected to the peer.
	if conns := r.query.dht.host.Network().ConnsToPeer(p); len(conns) == 0 {
		log.Infof("not connected. dialing.")

		pi := peer.PeerInfo{ID: p}
		if err := r.query.dht.host.Connect(cg.Context(), pi); err != nil {
			log.Debugf("Error connecting: %s", err)
			r.Lock()
			r.errs = append(r.errs, err)
			r.Unlock()
			return
		}

		log.Debugf("connected. dial success.")
	}

	// finally, run the query against this peer
	log.Debugf("query running")
	res, err := r.query.qfunc(cg.Context(), p)
	log.Debugf("query finished")

	if err != nil {
		log.Debugf("ERROR worker for: %v %v", p, err)
		r.Lock()
		r.errs = append(r.errs, err)
		r.Unlock()

	} else if res.success {
		log.Debugf("SUCCESS worker for: %v %s", p, res)
		r.Lock()
		r.result = res
		r.Unlock()
		go r.cg.Close() // signal to everyone that we're done.
		// must be async, as we're one of the children, and Close blocks.

	} else if len(res.closerPeers) > 0 {
		log.Debugf("PEERS CLOSER -- worker for: %v (%d closer peers)", p, len(res.closerPeers))
		for _, next := range res.closerPeers {
			if next.ID == r.query.dht.self { // dont add self.
				log.Debugf("PEERS CLOSER -- worker for: %v found self", p)
				continue
			}

			// add their addresses to the dialer's peerstore
			r.query.dht.peerstore.AddPeerInfo(next)
			r.addPeerToQuery(cg.Context(), next.ID)
			log.Debugf("PEERS CLOSER -- worker for: %v added %v (%v)", p, next.ID, next.Addrs)
		}
	} else {
		log.Debugf("QUERY worker for: %v - not found, and no closer peers.", p)
	}
}
<commit_msg>dont rate limit query during dials<commit_after>package dht

import (
	"sync"

	peer "github.com/jbenet/go-ipfs/p2p/peer"
	queue "github.com/jbenet/go-ipfs/p2p/peer/queue"
	"github.com/jbenet/go-ipfs/routing"
	eventlog "github.com/jbenet/go-ipfs/thirdparty/eventlog"
	u "github.com/jbenet/go-ipfs/util"
	pset "github.com/jbenet/go-ipfs/util/peerset"
	todoctr "github.com/jbenet/go-ipfs/util/todocounter"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ctxgroup "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup"
)

var maxQueryConcurrency = AlphaValue

type dhtQuery struct {
	dht         *IpfsDHT
	key         u.Key     // the key we're querying for
	qfunc       queryFunc // the function to execute per peer
	concurrency int       // the concurrency parameter
}

type dhtQueryResult struct {
	value         []byte          // GetValue
	peer          peer.PeerInfo   // FindPeer
	providerPeers []peer.PeerInfo // GetProviders
	closerPeers   []peer.PeerInfo // *
	success       bool
}

// constructs query
func (dht *IpfsDHT) newQuery(k u.Key, f queryFunc) *dhtQuery {
	return &dhtQuery{
		key:         k,
		dht:         dht,
		qfunc:       f,
		concurrency: maxQueryConcurrency,
	}
}

// QueryFunc is a function that runs a particular query with a given peer.
// It returns either:
// - the value
// - a list of peers potentially better able to serve the query
// - an error
type queryFunc func(context.Context, peer.ID) (*dhtQueryResult, error)

// Run runs the query at hand. pass in a list of peers to use first.
func (q *dhtQuery) Run(ctx context.Context, peers []peer.ID) (*dhtQueryResult, error) {
	runner := newQueryRunner(ctx, q)
	return runner.Run(peers)
}

type dhtQueryRunner struct {
	query          *dhtQuery        // query to run
	peersSeen      *pset.PeerSet    // all peers queried. prevent querying same peer 2x
	peersToQuery   *queue.ChanQueue // peers remaining to be queried
	peersRemaining todoctr.Counter  // peersToQuery + currently processing

	result *dhtQueryResult // query result
	errs   u.MultiErr      // result errors. maybe should be a map[peer.ID]error

	rateLimit chan struct{} // processing semaphore
	log       eventlog.EventLogger

	cg ctxgroup.ContextGroup
	sync.RWMutex
}

func newQueryRunner(ctx context.Context, q *dhtQuery) *dhtQueryRunner {
	return &dhtQueryRunner{
		query:          q,
		peersToQuery:   queue.NewChanQueue(ctx, queue.NewXORDistancePQ(q.key)),
		peersRemaining: todoctr.NewSyncCounter(),
		peersSeen:      pset.New(),
		rateLimit:      make(chan struct{}, q.concurrency),
		cg:             ctxgroup.WithContext(ctx),
	}
}

func (r *dhtQueryRunner) Run(peers []peer.ID) (*dhtQueryResult, error) {
	log := log.Prefix("dht(%s).Query(%s).Run(%d)", r.query.dht.self, r.query.key, len(peers))
	r.log = log
	log.Debug("enter")
	defer log.Debug("end")

	log.Debugf("Run query with %d peers.", len(peers))
	if len(peers) == 0 {
		log.Warning("Running query with no peers!")
		return nil, nil
	}

	// setup concurrency rate limiting
	for i := 0; i < r.query.concurrency; i++ {
		r.rateLimit <- struct{}{}
	}

	// add all the peers we got first.
	for _, p := range peers {
		r.addPeerToQuery(r.cg.Context(), p)
	}

	// go do this thing.
	// do it as a child func to make sure Run exits
	// ONLY AFTER spawn workers has exited.
	log.Debugf("go spawn workers")
	r.cg.AddChildFunc(r.spawnWorkers)

	// so workers are working.

	// wait until they're done.
	err := routing.ErrNotFound

	select {
	case <-r.peersRemaining.Done():
		log.Debug("all peers ended")
		r.cg.Close()
		r.RLock()
		defer r.RUnlock()

		err = routing.ErrNotFound

		// if every query to every peer failed, something must be very wrong.
		if len(r.errs) > 0 && len(r.errs) == r.peersSeen.Size() {
			log.Debugf("query errs: %s", r.errs)
			err = r.errs[0]
		}

	case <-r.cg.Closed():
		log.Debug("r.cg.Closed()")

		r.RLock()
		defer r.RUnlock()
		err = r.cg.Context().Err() // collect the error.
	}

	if r.result != nil && r.result.success {
		log.Debug("success: %s", r.result)
		return r.result, nil
	}

	log.Debug("failure: %s", err)
	return nil, err
}

func (r *dhtQueryRunner) addPeerToQuery(ctx context.Context, next peer.ID) {
	// if new peer is ourselves...
	if next == r.query.dht.self {
		r.log.Debug("addPeerToQuery skip self")
		return
	}

	if !r.peersSeen.TryAdd(next) {
		r.log.Debugf("addPeerToQuery skip seen %s", next)
		return
	}

	r.log.Debugf("addPeerToQuery adding %s", next)
	r.peersRemaining.Increment(1)
	select {
	case r.peersToQuery.EnqChan <- next:
	case <-ctx.Done():
	}
}

func (r *dhtQueryRunner) spawnWorkers(parent ctxgroup.ContextGroup) {
	log := r.log.Prefix("spawnWorkers")
	log.Debugf("begin")
	defer log.Debugf("end")

	for {

		select {
		case <-r.peersRemaining.Done():
			return

		case <-r.cg.Closing():
			return

		case p, more := <-r.peersToQuery.DeqChan:
			if !more {
				return // channel closed.
			}
			log.Debugf("spawning worker for: %v", p)

			// do it as a child func to make sure Run exits
			// ONLY AFTER spawn workers has exited.
			parent.AddChildFunc(func(cg ctxgroup.ContextGroup) {
				r.queryPeer(cg, p)
			})
		}
	}
}

func (r *dhtQueryRunner) queryPeer(cg ctxgroup.ContextGroup, p peer.ID) {
	log := r.log.Prefix("queryPeer(%s)", p)
	log.Debugf("spawned")
	defer log.Debugf("finished")

	// make sure we rate limit concurrency.
	select {
	case <-r.rateLimit:
	case <-cg.Closing():
		r.peersRemaining.Decrement(1)
		return
	}

	// ok let's do this!
	log.Debugf("running")

	// make sure we do this when we exit
	defer func() {
		// signal we're done proccessing peer p
		log.Debugf("completed")
		r.peersRemaining.Decrement(1)
		r.rateLimit <- struct{}{}
	}()

	// make sure we're connected to the peer.
	if conns := r.query.dht.host.Network().ConnsToPeer(p); len(conns) == 0 {
		log.Infof("not connected. dialing.")
		// while we dial, we do not take up a rate limit. this is to allow
		// forward progress during potentially very high latency dials.
		r.rateLimit <- struct{}{}

		pi := peer.PeerInfo{ID: p}
		if err := r.query.dht.host.Connect(cg.Context(), pi); err != nil {
			log.Debugf("Error connecting: %s", err)
			r.Lock()
			r.errs = append(r.errs, err)
			r.Unlock()
			<-r.rateLimit // need to grab it again, as we deferred.
			return
		}
		<-r.rateLimit // need to grab it again, as we deferred.
		log.Debugf("connected. dial success.")
	}

	// finally, run the query against this peer
	log.Debugf("query running")
	res, err := r.query.qfunc(cg.Context(), p)
	log.Debugf("query finished")

	if err != nil {
		log.Debugf("ERROR worker for: %v %v", p, err)
		r.Lock()
		r.errs = append(r.errs, err)
		r.Unlock()

	} else if res.success {
		log.Debugf("SUCCESS worker for: %v %s", p, res)
		r.Lock()
		r.result = res
		r.Unlock()
		go r.cg.Close() // signal to everyone that we're done.
		// must be async, as we're one of the children, and Close blocks.

	} else if len(res.closerPeers) > 0 {
		log.Debugf("PEERS CLOSER -- worker for: %v (%d closer peers)", p, len(res.closerPeers))
		for _, next := range res.closerPeers {
			if next.ID == r.query.dht.self { // dont add self.
				log.Debugf("PEERS CLOSER -- worker for: %v found self", p)
				continue
			}

			// add their addresses to the dialer's peerstore
			r.query.dht.peerstore.AddPeerInfo(next)
			r.addPeerToQuery(cg.Context(), next.ID)
			log.Debugf("PEERS CLOSER -- worker for: %v added %v (%v)", p, next.ID, next.Addrs)
		}
	} else {
		log.Debugf("QUERY worker for: %v - not found, and no closer peers.", p)
	}
}
<|endoftext|>


<commit_before>package resources

import (
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sns"
)

type SNSEndpoint struct {
	svc *sns.SNS
	ARN *string
}

func init() {
	register("SNSEndpoint", ListSNSEndpoints)
}

func ListSNSEndpoints(sess *session.Session) ([]Resource, error) {
	svc := sns.New(sess)
	resources := []Resource{}

	params := &sns.ListEndpointsByPlatformApplicationInput{}

	for {
		resp, err := svc.ListEndpointsByPlatformApplication(params)
		if err != nil {
			return nil, err
		}

		for _, endpoint := range resp.Endpoints {
			resources = append(resources, &SNSEndpoint{
				svc: svc,
				ARN: endpoint.EndpointArn,
			})
		}
		if resp.NextToken == nil {
			break
		}

		params.NextToken = resp.NextToken
	}
	return resources, nil
}

func (f *SNSEndpoint) Remove() error {

	_, err := f.svc.DeleteEndpoint(&sns.DeleteEndpointInput{
		EndpointArn: f.ARN,
	})

	return err
}

func (f *SNSEndpoint) String() string {
	return *f.ARN
}
<commit_msg>Fix missing required parameters<commit_after>package resources

import (
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sns"
)

type SNSEndpoint struct {
	svc *sns.SNS
	ARN *string
}

func init() {
	register("SNSEndpoint", ListSNSEndpoints)
}

func ListSNSEndpoints(sess *session.Session) ([]Resource, error) {
	svc := sns.New(sess)
	resources := []Resource{}
	platformApplications := []*sns.PlatformApplication{}

	platformParams := &sns.ListPlatformApplicationsInput{}

	for {
		resp, err := svc.ListPlatformApplications(platformParams)
		if err != nil {
			return nil, err
		}

		for _, platformApplication := range resp.PlatformApplications {
			platformApplications = append(platformApplications, platformApplication)
		}
		if resp.NextToken == nil {
			break
		}

		platformParams.NextToken = resp.NextToken

	}

	params := &sns.ListEndpointsByPlatformApplicationInput{}

	for _, platformApplication := range platformApplications {

		params.PlatformApplicationArn = platformApplication.PlatformApplicationArn

		resp, err := svc.ListEndpointsByPlatformApplication(params)
		if err != nil {
			return nil, err
		}

		for _, endpoint := range resp.Endpoints {
			resources = append(resources, &SNSEndpoint{
				svc: svc,
				ARN: endpoint.EndpointArn,
			})
		}
		if resp.NextToken == nil {
			break
		}

		params.NextToken = resp.NextToken
	}
	return resources, nil
}

func (f *SNSEndpoint) Remove() error {

	_, err := f.svc.DeleteEndpoint(&sns.DeleteEndpointInput{
		EndpointArn: f.ARN,
	})

	return err
}

func (f *SNSEndpoint) String() string {
	return *f.ARN
}
<|endoftext|>


<commit_before>package gservices

import (
	"math"
	"sync/atomic"
	"time"

	"github.com/HuKeping/rbtree"
	"github.com/gfandada/gserver/logger"
)

// 定时器初始值
const DEFAULT = time.Duration(math.MaxInt64)

// 自定义信号
var singal = struct{}{}

type LocalTimerServer struct {
	Id         uint64         // 自增的序列号
	JobList    *rbtree.Rbtree // 基于内存的事件容器
	Count      uint64         // 统计已执行的任务次数
	PauseChan  chan struct{}  // 暂停
	ResumeChan chan struct{}  // 重置
	ExitChan   chan struct{}  // 退出
}

type Ijob interface {
	Notify() <-chan Job // 获取一个chan:当job被执行时，可以从chan中获取消息
	GetCount() uint64   // 获取已经执行的次数
	GetTimes() uint64   // 获取允许执行的最大次数
}

type Job struct {
	Id           uint64          // 唯一id，由timerserver生成，用来区分同一时刻的不同事件
	Times        uint64          // 允许执行的最大次数:0表示无数次
	Count        uint64          // 表示已执行的次数
	IntervalTime time.Duration   // 间隔时间：支持ns
	CreateTime   time.Time       // 创建时间
	ActionTime   time.Time       // FIXME 计算得出的本次执行时间点，会有误差
	JobHandler   MessageHandler1 // 事件函数
	Args         []interface{}   // 函数调用参数
	MsgChan      chan Job        // 消息通道，执行时，控制器通过该通道向外部传递消息
}

func NewLocalTimerServer() *LocalTimerServer {
	server := &LocalTimerServer{
		JobList:    rbtree.New(),
		PauseChan:  make(chan struct{}, 0),
		ResumeChan: make(chan struct{}, 0),
		ExitChan:   make(chan struct{}, 0),
	}
	server.start()
	logger.Info("NewLocalTimerServer %v", server)
	return server
}

func (server *LocalTimerServer) start() {
	defalutJob := Job{
		CreateTime:   time.Now(),
		IntervalTime: time.Duration(math.MaxInt64),
	}
	server.addJob(time.Now(), defalutJob.IntervalTime, 1, defalutJob.JobHandler, nil)
	go server.schedule()
	server.resume()
}

func (server *LocalTimerServer) schedule() {
	timer := time.NewTimer(DEFAULT)
	defer timer.Stop()
PAUSE:
	<-server.ResumeChan
	for {
		value := server.JobList.Min()
		job, _ := value.(*Job)
		timeout := job.ActionTime.Sub(time.Now())
		timer.Reset(timeout)
		select {
		case <-timer.C:
			server.Count++
			job.ExecWithGo(true)
			if job.Times == 0 || job.Times > job.Count {
				server.JobList.Delete(job)
				job.ActionTime = job.ActionTime.Add(job.IntervalTime)
				server.JobList.Insert(job)
			} else {
				server.removeJob(job)
			}
		case <-server.PauseChan:
			goto PAUSE
		case <-server.ExitChan:
			goto EXIT
		}
	}
EXIT:
}

func (server *LocalTimerServer) resume() {
	server.ResumeChan <- singal
}

func (server *LocalTimerServer) exit() {
	server.ExitChan <- singal
}

func (server *LocalTimerServer) pause() {
	server.PauseChan <- singal
}

// 重设指定任务的超时时间
func (server *LocalTimerServer) UpdateJobTimeout(job Ijob, timeout time.Duration) bool {
	// 1s = 1e9 ns
	if timeout.Nanoseconds() <= 0 {
		return false
	}
	now := time.Now()
	server.pause()
	defer server.resume()
	item, ok := job.(*Job)
	if !ok {
		return false
	}
	server.JobList.Delete(item)
	item.ActionTime = now.Add(timeout)
	server.JobList.Insert(item)
	logger.Debug("UpdateJobTimeout job %v newTimeout %v", job, timeout)
	return true
}

// 添加单次任务，适用于单次定时业务逻辑
func (server *LocalTimerServer) AddJobWithInterval(timeout time.Duration, jobFunc MessageHandler1, args []interface{}) (Ijob, bool) {
	if timeout.Nanoseconds() <= 0 {
		return nil, false
	}
	server.pause()
	defer server.resume()
	logger.Debug("AddJobWithInterval timeout %v args %v", timeout, args)
	return server.addJob(time.Now(), timeout, 1, jobFunc, args)
}

// 添加单次任务，适用于特定时期的活动等类型的任务
func (server *LocalTimerServer) AddJobWithDeadtime(deadtime time.Time, jobFunc MessageHandler1, args []interface{}) (Ijob, bool) {
	now := time.Now()
	timeout := deadtime.Sub(now)
	if timeout.Nanoseconds() <= 0 {
		return nil, false
	}
	server.pause()
	defer server.resume()
	logger.Debug("AddJobWithDeadtime deadtime %v args %v", deadtime, args)
	return server.addJob(now, timeout, 1, jobFunc, args)
}

// 添加重复任务
func (server *LocalTimerServer) AddJobRepeat(jobInterval time.Duration, times uint64, jobFunc MessageHandler1, args []interface{}) (Ijob, bool) {
	if jobInterval.Nanoseconds() <= 0 {
		return nil, false
	}
	server.pause()
	defer server.resume()
	logger.Debug("AddJobRepeat jobInterval %v count %d args %v", jobInterval, times, args)
	return server.addJob(time.Now(), jobInterval, times, jobFunc, args)
}

// 移除指定的单项任务
func (server *LocalTimerServer) DelJob(job Ijob) bool {
	if job == nil {
		return false
	}
	server.pause()
	defer server.resume()
	item, ok := job.(*Job)
	if !ok {
		return false
	}
	server.removeJob(item)
	logger.Debug("DelJob %v", job)
	return true
}

// 移除指定的多项任务
func (server *LocalTimerServer) DelJobs(jobs []Ijob) {
	server.pause()
	defer server.resume()
	for _, job := range jobs {
		item, ok := job.(*Job)
		if !ok {
			continue
		}
		server.removeJob(item)
	}
	logger.Debug("DelJobs %v", jobs)
}

// 获取server当前执行次数
func (server *LocalTimerServer) GetCount() uint64 {
	return atomic.LoadUint64(&server.Count)
}

// 重置server的内部状态
func (server *LocalTimerServer) Reset() *LocalTimerServer {
	server.exit()
	server.Count = 0
	server.cleanJobs()
	server.start()
	logger.Info("gentimer Reset")
	return server
}

func (server *LocalTimerServer) cleanJobs() {
	item := server.JobList.Min()
	for item != nil {
		job, ok := item.(*Job)
		if ok {
			server.removeJob(job)
		}
		item = server.JobList.Min()
	}
}

// 获取pending的任务数量
func (server *LocalTimerServer) WaitJobs() uint {
	leng := server.JobList.Len() - 1
	if leng > 0 {
		return leng
	}
	return 0
}

func (server *LocalTimerServer) addJob(createTime time.Time, intervalTime time.Duration, jobTimes uint64, jobFunc MessageHandler1, args []interface{}) (job *Job, inserted bool) {
	inserted = true
	server.Id++
	job = &Job{
		Id:           server.Id,
		Times:        jobTimes,
		CreateTime:   createTime,
		ActionTime:   createTime.Add(intervalTime),
		IntervalTime: intervalTime,
		MsgChan:      make(chan Job, 10),
		JobHandler:   jobFunc,
		Args:         args,
	}
	server.JobList.Insert(job)
	return
}

func (server *LocalTimerServer) removeJob(job *Job) {
	server.JobList.Delete(job)
	close(job.MsgChan)
}

// 优雅的关闭
// 会依次执行一次（不按照actiontime）
func (server *LocalTimerServer) StopByGrace() {
	server.exit()
	server.immediate()
	logger.Warning("gentimer StopByGrace")
}

// 强制关闭
func (server *LocalTimerServer) StopByForce() {
	server.exit()
	server.cleanJobs()
	logger.Warning("gentimer StopByForce")
}

func (server *LocalTimerServer) immediate() {
	item := server.JobList.Min()
	for item != nil {
		job, ok := item.(*Job)
		if ok {
			atomic.AddUint64(&server.Count, 1)
			job.ExecWithGo(false)
			server.removeJob(job)
		}
		item = server.JobList.Min()
	}
}

/*************************************Job相关************************************/

/****************************实现了rbtree.Item接口****************************/

func (job Job) Less(another rbtree.Item) bool {
	item, ok := another.(*Job)
	if !ok {
		return false
	}
	if !job.ActionTime.Equal(item.ActionTime) {
		return job.ActionTime.Before(item.ActionTime)
	}
	return job.Id < item.Id
}

/********************************实现了Job接口***********************************/

func (job Job) Notify() <-chan Job {
	return job.MsgChan
}

func (job Job) GetCount() uint64 {
	return job.Count
}

func (job Job) GetTimes() uint64 {
	return job.Times
}

func (job *Job) action() {
	defer func() {
		if err := recover(); err != nil {
			panic(err)
		}
	}()
	job.JobHandler(job.Args)
	logger.Debug("action job %v ", job)
}

func (job *Job) ExecWithGo(isGo bool) {
	job.Count++
	if job.JobHandler == nil {
		return
	}
	if isGo {
		go job.action()
	} else {
		job.action()
	}
}
<commit_msg>fix action crash<commit_after>package gservices

import (
	"math"
	"sync/atomic"
	"time"

	"github.com/HuKeping/rbtree"
	"github.com/gfandada/gserver/logger"
)

// 定时器初始值
const DEFAULT = time.Duration(math.MaxInt64)

// 自定义信号
var singal = struct{}{}

type LocalTimerServer struct {
	Id         uint64         // 自增的序列号
	JobList    *rbtree.Rbtree // 基于内存的事件容器
	Count      uint64         // 统计已执行的任务次数
	PauseChan  chan struct{}  // 暂停
	ResumeChan chan struct{}  // 重置
	ExitChan   chan struct{}  // 退出
}

type Ijob interface {
	Notify() <-chan Job // 获取一个chan:当job被执行时，可以从chan中获取消息
	GetCount() uint64   // 获取已经执行的次数
	GetTimes() uint64   // 获取允许执行的最大次数
}

type Job struct {
	Id           uint64          // 唯一id，由timerserver生成，用来区分同一时刻的不同事件
	Times        uint64          // 允许执行的最大次数:0表示无数次
	Count        uint64          // 表示已执行的次数
	IntervalTime time.Duration   // 间隔时间：支持ns
	CreateTime   time.Time       // 创建时间
	ActionTime   time.Time       // FIXME 计算得出的本次执行时间点，会有误差
	JobHandler   MessageHandler1 // 事件函数
	Args         []interface{}   // 函数调用参数
	MsgChan      chan Job        // 消息通道，执行时，控制器通过该通道向外部传递消息
}

func NewLocalTimerServer() *LocalTimerServer {
	server := &LocalTimerServer{
		JobList:    rbtree.New(),
		PauseChan:  make(chan struct{}, 0),
		ResumeChan: make(chan struct{}, 0),
		ExitChan:   make(chan struct{}, 0),
	}
	server.start()
	logger.Info("NewLocalTimerServer %v", server)
	return server
}

func (server *LocalTimerServer) start() {
	defalutJob := Job{
		CreateTime:   time.Now(),
		IntervalTime: time.Duration(math.MaxInt64),
	}
	server.addJob(time.Now(), defalutJob.IntervalTime, 1, defalutJob.JobHandler, nil)
	go server.schedule()
	server.resume()
}

func (server *LocalTimerServer) schedule() {
	timer := time.NewTimer(DEFAULT)
	defer timer.Stop()
PAUSE:
	<-server.ResumeChan
	for {
		value := server.JobList.Min()
		job, _ := value.(*Job)
		timeout := job.ActionTime.Sub(time.Now())
		timer.Reset(timeout)
		select {
		case <-timer.C:
			server.Count++
			job.ExecWithGo(true)
			if job.Times == 0 || job.Times > job.Count {
				server.JobList.Delete(job)
				job.ActionTime = job.ActionTime.Add(job.IntervalTime)
				server.JobList.Insert(job)
			} else {
				server.removeJob(job)
			}
		case <-server.PauseChan:
			goto PAUSE
		case <-server.ExitChan:
			goto EXIT
		}
	}
EXIT:
}

func (server *LocalTimerServer) resume() {
	server.ResumeChan <- singal
}

func (server *LocalTimerServer) exit() {
	server.ExitChan <- singal
}

func (server *LocalTimerServer) pause() {
	server.PauseChan <- singal
}

// 重设指定任务的超时时间
func (server *LocalTimerServer) UpdateJobTimeout(job Ijob, timeout time.Duration) bool {
	// 1s = 1e9 ns
	if timeout.Nanoseconds() <= 0 {
		return false
	}
	now := time.Now()
	server.pause()
	defer server.resume()
	item, ok := job.(*Job)
	if !ok {
		return false
	}
	server.JobList.Delete(item)
	item.ActionTime = now.Add(timeout)
	server.JobList.Insert(item)
	logger.Debug("UpdateJobTimeout job %v newTimeout %v", job, timeout)
	return true
}

// 添加单次任务，适用于单次定时业务逻辑
func (server *LocalTimerServer) AddJobWithInterval(timeout time.Duration, jobFunc MessageHandler1, args []interface{}) (Ijob, bool) {
	if timeout.Nanoseconds() <= 0 {
		return nil, false
	}
	server.pause()
	defer server.resume()
	logger.Debug("AddJobWithInterval timeout %v args %v", timeout, args)
	return server.addJob(time.Now(), timeout, 1, jobFunc, args)
}

// 添加单次任务，适用于特定时期的活动等类型的任务
func (server *LocalTimerServer) AddJobWithDeadtime(deadtime time.Time, jobFunc MessageHandler1, args []interface{}) (Ijob, bool) {
	now := time.Now()
	timeout := deadtime.Sub(now)
	if timeout.Nanoseconds() <= 0 {
		return nil, false
	}
	server.pause()
	defer server.resume()
	logger.Debug("AddJobWithDeadtime deadtime %v args %v", deadtime, args)
	return server.addJob(now, timeout, 1, jobFunc, args)
}

// 添加重复任务
func (server *LocalTimerServer) AddJobRepeat(jobInterval time.Duration, times uint64, jobFunc MessageHandler1, args []interface{}) (Ijob, bool) {
	if jobInterval.Nanoseconds() <= 0 {
		return nil, false
	}
	server.pause()
	defer server.resume()
	logger.Debug("AddJobRepeat jobInterval %v count %d args %v", jobInterval, times, args)
	return server.addJob(time.Now(), jobInterval, times, jobFunc, args)
}

// 移除指定的单项任务
func (server *LocalTimerServer) DelJob(job Ijob) bool {
	if job == nil {
		return false
	}
	server.pause()
	defer server.resume()
	item, ok := job.(*Job)
	if !ok {
		return false
	}
	server.removeJob(item)
	logger.Debug("DelJob %v", job)
	return true
}

// 移除指定的多项任务
func (server *LocalTimerServer) DelJobs(jobs []Ijob) {
	server.pause()
	defer server.resume()
	for _, job := range jobs {
		item, ok := job.(*Job)
		if !ok {
			continue
		}
		server.removeJob(item)
	}
	logger.Debug("DelJobs %v", jobs)
}

// 获取server当前执行次数
func (server *LocalTimerServer) GetCount() uint64 {
	return atomic.LoadUint64(&server.Count)
}

// 重置server的内部状态
func (server *LocalTimerServer) Reset() *LocalTimerServer {
	server.exit()
	server.Count = 0
	server.cleanJobs()
	server.start()
	logger.Info("gentimer Reset")
	return server
}

func (server *LocalTimerServer) cleanJobs() {
	item := server.JobList.Min()
	for item != nil {
		job, ok := item.(*Job)
		if ok {
			server.removeJob(job)
		}
		item = server.JobList.Min()
	}
}

// 获取pending的任务数量
func (server *LocalTimerServer) WaitJobs() uint {
	leng := server.JobList.Len() - 1
	if leng > 0 {
		return leng
	}
	return 0
}

func (server *LocalTimerServer) addJob(createTime time.Time, intervalTime time.Duration, jobTimes uint64, jobFunc MessageHandler1, args []interface{}) (job *Job, inserted bool) {
	inserted = true
	server.Id++
	job = &Job{
		Id:           server.Id,
		Times:        jobTimes,
		CreateTime:   createTime,
		ActionTime:   createTime.Add(intervalTime),
		IntervalTime: intervalTime,
		MsgChan:      make(chan Job, 10),
		JobHandler:   jobFunc,
		Args:         args,
	}
	server.JobList.Insert(job)
	return
}

func (server *LocalTimerServer) removeJob(job *Job) {
	server.JobList.Delete(job)
	close(job.MsgChan)
}

// 优雅的关闭
// 会依次执行一次（不按照actiontime）
func (server *LocalTimerServer) StopByGrace() {
	server.exit()
	server.immediate()
	logger.Warning("gentimer StopByGrace")
}

// 强制关闭
func (server *LocalTimerServer) StopByForce() {
	server.exit()
	server.cleanJobs()
	logger.Warning("gentimer StopByForce")
}

func (server *LocalTimerServer) immediate() {
	item := server.JobList.Min()
	for item != nil {
		job, ok := item.(*Job)
		if ok {
			atomic.AddUint64(&server.Count, 1)
			job.ExecWithGo(false)
			server.removeJob(job)
		}
		item = server.JobList.Min()
	}
}

/*************************************Job相关************************************/

/****************************实现了rbtree.Item接口****************************/

func (job Job) Less(another rbtree.Item) bool {
	item, ok := another.(*Job)
	if !ok {
		return false
	}
	if !job.ActionTime.Equal(item.ActionTime) {
		return job.ActionTime.Before(item.ActionTime)
	}
	return job.Id < item.Id
}

/********************************实现了Job接口***********************************/

func (job Job) Notify() <-chan Job {
	return job.MsgChan
}

func (job Job) GetCount() uint64 {
	return job.Count
}

func (job Job) GetTimes() uint64 {
	return job.Times
}

func (job *Job) action() {
	defer func() {
		if err := recover(); err != nil {
			logger.Error("gentimer Exec job %v error: %v", job, err)
		}
	}()
	job.JobHandler(job.Args)
	logger.Debug("action job %v ", job)
}

func (job *Job) ExecWithGo(isGo bool) {
	job.Count++
	if job.JobHandler == nil {
		return
	}
	if isGo {
		go job.action()
	} else {
		job.action()
	}
}
<|endoftext|>


<commit_before>package vat

import (
	"encoding/json"
	"errors"
	"net/http"
	"time"
)

type RatePeriod struct {
	EffectiveFrom string `json:"effective_from"`
	Rates         map[string]float32
}

// CountryRates holds the various differing VAT rate periods for a given country
type CountryRates struct {
	Name        string `json:"name"`
	Code        string `json:"code"`
	CountryCode string `json:"country_code"`
	Periods     []RatePeriod
}

type apiResponse struct {
	Details string
	Version string
	Rates   []CountryRates
}

var countryRates []CountryRates

// ErrInvalidCountryCode will be returned when calling GetCountryRates with an invalid country code
var ErrInvalidCountryCode = errors.New("Unknown country code.")

// ErrInvalidRateLevel will be returned when getting wrong rate level
var ErrInvalidRateLevel = errors.New("Unknown rate level")

// RateOn returns the effective VAT rate on a given date
func (cr *CountryRates) RateOn(t time.Time, level string) (float32, error) {
	var activePeriod RatePeriod
	var activePeriodDate time.Time

	// find active period for the given time
	for _, p := range cr.Periods {
		periodDate, _ := time.Parse("2006-01-01", p.EffectiveFrom)
		if t.After(periodDate) && periodDate.After(activePeriodDate) {
			activePeriod = p
			activePeriodDate = periodDate
		}
	}

	activeRate, ok := activePeriod.Rates[level]
	if !ok {
		return 0.00, ErrInvalidRateLevel
	}

	return activeRate, nil
}

// Rate returns the currently active rate
func (cr *CountryRates) Rate(level string) (float32, error) {
	now := time.Now()
	return cr.RateOn(now, level)
}

// GetCountryRates gets the CountryRates struct for a country by its ISO-3166-1-alpha2 country code.
func GetCountryRates(c string) (CountryRates, error) {
	rates, err := fetch()
	var rate CountryRates

	if err != nil {
		return rate, err
	}

	for _, r := range rates {
		if r.CountryCode == c {
			return r, nil
		}
	}

	return rate, ErrInvalidCountryCode
}

func fetch() ([]CountryRates, error) {
	if countryRates != nil {
		return countryRates, nil
	}

	r, err := http.Get("https://jsonvat.com/")
	if err != nil {
		return nil, err
	}

	data := new(apiResponse)
	err = json.NewDecoder(r.Body).Decode(&data)
	countryRates = data.Rates

	return countryRates, err
}
<commit_msg>Export `FetchRates()` func so this can be called at first day of month (for example) to get latest VAT rates. Closes #1<commit_after>package vat

import (
	"encoding/json"
	"errors"
	"net/http"
	"time"
)

// RatePeriod represents a time and the various activate rates at that time.
type RatePeriod struct {
	EffectiveFromStr string `json:"effective_from"`
	EffectiveFrom    time.Time
	Rates            map[string]float32
}

// CountryRates holds the various differing VAT rate periods for a given country
type CountryRates struct {
	Name        string `json:"name"`
	Code        string `json:"code"`
	CountryCode string `json:"country_code"`
	Periods     []RatePeriod
}

var countriesRates []CountryRates

// ErrInvalidCountryCode will be returned when calling GetCountryRates with an invalid country code
var ErrInvalidCountryCode = errors.New("Unknown country code.")

// ErrInvalidRateLevel will be returned when getting wrong rate level
var ErrInvalidRateLevel = errors.New("Unknown rate level")

// RateOn returns the effective VAT rate on a given date
func (cr *CountryRates) RateOn(t time.Time, level string) (float32, error) {
	var activePeriod RatePeriod

	// find active period for the given time
	for _, p := range cr.Periods {
		if t.After(p.EffectiveFrom) && p.EffectiveFrom.After(activePeriod.EffectiveFrom) {
			activePeriod = p
		}
	}

	activeRate, ok := activePeriod.Rates[level]
	if !ok {
		return 0.00, ErrInvalidRateLevel
	}

	return activeRate, nil
}

// Rate returns the currently active rate
func (cr *CountryRates) Rate(level string) (float32, error) {
	now := time.Now()
	return cr.RateOn(now, level)
}

// GetCountryRates gets the CountryRates struct for a country by its ISO-3166-1-alpha2 country code.
func GetCountryRates(c string) (CountryRates, error) {
	var rate CountryRates
	rates, err := GetRates()

	if err != nil {
		return rate, err
	}

	for _, r := range rates {
		if r.CountryCode == c {
			return r, nil
		}
	}

	return rate, ErrInvalidCountryCode
}

// GetRates returns the in-memory VAT rates
func GetRates() ([]CountryRates, error) {
	var err error

	if countriesRates == nil {
		countriesRates, err = FetchRates()
	}

	return countriesRates, err
}

// FetchRates fetches the latest VAT rates from jsonvat.com and updates the in-memory rates
func FetchRates() ([]CountryRates, error) {

	r, err := http.Get("https://jsonvat.com/")
	if err != nil {
		return nil, err
	}

	type CR CountryRates
	apiResponse := &struct {
		Details string
		Version string
		Rates   []CountryRates
	}{}

	err = json.NewDecoder(r.Body).Decode(&apiResponse)

	// convert EffectiveFrom to a proper time.Time for each rate period
	for idx1, cr := range apiResponse.Rates {
		for idx2, crp := range cr.Periods {
			apiResponse.Rates[idx1].Periods[idx2].EffectiveFrom, _ = time.Parse("2006-01-01", crp.EffectiveFromStr)
		}
	}

	countriesRates = apiResponse.Rates
	return countriesRates, err
}
<|endoftext|>